diff --git a/src/main/java/fr/inra/oresing/checker/CheckerFactory.java b/src/main/java/fr/inra/oresing/checker/CheckerFactory.java index 1084dfb51349847127fd22ee9b28e4f23f324f1d..a82c5ae3683d1540278734008298cad21e7fed81 100644 --- a/src/main/java/fr/inra/oresing/checker/CheckerFactory.java +++ b/src/main/java/fr/inra/oresing/checker/CheckerFactory.java @@ -9,7 +9,6 @@ import fr.inra.oresing.model.Application; import fr.inra.oresing.model.Configuration; import fr.inra.oresing.model.ReferenceColumn; import fr.inra.oresing.model.VariableComponentKey; -import fr.inra.oresing.model.internationalization.InternationalizationDisplay; import fr.inra.oresing.persistence.DataRepository; import fr.inra.oresing.persistence.Ltree; import fr.inra.oresing.persistence.OreSiRepository; diff --git a/src/main/java/fr/inra/oresing/model/ReferenceColumnDisplayValue.java b/src/main/java/fr/inra/oresing/model/ReferenceColumnDisplayValue.java new file mode 100644 index 0000000000000000000000000000000000000000..5bfee2f749f3e70dfa3443dac2d84ff0408e0d25 --- /dev/null +++ b/src/main/java/fr/inra/oresing/model/ReferenceColumnDisplayValue.java @@ -0,0 +1,98 @@ +package fr.inra.oresing.model; + +import fr.inra.oresing.rest.ReferenceImporterContext; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.Value; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * Permet de stocker la valeur pour une colonne d'un référentiel lorsque cette colonne a une seule valeur associée ({@link fr.inra.oresing.checker.Multiplicity#ONE}). + */ +@Value +public class ReferenceColumnDisplayValue implements ReferenceColumnValue<String, ReferenceColumnDisplayValue.ReferenceColumnDisplayValueForLocale> { + + private static final ReferenceColumnDisplayValue EMPTY = new ReferenceColumnDisplayValue(); + + ReferenceColumnDisplayValueForLocale value = null; + + public ReferenceColumnDisplayValue() { + } + + /** + * Un {@link ReferenceColumnDisplayValue} vide (valeur non renseignée ?) + */ + public static ReferenceColumnDisplayValue empty() { + return EMPTY; + } + + @Override + public Collection<String> getValuesToCheck() { + return null; + } + + @Override + public ReferenceColumnValue<String, ReferenceColumnDisplayValueForLocale> transform(Function<String, String> transformation) { + return null; + } + + @Override + public String toValueString(ReferenceImporterContext referenceImporterContext, String referencedColumn, String key) { + return null; + } + + @Override + public ReferenceColumnDisplayValueForLocale toJsonForFrontend() { + return null; + } + + @Override + public String toJsonForDatabase() { + return null; + } + + + @Getter + @Setter + @ToString + public static class ReferenceColumnDisplayValueForLocale { + private String pattern; + private String toStringValue; + private Map<String, String> types; + private Map<String, ReferenceColumnDisplayToReplaceValue> values; + } + + @Getter + @Setter + @ToString + public abstract static class ReferenceColumnDisplayToReplaceValue<T> { + T value; + } + + @Getter + @Setter + @ToString + public static class ReferenceColumnDisplayToReplaceSingleValue extends ReferenceColumnDisplayToReplaceValue<String> { + + } + + @Getter + @Setter + @ToString + public static class ReferenceColumnDisplayToReplaceArrayValue extends ReferenceColumnDisplayToReplaceValue<List<String>> { + + } + + @Getter + @Setter + @ToString + public static class ReferenceColumnDisplayToReplaceMapValue extends ReferenceColumnDisplayToReplaceValue<Map<String, String>> { + + } + +} \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/model/ReferenceColumnIndexedValue.java b/src/main/java/fr/inra/oresing/model/ReferenceColumnIndexedValue.java index 04ad1f729c67b39e55385f07fe05f07471c802c8..765c69ffe75ef26158ff015b2ae2b118341f15df 100644 --- a/src/main/java/fr/inra/oresing/model/ReferenceColumnIndexedValue.java +++ b/src/main/java/fr/inra/oresing/model/ReferenceColumnIndexedValue.java @@ -1,8 +1,8 @@ package fr.inra.oresing.model; -import com.google.common.base.Joiner; import com.google.common.collect.Maps; import fr.inra.oresing.persistence.Ltree; +import fr.inra.oresing.rest.ReferenceImporterContext; import lombok.Value; import java.util.Collection; @@ -11,7 +11,7 @@ import java.util.function.Function; import java.util.stream.Collectors; @Value -public class ReferenceColumnIndexedValue implements ReferenceColumnValue<Map<String, String>> { +public class ReferenceColumnIndexedValue implements ReferenceColumnValue<Map<String, String>, Map<String, String>> { Map<Ltree, String> values; @@ -27,14 +27,25 @@ public class ReferenceColumnIndexedValue implements ReferenceColumnValue<Map<Str } @Override - public String toJsonForFrontend() { - return Joiner.on(",").withKeyValueSeparator("=").join(toJsonForDatabase()); + public String toValueString(ReferenceImporterContext referenceImporterContext, String referencedColumn, String locale) { + return values.entrySet().stream() + .map(ltreeStringEntry -> String.format("\"%s\"\"=%s\"", referenceImporterContext.getDisplayByReferenceAndNaturalKey( referencedColumn,ltreeStringEntry.getKey().toString(), locale), ltreeStringEntry.getValue())) + .collect(Collectors.joining(",","[","]")); + } + + @Override + public Map<String, String> toJsonForFrontend() { + return toStringStringMap(); } @Override public Map<String, String> toJsonForDatabase() { + return toStringStringMap(); + } + + private Map<String, String> toStringStringMap() { Map<String, String> jsonForDatabase = values.entrySet().stream() .collect(Collectors.toMap(entry -> entry.getKey().getSql(), Map.Entry::getValue)); return jsonForDatabase; } -} +} \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/model/ReferenceColumnMultipleValue.java b/src/main/java/fr/inra/oresing/model/ReferenceColumnMultipleValue.java index cb6e7e87ef02bfdada911e76f38509b615005111..1334c4899bdb05cabfaeab4f3a52730af5ea7b8a 100644 --- a/src/main/java/fr/inra/oresing/model/ReferenceColumnMultipleValue.java +++ b/src/main/java/fr/inra/oresing/model/ReferenceColumnMultipleValue.java @@ -1,6 +1,6 @@ package fr.inra.oresing.model; -import com.google.common.base.Preconditions; +import fr.inra.oresing.rest.ReferenceImporterContext; import lombok.Value; import java.util.Set; @@ -11,7 +11,7 @@ import java.util.stream.Collectors; * Permet de stocker la valeur pour une colonne d'un référentiel lorsque cette colonne est multi-valuées ({@link fr.inra.oresing.checker.Multiplicity#MANY}). */ @Value -public class ReferenceColumnMultipleValue implements ReferenceColumnValue<Set<String>> { +public class ReferenceColumnMultipleValue implements ReferenceColumnValue<Set<String>, Set<String>> { private static final String COLLECTION_AS_JSON_STRING_SEPARATOR = ","; @@ -28,16 +28,20 @@ public class ReferenceColumnMultipleValue implements ReferenceColumnValue<Set<St } @Override - public ReferenceColumnValue<Set<String>> transform(Function<String, String> transformation) { + public ReferenceColumnMultipleValue transform(Function<String, String> transformation) { Set<String> transformedValues = values.stream().map(transformation).collect(Collectors.toSet()); return new ReferenceColumnMultipleValue(transformedValues); } @Override - public String toJsonForFrontend() { - String jsonContent = values.stream() - .peek(value -> Preconditions.checkState(value.contains(COLLECTION_AS_JSON_STRING_SEPARATOR), value + " contient " + COLLECTION_AS_JSON_STRING_SEPARATOR)) - .collect(Collectors.joining(COLLECTION_AS_JSON_STRING_SEPARATOR)); - return jsonContent; + public String toValueString(ReferenceImporterContext referenceImporterContext, String referencedColumn, String locale) { + return values.stream() + .map(s->referenceImporterContext.getDisplayByReferenceAndNaturalKey(referencedColumn, s, locale)) + .collect(Collectors.joining(",","[","]")); } -} + + @Override + public Set<String> toJsonForFrontend() { + return values; + } +} \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/model/ReferenceColumnSingleValue.java b/src/main/java/fr/inra/oresing/model/ReferenceColumnSingleValue.java index 5b50bfbc4144f4e49466863ac131f5944b272551..4e8166ff8cc9fa7f7dea81783ae0bf6625b2fbbb 100644 --- a/src/main/java/fr/inra/oresing/model/ReferenceColumnSingleValue.java +++ b/src/main/java/fr/inra/oresing/model/ReferenceColumnSingleValue.java @@ -1,5 +1,6 @@ package fr.inra.oresing.model; +import fr.inra.oresing.rest.ReferenceImporterContext; import lombok.Value; import java.util.Set; @@ -9,7 +10,7 @@ import java.util.function.Function; * Permet de stocker la valeur pour une colonne d'un référentiel lorsque cette colonne a une seule valeur associée ({@link fr.inra.oresing.checker.Multiplicity#ONE}). */ @Value -public class ReferenceColumnSingleValue implements ReferenceColumnValue<String> { +public class ReferenceColumnSingleValue implements ReferenceColumnValue<String, String> { private static final ReferenceColumnSingleValue EMPTY = new ReferenceColumnSingleValue(""); @@ -33,13 +34,18 @@ public class ReferenceColumnSingleValue implements ReferenceColumnValue<String> } @Override - public ReferenceColumnValue<String> transform(Function<String, String> transformation) { + public ReferenceColumnSingleValue transform(Function<String, String> transformation) { String transformedValue = transformation.apply(value); return new ReferenceColumnSingleValue(transformedValue); } + @Override + public String toValueString(ReferenceImporterContext referenceImporterContext, String referencedColumn, String locale) { + return referenceImporterContext.getDisplayByReferenceAndNaturalKey(referencedColumn, value, locale); + } + @Override public String toJsonForFrontend() { return value; } -} +} \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/model/ReferenceColumnValue.java b/src/main/java/fr/inra/oresing/model/ReferenceColumnValue.java index 2ea25c8b4c8e2e4b460f719ab3cfae8fbf7dc0f4..ea10fcbd00f5c512af7e70c00c6482c410151293 100644 --- a/src/main/java/fr/inra/oresing/model/ReferenceColumnValue.java +++ b/src/main/java/fr/inra/oresing/model/ReferenceColumnValue.java @@ -1,14 +1,29 @@ package fr.inra.oresing.model; +import fr.inra.oresing.rest.ReferenceImporterContext; + import java.util.Collection; import java.util.function.Function; -public interface ReferenceColumnValue<T> extends SomethingToBeStoredAsJsonInDatabase<T>, SomethingToBeSentToFrontend<String> { +/** + * Représente la valeur pour une colonne donnée d'une ligne d'un référentiel donné. + * + * Voir les sous-classes qui gère chacune une forme de multiplicité. + * + * @param <T> le type dans lequel ça va être transformé pour être stocké sous forme de JSON en base + * @param <F> le type dans lequel ça va être transformé pour être envoyé au frontend sous forme de JSON + */ +public interface ReferenceColumnValue<T, F> extends SomethingToBeStoredAsJsonInDatabase<T>, SomethingToBeSentToFrontend<F> { + /** + * L'ensemble des valeurs pour lesquelles il faut appliquer les checkers + */ Collection<String> getValuesToCheck(); - ReferenceColumnValue<T> transform(Function<String, String> transformation); + /** + * Une copie de l'objet mais après avoir appliqué une transformation sur toutes les valeurs contenues. + */ + ReferenceColumnValue<T, F> transform(Function<String, String> transformation); - @Override - String toJsonForFrontend(); -} + String toValueString(ReferenceImporterContext referenceImporterContext, String referencedColumn, String key); +} \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/model/ReferenceDatum.java b/src/main/java/fr/inra/oresing/model/ReferenceDatum.java index f32eb134ef1a40465c7d89654b602d1771518a16..8ddfc1984eea079b64318a4abf9b426ef8d11734 100644 --- a/src/main/java/fr/inra/oresing/model/ReferenceDatum.java +++ b/src/main/java/fr/inra/oresing/model/ReferenceDatum.java @@ -12,7 +12,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext, SomethingToBeStoredAsJsonInDatabase<Map<String, Object>>, SomethingToBeSentToFrontend<Map<String, String>> { +public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext, SomethingToBeStoredAsJsonInDatabase<Map<String, Object>>, SomethingToBeSentToFrontend<Map<String, Object>> { private final Map<ReferenceColumn, ReferenceColumnValue> values; @@ -108,10 +108,10 @@ public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext, } @Override - public Map<String, String> toJsonForFrontend() { - Map<String, String> map = new LinkedHashMap<>(); + public Map<String, Object> toJsonForFrontend() { + Map<String, Object> map = new LinkedHashMap<>(); for (Map.Entry<ReferenceColumn, ReferenceColumnValue> entry : values.entrySet()) { - String valueThatMayBeNull = Optional.ofNullable(entry.getValue()) + Object valueThatMayBeNull = Optional.ofNullable(entry.getValue()) .map(ReferenceColumnValue::toJsonForFrontend) .orElse(null); map.put(entry.getKey().toJsonForDatabase(), valueThatMayBeNull); diff --git a/src/main/java/fr/inra/oresing/model/internationalization/InternationalizationDisplay.java b/src/main/java/fr/inra/oresing/model/internationalization/InternationalizationDisplay.java index 9ce92c431a73f0ddb04fcb471ed0f409c77c80b5..5ecf041b313652cfad9c64dcc4e08a719367157c 100644 --- a/src/main/java/fr/inra/oresing/model/internationalization/InternationalizationDisplay.java +++ b/src/main/java/fr/inra/oresing/model/internationalization/InternationalizationDisplay.java @@ -3,6 +3,7 @@ package fr.inra.oresing.model.internationalization; import fr.inra.oresing.model.ReferenceColumn; import fr.inra.oresing.model.ReferenceColumnSingleValue; import fr.inra.oresing.model.ReferenceDatum; +import fr.inra.oresing.rest.ReferenceImporterContext; import lombok.Getter; import lombok.Setter; import org.assertj.core.util.Strings; @@ -18,7 +19,10 @@ import java.util.stream.Stream; public class InternationalizationDisplay { Map<String, String> pattern; - public static ReferenceDatum getDisplays(Optional<Map<String, String>> displayPattern, Map<String, Internationalization> displayColumns, ReferenceDatum refValues) { + public static ReferenceDatum getDisplays(ReferenceImporterContext referenceImporterContext, ReferenceDatum refValues) { + Optional<Map<String, String>> displayPattern =referenceImporterContext.getDisplayPattern(); + Map<String, Internationalization> displayColumns = referenceImporterContext.getDisplayColumns(); + String refType = referenceImporterContext.getRefType(); ReferenceDatum displays = new ReferenceDatum(); displayPattern .ifPresent(patterns -> { @@ -33,7 +37,7 @@ public class InternationalizationDisplay { if (displayColumns.containsKey(referencedColumn)) { referencedColumn = displayColumns.get(referencedColumn).getOrDefault(stringEntry.getKey(), referencedColumn); } - internationalizedPattern += refValues.get(new ReferenceColumn(referencedColumn)); + internationalizedPattern += refValues.get(new ReferenceColumn(referencedColumn)).toValueString(referenceImporterContext, referencedColumn, stringEntry.getKey()); } return internationalizedPattern; } diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java index 42f743b54d8bd2f2e3edde8220e8aded0daf35db..3c9a7bd74f3227af9052c57715f1dedf5d218ec2 100644 --- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java +++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java @@ -15,6 +15,7 @@ import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import java.sql.PreparedStatement; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -38,7 +39,14 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi @Override protected String getUpsertQuery() { - return "INSERT INTO " + getTable().getSqlIdentifier() + "\n" + "(id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile) \n" + "SELECT id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile \n" + "FROM json_populate_recordset(NULL::" + getTable().getSqlIdentifier() + ", \n" + ":json::json) \n" + " ON CONFLICT ON CONSTRAINT \"hierarchicalKey_uniqueness\" \n" + "DO UPDATE SET updateDate=current_timestamp, hierarchicalKey=EXCLUDED.hierarchicalKey, hierarchicalReference=EXCLUDED.hierarchicalReference, naturalKey=EXCLUDED.naturalKey, refsLinkedTo=EXCLUDED.refsLinkedTo, refValues=EXCLUDED.refValues, binaryFile=EXCLUDED.binaryFile" + " RETURNING id"; + return "INSERT INTO " + getTable().getSqlIdentifier() + "\n" + + "(id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile) \n" + + "SELECT id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile \n" + + "FROM json_populate_recordset(NULL::" + getTable().getSqlIdentifier() + ", \n" + + ":json::json) \n" + + " ON CONFLICT ON CONSTRAINT \"hierarchicalKey_uniqueness\" \n" + + "DO UPDATE SET updateDate=current_timestamp, hierarchicalKey=EXCLUDED.hierarchicalKey, hierarchicalReference=EXCLUDED.hierarchicalReference, naturalKey=EXCLUDED.naturalKey, refsLinkedTo=EXCLUDED.refsLinkedTo, refValues=EXCLUDED.refValues, binaryFile=EXCLUDED.binaryFile" + + " RETURNING id"; } @Override @@ -57,8 +65,10 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi */ public List<ReferenceValue> findAllByReferenceType(String refType, MultiValueMap<String, String> params) { MultiValueMap<String, String> toto = new LinkedMultiValueMap<>(); - String query = "SELECT DISTINCT '" + ReferenceValue.class.getName() + "' as \"@class\", to_jsonb(t) as json FROM " + getTable().getSqlIdentifier() + " t, jsonb_each_text(t.refvalues) kv WHERE application=:applicationId::uuid AND referenceType=:refType"; - MapSqlParameterSource paramSource = new MapSqlParameterSource("applicationId", getApplication().getId()).addValue("refType", refType); + String query = "SELECT DISTINCT '" + ReferenceValue.class.getName() + "' as \"@class\", to_jsonb(t) as json FROM " + + getTable().getSqlIdentifier() + " t, jsonb_each_text(t.refvalues) kv WHERE application=:applicationId::uuid AND referenceType=:refType"; + MapSqlParameterSource paramSource = new MapSqlParameterSource("applicationId", getApplication().getId()) + .addValue("refType", refType); AtomicInteger i = new AtomicInteger(); // kv.value='LPF' OR t.refvalues @> '{"esp_nom":"ALO"}'::jsonb @@ -66,12 +76,13 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi String k = e.getKey(); if (StringUtils.equalsAnyIgnoreCase("_row_id_", k)) { String collect = e.getValue().stream().map(v -> { - String arg = ":arg" + i.getAndIncrement(); - paramSource.addValue(arg, v); - return String.format("'%s'::uuid", v); - }).collect(Collectors.joining(", ")); + String arg = ":arg" + i.getAndIncrement(); + paramSource.addValue(arg, v); + return String.format("'%s'::uuid", v); + }) + .collect(Collectors.joining(", ")); return Stream.ofNullable(String.format("array[id]::uuid[] <@ array[%s]::uuid[]", collect)); - } else if (StringUtils.equalsAnyIgnoreCase("any", k)) { + }else if (StringUtils.equalsAnyIgnoreCase("any", k)) { return e.getValue().stream().map(v -> { String arg = ":arg" + i.getAndIncrement(); paramSource.addValue(arg, v); @@ -80,7 +91,9 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi } else { return e.getValue().stream().map(v -> "t.refvalues @> '{\"" + k + "\":\"" + v + "\"}'::jsonb"); } - }).filter(k -> k != null).collect(Collectors.joining(" OR ")); + }) + .filter(k->k!=null). + collect(Collectors.joining(" OR ")); if (StringUtils.isNotBlank(cond)) { cond = " AND (" + cond + ")"; @@ -90,12 +103,51 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi return (List<ReferenceValue>) result; } + public Map<String, Map<String, String>> findDisplayByNaturalKey(String refType) { + String query = "select 'java.util.Map' as \"@class\" , jsonb_build_object(naturalkey, jsonb_agg(display)) json\n" + + " from " + getTable().getSqlIdentifier() + ",\n" + + "lateral\n" + + "(select jsonb_build_object(\n" + + " trim('\"__display_' from\n" + + " jsonb_path_query(refvalues, '$.keyvalue()?(@.key like_regex \"__display.*\").key')::text),\n" + + " trim('\"' FROM jsonb_path_query(refvalues, '$.keyvalue()?(@.key like_regex \"__display.*\").value')::text)\n" + + " ) as display\n" + + " )displays\n" + + "where referencetype = :refType\n" + + "group by naturalkey"; + Map<String, Map<String, String>> displayForNaturalKey = new HashMap<>(); + List result = getNamedParameterJdbcTemplate().query(query, new MapSqlParameterSource("refType", refType), getJsonRowMapper()); + for (Object o : result) { + final Map<String, List<Map<String, String>>> o1 = (Map<String, List<Map<String, String>>>) o; + final Map<String, Map<String, String>> collect = o1.entrySet() + .stream().collect(Collectors.toMap( + e -> e.getKey(), + e -> { + Map<String, String> displayMap = new HashMap<>(); + for (Map<String, String> s : e.getValue()) { + displayMap.putAll(s); + } + return displayMap; + } + )); + displayForNaturalKey.putAll(collect); + } + return displayForNaturalKey; + } + public List<List<String>> findReferenceValue(String refType, String column) { AtomicInteger ai = new AtomicInteger(0); - String select = Stream.of(column.split(",")).map(c -> String.format("refValues->>'%1$s' as \"%1$s" + ai.getAndIncrement() + "\"", c)).collect(Collectors.joining(", ")); - String sqlPattern = " SELECT %s " + " FROM " + getTable().getSqlIdentifier() + " t" + " WHERE application=:applicationId::uuid AND referenceType=:refType"; + String select = Stream.of(column.split(",")) + .map(c -> String.format("refValues->>'%1$s' as \"%1$s"+ai.getAndIncrement()+"\"", c)) + .collect(Collectors.joining(", ")); + String sqlPattern = " SELECT %s " + + " FROM " + getTable().getSqlIdentifier() + " t" + + " WHERE application=:applicationId::uuid AND referenceType=:refType"; String query = String.format(sqlPattern, select); - List<List<String>> result = getNamedParameterJdbcTemplate().queryForList(query, new MapSqlParameterSource("applicationId", getApplication().getId()).addValue("refType", refType)).stream().map(m -> m.values().stream().map(v -> (String) v).collect(Collectors.toList())).collect(Collectors.toList()); + List<List<String>> result = getNamedParameterJdbcTemplate().queryForList(query, new MapSqlParameterSource("applicationId", getApplication().getId()).addValue("refType", refType)) + .stream() + .map(m -> m.values().stream().map(v -> (String) v).collect(Collectors.toList())) + .collect(Collectors.toList()); return result; } @@ -111,7 +163,7 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi } else { display = null; } - Map<String, String> values = referenceDatum.toJsonForFrontend(); + Map<String, Object> values = referenceDatum.toJsonForFrontend(); return new ApplicationResult.Reference.ReferenceUUIDAndDisplay(display, result.getId(), values); }; return findAllByReferenceType(referenceType).stream().collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, referenceValueToReferenceUuidAndDisplayFunction)); diff --git a/src/main/java/fr/inra/oresing/rest/ApplicationResult.java b/src/main/java/fr/inra/oresing/rest/ApplicationResult.java index d01f968c83dd76801285743c9ee9559016b70b56..e63a155bcc973d6c22842441870077b2698c1269 100644 --- a/src/main/java/fr/inra/oresing/rest/ApplicationResult.java +++ b/src/main/java/fr/inra/oresing/rest/ApplicationResult.java @@ -41,7 +41,7 @@ public class ApplicationResult { public static class ReferenceUUIDAndDisplay { String display; UUID uuid; - Map<String, String> values; + Map<String, Object> values; } } diff --git a/src/main/java/fr/inra/oresing/rest/GetReferenceResult.java b/src/main/java/fr/inra/oresing/rest/GetReferenceResult.java index e7aa4f5e5a902915360aa45111f7bbe0debefcd0..2618763e7fb4673c9ab64aa16f6f35f228624210 100644 --- a/src/main/java/fr/inra/oresing/rest/GetReferenceResult.java +++ b/src/main/java/fr/inra/oresing/rest/GetReferenceResult.java @@ -14,6 +14,6 @@ public class GetReferenceResult { String hierarchicalKey; String hierarchicalReference; String naturalKey; - Map<String, String> values; + Map<String, Object> values; } } \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java index 1838bf246cffc585987ca6d5df58ee92e43e334b..77c68ad6e2e03facf75dd0d7feb3a9b11cb28889 100644 --- a/src/main/java/fr/inra/oresing/rest/OreSiService.java +++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java @@ -1,9 +1,6 @@ package fr.inra.oresing.rest; -import com.google.common.base.Charsets; -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; +import com.google.common.base.*; import com.google.common.collect.*; import com.google.common.primitives.Ints; import fr.inra.oresing.OreSiTechnicalException; @@ -29,7 +26,6 @@ import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.StringUtils; import org.assertj.core.util.Streams; -import org.assertj.core.util.Strings; import org.flywaydb.core.Flyway; import org.flywaydb.core.api.Location; import org.flywaydb.core.api.configuration.ClassicConfiguration; @@ -52,6 +48,7 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.*; +import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; import java.util.regex.Matcher; @@ -332,6 +329,7 @@ public class OreSiService { ReferenceValueRepository referenceValueRepository = repo.getRepository(app).referenceValue(); Configuration conf = app.getConfiguration(); ImmutableSet<LineChecker> lineCheckers = checkerFactory.getReferenceValidationLineCheckers(app, refType); + final ImmutableMap<Ltree, UUID> storedReferences = referenceValueRepository.getReferenceIdPerKeys(refType); ImmutableMap<ReferenceColumn, Multiplicity> multiplicityPerColumns = lineCheckers.stream() .filter(lineChecker -> lineChecker instanceof ReferenceLineChecker) @@ -388,14 +386,36 @@ public class OreSiService { ReferenceImporterContext.Column::getExpectedHeader, Function.identity() )); - + final ReferenceImporterContext.Constants constants = new ReferenceImporterContext.Constants( + app.getId(), + conf, + refType, + repo.getRepository(app).referenceValue()); + /* final Set<Object> patternColumns = constants.getPatternColumns() + .map(pt -> pt.values()) + .flatMap(Collection::stream) + .stream().collect(Collectors.toSet());*/ + Set<String> patternColumns = constants.getPatternColumns() + .map(m->m.values().stream().flatMap(List::stream).collect(Collectors.toSet())) + .orElseGet(HashSet::new); + final Map<String, String> referenceToColumnName = lineCheckers.stream() + .filter(ReferenceLineChecker.class::isInstance) + .map(ReferenceLineChecker.class::cast) + .collect(Collectors.toMap(ReferenceLineChecker::getRefType, referenceLineChecker -> ((ReferenceColumn) referenceLineChecker.getTarget().getTarget()).getColumn())); + final Map<String, Map<String, Map<String, String>>> displayByReferenceAndNaturalKey = + lineCheckers.stream() + .filter(ReferenceLineChecker.class::isInstance) + .map(ReferenceLineChecker.class::cast) + .map(ReferenceLineChecker::getRefType) + .filter(rt -> patternColumns.contains(rt)) + .collect(Collectors.toMap(ref -> referenceToColumnName.getOrDefault(ref, ref), ref -> repo.getRepository(app).referenceValue().findDisplayByNaturalKey(ref))); final ReferenceImporterContext referenceImporterContext = new ReferenceImporterContext( - app.getId(), - conf, - refType, + constants, lineCheckers, - columns + storedReferences, + columns, + displayByReferenceAndNaturalKey ); return referenceImporterContext; } @@ -1032,6 +1052,9 @@ public class OreSiService { /** * read some post header as example line, units, min and max values for each columns + * + * @param formatDescription + * @param linesIterator */ private void readPostHeader(Configuration.FormatDescription formatDescription, ImmutableList<String> headerRow, Datum constantValues, Iterator<CSVRecord> linesIterator) { ImmutableSetMultimap<Integer, Configuration.HeaderConstantDescription> perRowNumberConstants = @@ -1433,4 +1456,4 @@ public class OreSiService { int lineNumber; List<Map.Entry<String, String>> columns; } -} +} \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java index 8ea34b7efacda47ad890beed4114ef988aab6ea1..37ad672228d691d9625d1f39a94af74f1234ad63 100644 --- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java +++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java @@ -44,16 +44,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.io.InputStream; import java.time.format.DateTimeFormatter; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.UUID; +import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -74,6 +65,10 @@ abstract class ReferenceImporter { } } + public String getDisplayByReferenceAndNaturalKey(String referencedColumn, String naturalKey, String locale){ + return referenceImporterContext.getDisplayByReferenceAndNaturalKey(referencedColumn, naturalKey, locale); + } + /** * Importer le fichier passé en paramètre. * @@ -115,7 +110,8 @@ abstract class ReferenceImporter { boolean canSave = encounteredHierarchicalKeysForConflictDetection.get(hierarchicalKey).size() == 1; return canSave; }) - .map(keysAndReferenceDatumAfterChecking -> toEntity(keysAndReferenceDatumAfterChecking, fileId)); + .map(keysAndReferenceDatumAfterChecking -> toEntity(keysAndReferenceDatumAfterChecking, fileId)) + .sorted(Comparator.comparing(a -> a.getHierarchicalKey().getSql())); storeAll(referenceValuesStream); } @@ -284,9 +280,18 @@ abstract class ReferenceImporter { final ReferenceValue e = new ReferenceValue(); final Ltree naturalKey = keysAndReferenceDatumAfterChecking.getNaturalKey(); + recursionStrategy.getKnownId(naturalKey) + .ifPresent(e::setId); final Ltree hierarchicalReference = recursionStrategy.getHierarchicalReference(naturalKey); - referenceDatum.putAll(InternationalizationDisplay.getDisplays(referenceImporterContext.getDisplayPattern(), referenceImporterContext.getDisplayColumns(), referenceDatum)); + referenceDatum.putAll(InternationalizationDisplay.getDisplays(referenceImporterContext, referenceDatum)); + /** + * on remplace l'id par celle en base si elle existe + * a noter que pour les references récursives on récupère l'id depuis referenceLineChecker.getReferenceValues() ce qui revient au même + */ + + referenceImporterContext.getIdForSameHierarchicalKeyInDatabase(hierarchicalKey) + .ifPresent(e::setId); e.setBinaryFile(fileId); e.setReferenceType(referenceImporterContext.getRefType()); e.setHierarchicalKey(hierarchicalKey); @@ -358,6 +363,8 @@ abstract class ReferenceImporter { Ltree getHierarchicalReference(Ltree naturalKey); + Optional<UUID> getKnownId(Ltree naturalKey); + Stream<RowWithReferenceDatum> firstPass(Stream<RowWithReferenceDatum> streamBeforePreloading); } @@ -366,6 +373,16 @@ abstract class ReferenceImporter { private final Map<Ltree, Ltree> parentReferenceMap = new LinkedHashMap<>(); + private final Map<Ltree, UUID> afterPreloadReferenceUuids = new LinkedHashMap<>(); + + @Override + public Optional<UUID> getKnownId(Ltree naturalKey) { + if (afterPreloadReferenceUuids.containsKey(naturalKey)) { + return Optional.of(afterPreloadReferenceUuids.get(naturalKey)); + } + return Optional.empty(); + } + @Override public Ltree getHierarchicalKey(Ltree naturalKey, ReferenceDatum referenceDatum) { Ltree recursiveNaturalKey = getRecursiveNaturalKey(naturalKey); @@ -400,7 +417,7 @@ abstract class ReferenceImporter { final ReferenceColumn columnToLookForParentKey = referenceImporterContext.getColumnToLookForParentKey(); ReferenceLineChecker referenceLineChecker = referenceImporterContext.getReferenceLineChecker(); final ImmutableMap<Ltree, UUID> beforePreloadReferenceUuids = referenceLineChecker.getReferenceValues(); - final Map<Ltree, UUID> afterPreloadReferenceUuids = new LinkedHashMap<>(beforePreloadReferenceUuids); + afterPreloadReferenceUuids.putAll(beforePreloadReferenceUuids); ListMultimap<Ltree, Integer> missingParentReferences = LinkedListMultimap.create(); List<RowWithReferenceDatum> collect = streamBeforePreloading .peek(rowWithReferenceDatum -> { @@ -422,8 +439,7 @@ abstract class ReferenceImporter { missingParentReferences.removeAll(naturalKey); }) .collect(Collectors.toList()); - Set<Ltree> knownReferences = afterPreloadReferenceUuids.keySet(); - checkMissingParentReferencesIsEmpty(missingParentReferences, knownReferences); + checkMissingParentReferencesIsEmpty(missingParentReferences); referenceLineChecker.setReferenceValues(ImmutableMap.copyOf(afterPreloadReferenceUuids)); return collect.stream(); } @@ -433,13 +449,13 @@ abstract class ReferenceImporter { * * @param missingParentReferences pour chaque parent manquant, les lignes du CSV où il est mentionné */ - private void checkMissingParentReferencesIsEmpty(ListMultimap<Ltree, Integer> missingParentReferences, Set<Ltree> knownReferences) { + private void checkMissingParentReferencesIsEmpty(ListMultimap<Ltree, Integer> missingParentReferences) { List<CsvRowValidationCheckResult> rowErrors = missingParentReferences.entries().stream() .map(entry -> { Ltree missingParentReference = entry.getKey(); Integer lineNumber = entry.getValue(); ValidationCheckResult validationCheckResult = - new MissingParentLineValidationCheckResult(lineNumber, referenceImporterContext.getRefType(), missingParentReference, knownReferences); + new MissingParentLineValidationCheckResult(lineNumber, referenceImporterContext.getRefType(), missingParentReference, afterPreloadReferenceUuids.keySet()); return new CsvRowValidationCheckResult(validationCheckResult, lineNumber); }) .collect(Collectors.toUnmodifiableList()); @@ -449,6 +465,11 @@ abstract class ReferenceImporter { private class WithoutRecursion implements RecursionStrategy { + @Override + public Optional<UUID> getKnownId(Ltree naturalKey) { + return Optional.empty(); + } + @Override public Ltree getHierarchicalKey(Ltree naturalKey, ReferenceDatum referenceDatum) { return referenceImporterContext.newHierarchicalKey(naturalKey, referenceDatum); diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java index 57e858ca6f55ec57f4e008cdb7bc29dc1915dec3..8d5eadc3341d88e49fcd25f47678598994318571 100644 --- a/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java +++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java @@ -23,14 +23,10 @@ import fr.inra.oresing.model.internationalization.InternationalizationDisplay; import fr.inra.oresing.model.internationalization.InternationalizationMap; import fr.inra.oresing.model.internationalization.InternationalizationReferenceMap; import fr.inra.oresing.persistence.Ltree; +import fr.inra.oresing.persistence.ReferenceValueRepository; import lombok.AllArgsConstructor; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; /** @@ -40,51 +36,23 @@ import java.util.stream.Collectors; */ @AllArgsConstructor public class ReferenceImporterContext { - private static final String COMPOSITE_NATURAL_KEY_COMPONENTS_SEPARATOR = "__"; - - /** - * Identifiant de l'application à laquelle le référentiel importé appartient - */ - private final UUID applicationId; - - /** - * La configuration de l'application qui contient le référentiel mais aussi les utilisations de ce référentiel - */ - private final Configuration conf; - - /** - * Le nom du référentiel - */ - private final String refType; - + private final Constants constants; /** * Tous les {@link LineChecker} qui s'appliquent sur chaque ligne à importer */ private final ImmutableSet<LineChecker> lineCheckers; - + /** + * Les clés techniques de chaque clé naturelle hiérarchique de toutes les lignes existantes en base (avant l'import) + */ + private final ImmutableMap<Ltree, UUID> storedReferences; private final ImmutableMap<String, Column> columnsPerHeader; + Map<String, Map<String, Map<String, String>>> displayByReferenceAndNaturalKey; - private Optional<InternationalizationReferenceMap> getInternationalizationReferenceMap() { - Optional<InternationalizationReferenceMap> internationalizationReferenceMap = Optional.ofNullable(conf) - .map(Configuration::getInternationalization) - .map(InternationalizationMap::getReferences) - .map(references -> references.getOrDefault(refType, null)); - return internationalizationReferenceMap; - } - - public Map<String, Internationalization> getDisplayColumns() { - Optional<InternationalizationReferenceMap> internationalizationReferenceMap = getInternationalizationReferenceMap(); - return internationalizationReferenceMap - .map(InternationalizationReferenceMap::getInternationalizedColumns) - .orElseGet(HashMap::new); - } - - public Optional<Map<String, String>> getDisplayPattern() { - Optional<InternationalizationReferenceMap> internationalizationReferenceMap = getInternationalizationReferenceMap(); - return internationalizationReferenceMap - .map(InternationalizationReferenceMap::getInternationalizationDisplay) - .map(InternationalizationDisplay::getPattern); + public String getDisplayByReferenceAndNaturalKey(String referencedColumn, String naturalKey, String locale){ + return this.displayByReferenceAndNaturalKey.getOrDefault(referencedColumn, new HashMap<>()) + .getOrDefault(naturalKey, new HashMap<>()) + .getOrDefault(locale, naturalKey); } /** @@ -97,17 +65,12 @@ public class ReferenceImporterContext { return COMPOSITE_NATURAL_KEY_COMPONENTS_SEPARATOR; } - private HierarchicalKeyFactory getHierarchicalKeyFactory() { - HierarchicalKeyFactory hierarchicalKeyFactory = HierarchicalKeyFactory.build(conf, refType); - return hierarchicalKeyFactory; - } - public String getRefType() { - return refType; + return constants.getRefType(); } public Ltree getRefTypeAsLabel() { - return Ltree.fromUnescapedString(refType); + return Ltree.fromUnescapedString(getRefType()); } /** @@ -117,6 +80,10 @@ public class ReferenceImporterContext { return getHierarchicalKeyFactory().newHierarchicalKey(recursiveNaturalKey, referenceDatum); } + private HierarchicalKeyFactory getHierarchicalKeyFactory() { + return constants.getHierarchicalKeyFactory(); + } + /** * Crée une nom de référentiel hiérarchique */ @@ -135,12 +102,11 @@ public class ReferenceImporterContext { } private Configuration.ReferenceDescription getRef() { - Configuration.ReferenceDescription ref = conf.getReferences().get(refType); - return ref; + return constants.getRef(); } private Optional<Configuration.CompositeReferenceComponentDescription> getRecursiveComponentDescription() { - return conf.getCompositeReferences().values().stream() + return constants.getConf().getCompositeReferences().values().stream() .map(compositeReferenceDescription -> compositeReferenceDescription.getComponents().stream().filter(compositeReferenceComponentDescription -> getRefType().equals(compositeReferenceComponentDescription.getReference()) && compositeReferenceComponentDescription.getParentRecursiveKey() != null).findFirst().orElse(null)) .filter(e -> e != null) .findFirst(); @@ -188,7 +154,11 @@ public class ReferenceImporterContext { } public UUID getApplicationId() { - return applicationId; + return constants.getApplicationId(); + } + + public Optional<UUID> getIdForSameHierarchicalKeyInDatabase(Ltree hierarchicalKey) { + return Optional.ofNullable(storedReferences.get(hierarchicalKey)); } public void pushValue(ReferenceDatum referenceDatum, String header, String cellContent, SetMultimap<String, UUID> refsLinkedTo) { @@ -212,6 +182,124 @@ public class ReferenceImporterContext { return column.getCsvCellContent(referenceDatum); } + public Optional<Map<String, String>> getDisplayPattern() { + return constants.getDisplayPattern(); + } + + public Map<String, Internationalization> getDisplayColumns() { + return constants.getDisplayColumns(); + } + + public static class Constants { + /** + * Identifiant de l'application à laquelle le référentiel importé appartient + */ + private final UUID applicationId; + /** + * La configuration de l'application qui contient le référentiel mais aussi les utilisations de ce référentiel + */ + private final Configuration conf; + /** + * Le nom du référentiel + */ + private final String refType; + private final Optional<InternationalizationReferenceMap> internationalizationReferenceMap; + private final Map<String, Internationalization> displayColumns; + private final Optional<Map<String, String>> displayPattern; + private final HierarchicalKeyFactory hierarchicalKeyFactory; + private final Optional<Map<String, List<String>>> patternColumns; + private final Optional<Map<String, List<InternationalizationDisplay.PatternSection>>> patternSection; + Constants constants; + + public Constants(UUID applicationId, Configuration conf, String refType, ReferenceValueRepository referenceValueRepository) { + this.applicationId = applicationId; + this.conf = conf; + this.refType = refType; + this.internationalizationReferenceMap = buildInternationalizationReferenceMap(conf, refType); + this.displayColumns = buildDisplayColumns(); + this.displayPattern = buildDisplayPattern(); + this.hierarchicalKeyFactory = buildHierarchicalKeyFactory(); + this.patternColumns = this.buildPatternColumns(); + this.patternSection = this.buildPatternSection(); + } + + public Configuration.ReferenceDescription getRef() { + return conf.getReferences().get(refType); + } + + private Optional<InternationalizationReferenceMap> buildInternationalizationReferenceMap(Configuration conf, String refType) { + Optional<InternationalizationReferenceMap> internationalizationReferenceMap = Optional.ofNullable(conf) + .map(Configuration::getInternationalization) + .map(InternationalizationMap::getReferences) + .map(references -> references.getOrDefault(refType, null)); + return internationalizationReferenceMap; + } + + private Map<String, Internationalization> buildDisplayColumns() { + return this.internationalizationReferenceMap + .map(InternationalizationReferenceMap::getInternationalizedColumns) + .orElseGet(HashMap::new); + } + + private Optional<Map<String, String>> buildDisplayPattern() { + return this.internationalizationReferenceMap + .map(InternationalizationReferenceMap::getInternationalizationDisplay) + .map(InternationalizationDisplay::getPattern); + } + + + private HierarchicalKeyFactory buildHierarchicalKeyFactory() { + HierarchicalKeyFactory hierarchicalKeyFactory = HierarchicalKeyFactory.build(conf, refType); + return hierarchicalKeyFactory; + } + + private Optional<Map<String, List<InternationalizationDisplay.PatternSection>>> buildPatternSection() { + return displayPattern + .map(dp -> dp.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, k -> InternationalizationDisplay.parsePattern(k.getValue())))); + } + + private Optional<Map<String, List<String>>> buildPatternColumns() { + return displayPattern + .map(dp -> dp.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, k -> InternationalizationDisplay.getPatternColumns(k.getValue())))); + } + + public UUID getApplicationId() { + return applicationId; + } + + public Configuration getConf() { + return conf; + } + + public String getRefType() { + return refType; + } + + public Optional<InternationalizationReferenceMap> getInternationalizationReferenceMap() { + return internationalizationReferenceMap; + } + + public Map<String, Internationalization> getDisplayColumns() { + return displayColumns; + } + + public Optional<Map<String, String>> getDisplayPattern() { + return displayPattern; + } + + public HierarchicalKeyFactory getHierarchicalKeyFactory() { + return hierarchicalKeyFactory; + } + + public Optional<Map<String, List<String>>> getPatternColumns() { + return patternColumns; + } + + public Optional<Map<String, List<InternationalizationDisplay.PatternSection>>> getPatternSection() { + return patternSection; + } + } + /** * Contrat permettant de créer pour chaque ligne de référentiel sa clé hiérarchique. * <p> @@ -446,4 +534,4 @@ public class ReferenceImporterContext { return referenceColumnIndexedValue.getValues().get(expectedHierarchicalKey); } } -} +} \ No newline at end of file diff --git a/src/test/resources/data/acbb/acbb.yaml b/src/test/resources/data/acbb/acbb.yaml index 914b6522cc8296dd20f246316ed56c70a5558173..453775ed6c8faa0745c36ede6beab41b8c525544 100644 --- a/src/test/resources/data/acbb/acbb.yaml +++ b/src/test/resources/data/acbb/acbb.yaml @@ -86,6 +86,14 @@ references: nom_fr: nom_en: modalites: + internationalizedColumns: + nom_fr: + fr: nom_fr + en: nom_en + internationalizationDisplay: + pattern: + fr: '{nom_fr} ({code})' + en: '{nom_fr} ({code})' keyColumns: [code] columns: Variable de forcage: @@ -96,6 +104,10 @@ references: description_en: version_de_traitement: keyColumns: [site, traitement] + internationalizationDisplay: + pattern: + fr: '{traitement} ({modalites})' + en: '{traitement} ({modalites})' columns: site: traitement: