fix: suggester (#792)

* fix(suggester): use lunatic-model fixed version * fix(suggester): map synonyms as key/value pairs * fix(suggester): rules property with current string or array value... * fix(suggester): correct suggesters schema * refactor(treatments): split json schemas per treatment * refactor(suggester): rules property * test(suggester): update tests * chore: bump version
InseeFr · Nov 22, 2023 · 69f03f2 · 69f03f2
1 parent 85d31ed
commit 69f03f2
Show file tree

Hide file tree

Showing 22 changed files with 1,131 additions and 189 deletions.
diff --git a/build.gradle b/build.gradle
@@ -7,7 +7,7 @@ plugins {
 
 allprojects {
     group = 'fr.insee.eno'
-    version = '3.12.0'
+    version = '3.12.1-SNAPSHOT'
     sourceCompatibility = '17'
 }
 

diff --git a/eno-treatments/src/main/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializer.java b/eno-treatments/src/main/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializer.java
@@ -29,20 +29,17 @@ public class SpecificTreatmentsDeserializer {
     public SpecificTreatments deserialize(InputStream treatmentsStream) {
         ObjectMapper mapper = new ObjectMapper();
         JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012);
-        JsonSchema jsonSchema = factory.getSchema(
-                classLoader.getResourceAsStream("schema.treatments.json"));
+        JsonSchema suggesterSchema = factory.getSchema(
+                classLoader.getResourceAsStream("schema.suggesters.json"));
+        JsonSchema regroupingSchema = factory.getSchema(
+                classLoader.getResourceAsStream("schema.regrouping.json"));
 
         try {
             JsonNode jsonTreatments = mapper.readTree(treatmentsStream);
-            Set<ValidationMessage> errors = jsonSchema.validate(jsonTreatments);
-
-            if(!errors.isEmpty()) {
-                StringBuilder messageBuilder = new StringBuilder();
-                for(ValidationMessage errorMessage : errors) {
-                    messageBuilder.append(errorMessage.getMessage());
-                    messageBuilder.append("\n");
-                }
-                throw new SpecificTreatmentsValidationException(messageBuilder.toString());
+            if (jsonTreatments.has("suggesters"))
+                validateTreatmentInput(suggesterSchema, jsonTreatments.get("suggesters"));
+            if (jsonTreatments.has("regroupements")) {
+                validateTreatmentInput(regroupingSchema, jsonTreatments.get("regroupements"));
             }
 
             ObjectReader reader = mapper.readerFor(SpecificTreatments.class);
@@ -51,4 +48,18 @@ public SpecificTreatments deserialize(InputStream treatmentsStream) {
             throw new SpecificTreatmentsDeserializationException(ex.getMessage());
         }
     }
+
+    private static void validateTreatmentInput(JsonSchema suggesterSchema, JsonNode treatmentInput) {
+        Set<ValidationMessage> errors = suggesterSchema.validate(treatmentInput);
+
+        if(!errors.isEmpty()) {
+            StringBuilder messageBuilder = new StringBuilder();
+            for(ValidationMessage errorMessage : errors) {
+                messageBuilder.append(errorMessage.getMessage());
+                messageBuilder.append("\n");
+            }
+            throw new SpecificTreatmentsValidationException(messageBuilder.toString());
+        }
+    }
+
 }
diff --git a/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoFieldSynonym.java b/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoFieldSynonym.java
diff --git a/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoSuggesterField.java b/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoSuggesterField.java
@@ -3,12 +3,14 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import fr.insee.lunatic.model.flat.FieldRules;
 import fr.insee.lunatic.model.flat.SuggesterField;
-import lombok.*;
+import lombok.Data;
 
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 @Data
 public class EnoSuggesterField {
@@ -19,15 +21,15 @@ public class EnoSuggesterField {
     private String language;
     private BigInteger min;
     private Boolean stemmer;
-    private List<EnoFieldSynonym> synonyms;
+    private Map<String, List<String>> synonyms;
 
     @JsonCreator
     public EnoSuggesterField(@JsonProperty(value = "name", required = true) String name,
                              @JsonProperty("rules") List<String> rules,
                              @JsonProperty("language") String language,
                              @JsonProperty("min") BigInteger min,
                              @JsonProperty("stemmer") Boolean stemmer,
-                             @JsonProperty("synonyms") List<EnoFieldSynonym> synonyms) {
+                             @JsonProperty("synonyms") Map<String, List<String>> synonyms) {
         this.name = name;
         this.rules = rules;
         this.language = language;
@@ -53,16 +55,26 @@ public static SuggesterField toLunaticModel(EnoSuggesterField suggesterField) {
         lunaticField.setStemmer(suggesterField.getStemmer());
 
         if(suggesterField.getRules() != null) {
-            lunaticField.getRules().addAll(suggesterField.getRules());
+            lunaticField.setRules(new FieldRules());
+            if (isSoftRule(suggesterField.getRules())) {
+                lunaticField.getRules().setRule(FieldRules.SOFT_RULE);
+            } else {
+                suggesterField.getRules().forEach(pattern -> lunaticField.getRules().addPattern(pattern));
+            }
         }
 
         if(suggesterField.getSynonyms() != null) {
-            lunaticField.getSynonyms().addAll(EnoFieldSynonym.toLunaticModelList(suggesterField.getSynonyms()));
+            suggesterField.getSynonyms().forEach((stringKey, stringsValue) ->
+                lunaticField.getSynonyms().put(stringKey, stringsValue));
         }
 
         return lunaticField;
     }
 
+    private static boolean isSoftRule(List<String> enoFieldRules) {
+        return enoFieldRules.size() == 1 && FieldRules.SOFT_RULE.equals(enoFieldRules.get(0));
+    }
+
     /**
      * @param enoFields EnoSuggesterField list object to convert
      * @return the corresponding lunatic model list object

diff --git a/eno-treatments/src/main/resources/schema.regrouping.json b/eno-treatments/src/main/resources/schema.regrouping.json
@@ -0,0 +1,15 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://insee.fr/eno/regrouping.schema.json",
+  "title": "Regrouping specific treatments schema",
+  "description": "List of variable names to be grouped in the same page in the Lunatic questionnaire.",
+  "type": "array",
+  "items": {
+    "type": "array",
+    "items": {
+      "type": "string"
+    },
+    "minItems": 2,
+    "uniqueItems": true
+  }
+}
diff --git a/...src/main/resources/schema.treatments.json → ...src/main/resources/schema.suggesters.json b/...src/main/resources/schema.treatments.json → ...src/main/resources/schema.suggesters.json
@@ -1,29 +1,20 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://insee.fr/suggesters.schema.json",
-  "title": "Specific treatments schema",
-  "description": "Specific treatments schema",
-  "type": "object",
-  "properties": {
-    "suggesters": {
-      "type": "array",
-      "items": {
-        "$ref": "#/$defs/suggester"
-      }
-    },
-    "regroupements": {
-      "type": "array",
-      "items": {
-        "$ref": "#/$defs/regroupement"
-      }
-    }
+  "$id": "https://insee.fr/eno/suggesters.schema.json",
+  "title": "Lunatic suggesters' schema for Eno specific treatment",
+  "description": "Fits Lunatic suggesters property's schema, except a 'responseName' attribute is added here.",
+  "type": "array",
+  "items": {
+    "$ref": "#/$defs/suggester"
   },
 
   "$defs": {
+
     "suggester": {
       "type": "object",
       "properties": {
         "responseNames": {
+          "description": "Property that is not in Lunatic schema, used to associate suggester with components.",
           "type": "array",
           "items": {
             "type": "string"
@@ -33,15 +24,18 @@
         "name": {
           "type": "string"
         },
-        "queryParser": {
-          "$ref": "#/$defs/query_parser"
-        },
         "fields": {
           "type": "array",
           "items": {
             "$ref": "#/$defs/suggester_field"
           }
         },
+        "queryParser": {
+          "$ref": "#/$defs/query_parser"
+        },
+        "meloto": {
+          "type": "boolean"
+        },
         "stopWords": {
           "type": "array",
           "items": {
@@ -61,7 +55,7 @@
           "$ref": "#/$defs/suggester_order"
         }
       },
-      "required": ["responseNames", "name", "fields", "queryParser", "version"]
+      "required": ["name", "fields", "queryParser", "version"]
     },
 
     "suggester_field": {
@@ -71,32 +65,30 @@
           "type": "string"
         },
         "rules": {
-          "type": "array"
+          "type": ["string", "array"],
+          "pattern": "^soft$"
         },
         "min": {
           "type": "number"
         },
         "language": {
-          "type": "string"
+          "enum": ["French", "English"]
         },
         "stemmer": {
           "type": "boolean"
         },
         "synonyms": {
-          "type" : "array",
-          "items" : {
-            "$ref": "#/$defs/field_synonym"
-          }
+          "$ref": "#/$defs/field_synonym"
         }
       },
-      "required": ["name", "rules"]
+      "required": ["name"]
     },
 
     "query_parser": {
       "type": "object",
       "properties": {
         "type": {
-          "type": "string"
+          "enum": ["soft", "tokenized"]
         },
         "params": {
           "$ref": "#/$defs/query_parser_params"
@@ -109,7 +101,7 @@
       "type": "object",
       "properties": {
         "type": {
-          "type": "string"
+          "enum": ["ascending", "desceding"]
         },
         "field": {
           "type": "string"
@@ -121,8 +113,11 @@
     "query_parser_params": {
       "type": "object",
       "properties": {
+        "stemmer": {
+          "type": "boolean"
+        },
         "language": {
-          "type": "string"
+          "enum": ["French", "English"]
         },
         "min": {
           "type": "number"
@@ -132,25 +127,19 @@
         }
       }
     },
+
     "field_synonym": {
       "type": "object",
-      "properties": {
-        "source": {
-          "type": "string"
-        },
-        "target": {
+      "patternProperties": {
+        ".*" : {
           "type": "array",
           "items": {
             "type": "string"
           },
           "minItems": 1
-        },
-        "required": ["source", "target"]
+        }
       }
-    },
-    "regroupement": {
-      "type": "array",
-      "minItems": 2
     }
+
   }
 }