diff --git a/build.gradle b/build.gradle index f916736e9..668323935 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ plugins { allprojects { group = 'fr.insee.eno' - version = '3.12.0' + version = '3.12.1-SNAPSHOT' sourceCompatibility = '17' } diff --git a/eno-treatments/src/main/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializer.java b/eno-treatments/src/main/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializer.java index 0675e62f6..7c8474727 100644 --- a/eno-treatments/src/main/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializer.java +++ b/eno-treatments/src/main/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializer.java @@ -29,20 +29,17 @@ public class SpecificTreatmentsDeserializer { public SpecificTreatments deserialize(InputStream treatmentsStream) { ObjectMapper mapper = new ObjectMapper(); JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012); - JsonSchema jsonSchema = factory.getSchema( - classLoader.getResourceAsStream("schema.treatments.json")); + JsonSchema suggesterSchema = factory.getSchema( + classLoader.getResourceAsStream("schema.suggesters.json")); + JsonSchema regroupingSchema = factory.getSchema( + classLoader.getResourceAsStream("schema.regrouping.json")); try { JsonNode jsonTreatments = mapper.readTree(treatmentsStream); - Set errors = jsonSchema.validate(jsonTreatments); - - if(!errors.isEmpty()) { - StringBuilder messageBuilder = new StringBuilder(); - for(ValidationMessage errorMessage : errors) { - messageBuilder.append(errorMessage.getMessage()); - messageBuilder.append("\n"); - } - throw new SpecificTreatmentsValidationException(messageBuilder.toString()); + if (jsonTreatments.has("suggesters")) + validateTreatmentInput(suggesterSchema, jsonTreatments.get("suggesters")); + if (jsonTreatments.has("regroupements")) { + validateTreatmentInput(regroupingSchema, jsonTreatments.get("regroupements")); } ObjectReader reader = mapper.readerFor(SpecificTreatments.class); @@ -51,4 +48,18 @@ public SpecificTreatments deserialize(InputStream treatmentsStream) { throw new SpecificTreatmentsDeserializationException(ex.getMessage()); } } + + private static void validateTreatmentInput(JsonSchema suggesterSchema, JsonNode treatmentInput) { + Set errors = suggesterSchema.validate(treatmentInput); + + if(!errors.isEmpty()) { + StringBuilder messageBuilder = new StringBuilder(); + for(ValidationMessage errorMessage : errors) { + messageBuilder.append(errorMessage.getMessage()); + messageBuilder.append("\n"); + } + throw new SpecificTreatmentsValidationException(messageBuilder.toString()); + } + } + } diff --git a/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoFieldSynonym.java b/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoFieldSynonym.java deleted file mode 100644 index 2a5071ab9..000000000 --- a/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoFieldSynonym.java +++ /dev/null @@ -1,51 +0,0 @@ -package fr.insee.eno.treatments.dto; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import fr.insee.lunatic.model.flat.FieldSynonym; -import lombok.*; - -import java.util.ArrayList; -import java.util.List; - -@Data -public class EnoFieldSynonym { - - private String source; - private List target; - - @JsonCreator - public EnoFieldSynonym(@JsonProperty(value = "source", required = true) String source, - @JsonProperty(value = "target", required = true) List target) { - this.source = source; - this.target = target; - } - - /** - * - * @param enoFieldSynonym EnoFieldSynonym object - * @return the corresponding lunatic model object - */ - public static FieldSynonym toLunaticModel(EnoFieldSynonym enoFieldSynonym) { - if(enoFieldSynonym == null) { - return null; - } - FieldSynonym fieldSynonym = new FieldSynonym(); - fieldSynonym.setSource(enoFieldSynonym.getSource()); - fieldSynonym.getTarget().addAll(enoFieldSynonym.getTarget()); - return fieldSynonym; - } - - /** - * - * @param enoFieldSynonyms EnoFieldSynonym list object - * @return the corresponding lunatic model list - */ - public static List toLunaticModelList(List enoFieldSynonyms) { - if(enoFieldSynonyms == null) { - return new ArrayList<>(); - } - return enoFieldSynonyms.stream().map(EnoFieldSynonym::toLunaticModel).toList(); - } -} - diff --git a/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoSuggesterField.java b/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoSuggesterField.java index 094cfae2b..5b9c49bc9 100644 --- a/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoSuggesterField.java +++ b/eno-treatments/src/main/java/fr/insee/eno/treatments/dto/EnoSuggesterField.java @@ -3,12 +3,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; +import fr.insee.lunatic.model.flat.FieldRules; import fr.insee.lunatic.model.flat.SuggesterField; -import lombok.*; +import lombok.Data; import java.math.BigInteger; import java.util.ArrayList; import java.util.List; +import java.util.Map; @Data public class EnoSuggesterField { @@ -19,7 +21,7 @@ public class EnoSuggesterField { private String language; private BigInteger min; private Boolean stemmer; - private List synonyms; + private Map> synonyms; @JsonCreator public EnoSuggesterField(@JsonProperty(value = "name", required = true) String name, @@ -27,7 +29,7 @@ public EnoSuggesterField(@JsonProperty(value = "name", required = true) String n @JsonProperty("language") String language, @JsonProperty("min") BigInteger min, @JsonProperty("stemmer") Boolean stemmer, - @JsonProperty("synonyms") List synonyms) { + @JsonProperty("synonyms") Map> synonyms) { this.name = name; this.rules = rules; this.language = language; @@ -53,16 +55,26 @@ public static SuggesterField toLunaticModel(EnoSuggesterField suggesterField) { lunaticField.setStemmer(suggesterField.getStemmer()); if(suggesterField.getRules() != null) { - lunaticField.getRules().addAll(suggesterField.getRules()); + lunaticField.setRules(new FieldRules()); + if (isSoftRule(suggesterField.getRules())) { + lunaticField.getRules().setRule(FieldRules.SOFT_RULE); + } else { + suggesterField.getRules().forEach(pattern -> lunaticField.getRules().addPattern(pattern)); + } } if(suggesterField.getSynonyms() != null) { - lunaticField.getSynonyms().addAll(EnoFieldSynonym.toLunaticModelList(suggesterField.getSynonyms())); + suggesterField.getSynonyms().forEach((stringKey, stringsValue) -> + lunaticField.getSynonyms().put(stringKey, stringsValue)); } return lunaticField; } + private static boolean isSoftRule(List enoFieldRules) { + return enoFieldRules.size() == 1 && FieldRules.SOFT_RULE.equals(enoFieldRules.get(0)); + } + /** * @param enoFields EnoSuggesterField list object to convert * @return the corresponding lunatic model list object diff --git a/eno-treatments/src/main/resources/schema.regrouping.json b/eno-treatments/src/main/resources/schema.regrouping.json new file mode 100644 index 000000000..b67f3bf1f --- /dev/null +++ b/eno-treatments/src/main/resources/schema.regrouping.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://insee.fr/eno/regrouping.schema.json", + "title": "Regrouping specific treatments schema", + "description": "List of variable names to be grouped in the same page in the Lunatic questionnaire.", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2, + "uniqueItems": true + } +} \ No newline at end of file diff --git a/eno-treatments/src/main/resources/schema.treatments.json b/eno-treatments/src/main/resources/schema.suggesters.json similarity index 69% rename from eno-treatments/src/main/resources/schema.treatments.json rename to eno-treatments/src/main/resources/schema.suggesters.json index 93a4d6b96..768c5526c 100644 --- a/eno-treatments/src/main/resources/schema.treatments.json +++ b/eno-treatments/src/main/resources/schema.suggesters.json @@ -1,29 +1,20 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://insee.fr/suggesters.schema.json", - "title": "Specific treatments schema", - "description": "Specific treatments schema", - "type": "object", - "properties": { - "suggesters": { - "type": "array", - "items": { - "$ref": "#/$defs/suggester" - } - }, - "regroupements": { - "type": "array", - "items": { - "$ref": "#/$defs/regroupement" - } - } + "$id": "https://insee.fr/eno/suggesters.schema.json", + "title": "Lunatic suggesters' schema for Eno specific treatment", + "description": "Fits Lunatic suggesters property's schema, except a 'responseName' attribute is added here.", + "type": "array", + "items": { + "$ref": "#/$defs/suggester" }, "$defs": { + "suggester": { "type": "object", "properties": { "responseNames": { + "description": "Property that is not in Lunatic schema, used to associate suggester with components.", "type": "array", "items": { "type": "string" @@ -33,15 +24,18 @@ "name": { "type": "string" }, - "queryParser": { - "$ref": "#/$defs/query_parser" - }, "fields": { "type": "array", "items": { "$ref": "#/$defs/suggester_field" } }, + "queryParser": { + "$ref": "#/$defs/query_parser" + }, + "meloto": { + "type": "boolean" + }, "stopWords": { "type": "array", "items": { @@ -61,7 +55,7 @@ "$ref": "#/$defs/suggester_order" } }, - "required": ["responseNames", "name", "fields", "queryParser", "version"] + "required": ["name", "fields", "queryParser", "version"] }, "suggester_field": { @@ -71,32 +65,30 @@ "type": "string" }, "rules": { - "type": "array" + "type": ["string", "array"], + "pattern": "^soft$" }, "min": { "type": "number" }, "language": { - "type": "string" + "enum": ["French", "English"] }, "stemmer": { "type": "boolean" }, "synonyms": { - "type" : "array", - "items" : { - "$ref": "#/$defs/field_synonym" - } + "$ref": "#/$defs/field_synonym" } }, - "required": ["name", "rules"] + "required": ["name"] }, "query_parser": { "type": "object", "properties": { "type": { - "type": "string" + "enum": ["soft", "tokenized"] }, "params": { "$ref": "#/$defs/query_parser_params" @@ -109,7 +101,7 @@ "type": "object", "properties": { "type": { - "type": "string" + "enum": ["ascending", "desceding"] }, "field": { "type": "string" @@ -121,8 +113,11 @@ "query_parser_params": { "type": "object", "properties": { + "stemmer": { + "type": "boolean" + }, "language": { - "type": "string" + "enum": ["French", "English"] }, "min": { "type": "number" @@ -132,25 +127,19 @@ } } }, + "field_synonym": { "type": "object", - "properties": { - "source": { - "type": "string" - }, - "target": { + "patternProperties": { + ".*" : { "type": "array", "items": { "type": "string" }, "minItems": 1 - }, - "required": ["source", "target"] + } } - }, - "regroupement": { - "type": "array", - "minItems": 2 } + } } \ No newline at end of file diff --git a/eno-treatments/src/test/java/fr/insee/eno/treatments/JsonSchemaValidationTest.java b/eno-treatments/src/test/java/fr/insee/eno/treatments/JsonSchemaValidationTest.java index f16afacd8..a91d998c0 100644 --- a/eno-treatments/src/test/java/fr/insee/eno/treatments/JsonSchemaValidationTest.java +++ b/eno-treatments/src/test/java/fr/insee/eno/treatments/JsonSchemaValidationTest.java @@ -7,6 +7,8 @@ import com.networknt.schema.SpecVersion; import com.networknt.schema.ValidationMessage; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.util.Set; @@ -22,14 +24,51 @@ void givenValidJsonInput_whenValidating_thenNoErrors() throws IOException { ObjectMapper mapper = new ObjectMapper(); JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012); JsonSchema jsonSchema = factory.getSchema( - classLoader.getResourceAsStream("schema.treatments.json")); + classLoader.getResourceAsStream("schema.suggesters.json")); JsonNode jsonNode = mapper.readTree( - classLoader.getResourceAsStream("specific-treatments.json")); + classLoader.getResourceAsStream("specific-treatments.json")) + .get("suggesters"); + // Set errors = jsonSchema.validate(jsonNode); + // + assertTrue(errors.isEmpty()); + } - for(ValidationMessage error : errors) { - System.out.println(error.toString()); - } + @ParameterizedTest + @ValueSource(strings = { + "suggester-examples/suggesters-example.json", + "suggester-examples/suggesters-example-1.json", + "suggester-examples/suggesters-example-2.json", + "suggester-examples/suggesters-example-3.json", + "suggester-examples/suggesters-example-4.json", + }) + void suggestersSchemaTest(String relativeFilePath) throws IOException { + // + ObjectMapper mapper = new ObjectMapper(); + JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012); + JsonSchema jsonSchema = factory.getSchema( + classLoader.getResourceAsStream("schema.suggesters.json")); + JsonNode jsonNode = mapper.readTree( + classLoader.getResourceAsStream(relativeFilePath)); + // + Set errors = jsonSchema.validate(jsonNode); + // assertTrue(errors.isEmpty()); } + + @Test + void treatmentsSchema_suggestersTest() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012); + JsonSchema jsonSchema = factory.getSchema( + classLoader.getResourceAsStream("schema.suggesters.json")); + JsonNode jsonNode = mapper.readTree( + classLoader.getResourceAsStream("suggester-treatment/suggesters.json")) + .get("suggesters"); + // + Set errors = jsonSchema.validate(jsonNode); + // + assertTrue(errors.isEmpty()); + } + } diff --git a/eno-treatments/src/test/java/fr/insee/eno/treatments/LunaticSuggesterProcessingTest.java b/eno-treatments/src/test/java/fr/insee/eno/treatments/LunaticSuggesterProcessingTest.java index 54b95a702..566c893bc 100644 --- a/eno-treatments/src/test/java/fr/insee/eno/treatments/LunaticSuggesterProcessingTest.java +++ b/eno-treatments/src/test/java/fr/insee/eno/treatments/LunaticSuggesterProcessingTest.java @@ -10,18 +10,18 @@ import static org.junit.jupiter.api.Assertions.assertFalse; -public class LunaticSuggesterProcessingTest { +class LunaticSuggesterProcessingTest { @Test void suggesterTest() throws DDIParsingException { // Questionnaire lunaticQuestionnaire = DDIToLunatic.transform( - this.getClass().getClassLoader().getResourceAsStream("suggester/ddi-lgl1kmol.xml"), + this.getClass().getClassLoader().getResourceAsStream("suggester-treatment/ddi-lgl1kmol.xml"), EnoParameters.of(EnoParameters.Context.HOUSEHOLD, EnoParameters.ModeParameter.CAWI, Format.LUNATIC)); // SpecificTreatmentsDeserializer treatmentsDeserializer = new SpecificTreatmentsDeserializer(); SpecificTreatments treatmentsInput = treatmentsDeserializer.deserialize( - this.getClass().getClassLoader().getResourceAsStream("suggester/suggester.json")); + this.getClass().getClassLoader().getResourceAsStream("suggester-treatment/suggesters.json")); LunaticSuggesterProcessing suggesterProcessing = new LunaticSuggesterProcessing(treatmentsInput.suggesters()); suggesterProcessing.apply(lunaticQuestionnaire); // diff --git a/eno-treatments/src/test/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializerTest.java b/eno-treatments/src/test/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializerTest.java index 8cc2f0c1c..d398e77c2 100644 --- a/eno-treatments/src/test/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializerTest.java +++ b/eno-treatments/src/test/java/fr/insee/eno/treatments/SpecificTreatmentsDeserializerTest.java @@ -47,7 +47,7 @@ void deserializeSuggesterTest() { assertEquals(2, field.getSynonyms().size()); assertEquals(1, field.getRules().size()); assertEquals("field", suggester.getOrder().getField()); - assertEquals("type", suggester.getOrder().getType()); + assertEquals("ascending", suggester.getOrder().getType()); assertEquals("tokenized", suggester.getQueryParser().getType()); EnoSuggesterQueryParserParams params = suggester.getQueryParser().getParams(); diff --git a/eno-treatments/src/test/java/fr/insee/eno/treatments/dto/EnoFieldSynonymTest.java b/eno-treatments/src/test/java/fr/insee/eno/treatments/dto/EnoFieldSynonymTest.java deleted file mode 100644 index afe49a982..000000000 --- a/eno-treatments/src/test/java/fr/insee/eno/treatments/dto/EnoFieldSynonymTest.java +++ /dev/null @@ -1,42 +0,0 @@ -package fr.insee.eno.treatments.dto; - -import fr.insee.lunatic.model.flat.FieldSynonym; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -class EnoFieldSynonymTest { - - @Test - void whenConvertingToLunaticMappingIsCorrect() { - EnoFieldSynonym field = new EnoFieldSynonym("source", List.of("target1", "target2")); - FieldSynonym fieldLunatic = EnoFieldSynonym.toLunaticModel(field); - assertEquals(fieldLunatic.getSource(), field.getSource()); - assertTrue(fieldLunatic.getTarget().contains("target1")); - assertTrue(fieldLunatic.getTarget().contains("target2")); - } - - @Test - void whenConvertingToLunaticMappingIfNullParameterReturnNull() { - assertNull(EnoFieldSynonym.toLunaticModel(null)); - } - - @Test - void whenConvertingToLunaticMappingIfNullParameterListReturnEmptyList() { - assertEquals(0, EnoFieldSynonym.toLunaticModelList(null).size()); - } - - @Test - void whenMultipleSynonymsCheckEachSynonymIdIsTransformed() { - EnoFieldSynonym enoField1 = new EnoFieldSynonym("source1", List.of("target1", "target2")); - EnoFieldSynonym enoField2 = new EnoFieldSynonym("source2", List.of("target1", "target2")); - List enoFields = List.of(enoField1, enoField2); - List fields = EnoFieldSynonym.toLunaticModelList(enoFields); - - assertEquals(2, fields.size()); - assertEquals("source1", fields.get(0).getSource()); - assertEquals("source2", fields.get(1).getSource()); - } -} diff --git a/eno-treatments/src/test/java/fr/insee/eno/treatments/dto/EnoSuggesterFieldTest.java b/eno-treatments/src/test/java/fr/insee/eno/treatments/dto/EnoSuggesterFieldTest.java index 512c5b11e..f88e1e84d 100644 --- a/eno-treatments/src/test/java/fr/insee/eno/treatments/dto/EnoSuggesterFieldTest.java +++ b/eno-treatments/src/test/java/fr/insee/eno/treatments/dto/EnoSuggesterFieldTest.java @@ -9,6 +9,7 @@ import java.math.BigInteger; import java.util.List; +import java.util.Map; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; @@ -16,7 +17,7 @@ class EnoSuggesterFieldTest { @Mock - private List synonyms; + private Map> synonyms; @Test void whenConvertingToLunaticCheckMappingIsCorrect() { @@ -24,8 +25,8 @@ void whenConvertingToLunaticCheckMappingIsCorrect() { EnoSuggesterField enoField = new EnoSuggesterField("name", rules, "French", BigInteger.ONE, true, synonyms); SuggesterField field = EnoSuggesterField.toLunaticModel(enoField); assertEquals(field.getName(), enoField.getName()); - assertTrue(field.getRules().contains("rule1")); - assertTrue(field.getRules().contains("rule2")); + assertTrue(field.getRules().getPatterns().contains("rule1")); + assertTrue(field.getRules().getPatterns().contains("rule2")); assertEquals(field.getLanguage(), enoField.getLanguage()); assertEquals(field.getMin(), enoField.getMin()); assertEquals(field.getStemmer(), enoField.getStemmer()); @@ -37,7 +38,7 @@ void whenOtherRulesCheckMappingRulesProperty(List rules) { EnoSuggesterField enoField = new EnoSuggesterField("name", rules, "French", BigInteger.ONE, true, synonyms); SuggesterField field = EnoSuggesterField.toLunaticModel(enoField); for(String rule : rules) { - assertTrue(field.getRules().contains(rule)); + assertTrue(field.getRules().getPatterns().contains(rule)); } } @@ -45,7 +46,6 @@ static Stream generateRules() { return Stream.of( Arguments.of(List.of("rule1", "rule2")), Arguments.of(List.of("rule1")), - Arguments.of(List.of("soft", "rule2")), Arguments.of(List.of("rule1", "rule2", "rule3")) ); } diff --git a/eno-treatments/src/test/resources/specific-treatments.json b/eno-treatments/src/test/resources/specific-treatments.json index 904c1d43e..6179d46a4 100644 --- a/eno-treatments/src/test/resources/specific-treatments.json +++ b/eno-treatments/src/test/resources/specific-treatments.json @@ -14,22 +14,16 @@ "language": "French", "min": 2, "stemmer": true, - "synonyms": [ - { - "source": "source1", - "target": [ - "target1", - "target2" - ] - }, - { - "source": "source2", - "target": [ - "target1", - "target2" - ] - } - ] + "synonyms": { + "source1": [ + "target1", + "target2" + ], + "source2": [ + "target1", + "target2" + ] + } }, { "name": "id", @@ -75,7 +69,7 @@ "url": "plop", "max": 2, "order": { - "type": "type", + "type": "ascending", "field": "field" } }, @@ -87,14 +81,14 @@ "fields": [ { "name": "label", - "rules": ["soft"], + "rules": [ + "soft" + ], "min": 2 }, { "name": "id", - "rules": [ - "soft" - ] + "rules": "soft" } ], "queryParser": { diff --git a/eno-treatments/src/test/resources/suggester-examples/suggesters-example-1.json b/eno-treatments/src/test/resources/suggester-examples/suggesters-example-1.json new file mode 100644 index 000000000..6e830349d --- /dev/null +++ b/eno-treatments/src/test/resources/suggester-examples/suggesters-example-1.json @@ -0,0 +1,19 @@ +[ + { + "name": "L_COMMUNEPASSEE-1-3-0", + "fields": [ + { + "name": "label", + "rules": "soft" + }, + { + "name": "id", + "rules": "soft" + } + ], + "queryParser": { + "type": "soft" + }, + "version": "1" + } +] \ No newline at end of file diff --git a/eno-treatments/src/test/resources/suggester-examples/suggesters-example-2.json b/eno-treatments/src/test/resources/suggester-examples/suggesters-example-2.json new file mode 100644 index 000000000..e9899d583 --- /dev/null +++ b/eno-treatments/src/test/resources/suggester-examples/suggesters-example-2.json @@ -0,0 +1,54 @@ +[ + { + "name": "L_PCS_FEMMES-1-4-1-SANS", + "fields": [ + { + "name": "label", + "rules": [ + "[\\w]+" + ], + "language": "French", + "min": 2, + "stemmer": false + }, + { + "name": "id" + } + ], + "queryParser": { + "type": "tokenized", + "params": { + "language": "French", + "pattern": "[\\w.]+", + "stemmer": false + } + }, + "version": "1", + "meloto": true, + "stopWords": [ + "a", + "au", + "dans", + "de", + "des", + "du", + "en", + "er", + "la", + "le", + "ou", + "sur", + "d", + "l", + "aux", + "dans", + "un", + "une", + "pour", + "avec", + "chez", + "par", + "les" + ] + } +] \ No newline at end of file diff --git a/eno-treatments/src/test/resources/suggester-examples/suggesters-example-3.json b/eno-treatments/src/test/resources/suggester-examples/suggesters-example-3.json new file mode 100644 index 000000000..09d13eefd --- /dev/null +++ b/eno-treatments/src/test/resources/suggester-examples/suggesters-example-3.json @@ -0,0 +1,136 @@ +[ + { + "name": "L_ACTIVITES-1-0-0-SANS", + "fields": [ + { + "name": "label", + "rules": [ + "[\\w]+" + ], + "language": "French", + "min": 2, + "stemmer": false, + "synonyms": { + "accueil": [ + "ACCEUIL", + "ACCUEILLIR", + "ACCUEILLANTE", + "ACCUEILLANT", + "ACCUEILLE" + ], + "EHPAD": [ + "EPHAD", + "HEPAD", + "EPAD", + "EPAHD", + "EPADH" + ], + "plaquisterie": [ + "PLACO", + "PLACOPLATRE" + ], + "pneumatique": [ + "PNEU", + "PNEUS" + ], + "prestation": [ + "PRESTATAIRE" + ], + "echafaudage": [ + "ECHAFFAUDAGE", + "ECHAFFAUDEUR" + ], + "conseil": [ + "CONSULTING" + ], + "URSSAF": [ + "URSAF", + "URSAFF" + ], + "ingenierie": [ + "INGENIEURIE", + "INGENERIE", + "INGIENERIE" + ], + "construction": [ + "CONSTRUCTEUR" + ], + "distribution": [ + "DISTRIBUTEUR" + ], + "fabrication": [ + "FABRICANT" + ], + "abattage": [ + "ABATOIR", + "ABBATOIR", + "ABATTOIRS", + "ABATOIRE", + "ABATTOIRE", + "ABBATTAGE" + ], + "ascenseur": [ + "ASCENCEUR", + "ASCENCEURS" + ], + "briqueterie": [ + "BRIQUETTERIE" + ], + "joaillerie": [ + "JOAILLIER" + ], + "agroalimentaire": [ + "AGGRO", + "AGGROALIMANTAIRE", + "AGRO", + "AGROALIMENTAIRE" + ], + "alimentaire": [ + "ALIMANTAIRE" + ], + "recherche-développement": [ + "R & D", + "R&D" + ] + } + }, + { + "name": "id" + } + ], + "queryParser": { + "type": "tokenized", + "params": { + "language": "French", + "pattern": "[\\w.]+", + "stemmer": false + } + }, + "version": "1", + "stopWords": [ + "a", + "au", + "dans", + "de", + "des", + "du", + "en", + "er", + "la", + "le", + "ou", + "sur", + "d", + "l", + "aux", + "dans", + "un", + "une", + "pour", + "avec", + "chez", + "par", + "les" + ] + } +] \ No newline at end of file diff --git a/eno-treatments/src/test/resources/suggester-examples/suggesters-example-4.json b/eno-treatments/src/test/resources/suggester-examples/suggesters-example-4.json new file mode 100644 index 000000000..f21de23f6 --- /dev/null +++ b/eno-treatments/src/test/resources/suggester-examples/suggesters-example-4.json @@ -0,0 +1,19 @@ +[ + { + "name": "L_DIPLOMES", + "fields": [ + { + "name": "label", + "rules": "soft" + }, + { + "name": "id", + "rules": "soft" + } + ], + "queryParser": { + "type": "soft" + }, + "version": "1" + } +] \ No newline at end of file diff --git a/eno-treatments/src/test/resources/suggester-examples/suggesters-example.json b/eno-treatments/src/test/resources/suggester-examples/suggesters-example.json new file mode 100644 index 000000000..c564931f0 --- /dev/null +++ b/eno-treatments/src/test/resources/suggester-examples/suggesters-example.json @@ -0,0 +1,30 @@ +[ + { + "name": "naf-rev2", + "fields": [ + { + "name": "label", + "rules": [ + "[\\w]+" + ], + "language": "French", + "min": 2 + }, + { + "name": "id" + }, + { + "name": "demo", + "rules": "soft" + } + ], + "queryParser": { + "type": "tokenized", + "params": { + "language": "French", + "pattern": "[\\w.]+" + } + }, + "version": "1" + } +] \ No newline at end of file diff --git a/eno-treatments/src/test/resources/suggester-treatment/ddi-lgl1kmol.xml b/eno-treatments/src/test/resources/suggester-treatment/ddi-lgl1kmol.xml new file mode 100644 index 000000000..4ff0d3a73 --- /dev/null +++ b/eno-treatments/src/test/resources/suggester-treatment/ddi-lgl1kmol.xml @@ -0,0 +1,445 @@ + + + fr.insee + INSEE-lgl1kmol + 1 + + + Suggester - plusieurs types + + + + fr.insee + RessourcePackage-lgl1kmol + 1 + + fr.insee + InterviewerInstructionScheme-lgl1kmol + 1 + + A définir + + + fr.insee + l1uiu9hq + 1 + + help + + + SelfAdministeredQuestionnaire.WebBased + + + SelfAdministeredQuestionnaire.Paper + + + Interview.Telephone.CATI + + + Interview.FaceToFace.CAPIorCAMI + + + + Cette séquence comprend les questions ouvertes (ceci est une déclaration de type aide, associée au titre de séquence). + + + + + + fr.insee + ControlConstructScheme-lgl1kmol + 1 + + fr.insee + Sequence-lgl1kmol + 1 + + Suggester - plusieurs types + + template + + fr.insee + jfaz9kv9 + 1 + Sequence + + + + fr.insee + jfaz9kv9 + 1 + + SIMPLE + + + S1 + + + fr.insee + l1uiu9hq + 1 + Instruction + + module + + fr.insee + lhggytud-QC + 1 + QuestionConstruct + + + fr.insee + lgm7ka9g-QC + 1 + QuestionConstruct + + + + fr.insee + lhggytud-QC + 1 + + GEO + + + fr.insee + lhggytud + 1 + QuestionItem + + + + fr.insee + lgm7ka9g-QC + 1 + + PCS + + + fr.insee + lgm7ka9g + 1 + QuestionItem + + + + + fr.insee + QuestionScheme-lgl1kmol + 1 + + A définir + + + fr.insee + lhggytud + 1 + + GEO + + + fr.insee + lhggytud-QOP-lhggjlfa + 1 + + GEO + + + + + fr.insee + lhggytud-RDOP-lhggjlfa + 1 + OutParameter + + + fr.insee + lhggytud-QOP-lhggjlfa + 1 + OutParameter + + + + + "Suggester géo" + + + + + fr.insee + lhggytud-RDOP-lhggjlfa + 1 + + + + + + fr.insee + lgm7ka9g + 1 + + PCS + + + fr.insee + lgm7ka9g-QOP-lgm78onu + 1 + + PCS + + + + + fr.insee + lgm7ka9g-RDOP-lgm78onu + 1 + OutParameter + + + fr.insee + lgm7ka9g-QOP-lgm78onu + 1 + OutParameter + + + + + "Suggester pcs" + + + + + fr.insee + lgm7ka9g-RDOP-lgm78onu + 1 + + + + + + + fr.insee + CategoryScheme-lgl1kmol + 1 + + A définir + + + fr.insee + INSEE-COMMUN-CA-Booleen-1 + 1 + + + + + + + fr.insee + REFLUNATIC-CLS + 1 + + REFLUNATIC + + + fr.insee + INSEE-COMMUN-CL-Booleen + 1 + + Booleen + + Regular + + Ordinal + + + fr.insee + INSEE-COMMUN-CL-Booleen-1 + 1 + + fr.insee + INSEE-COMMUN-CA-Booleen-1 + 1 + Category + + 1 + + + + + fr.insee + VariableScheme-lgl1kmol + 1 + + Variable Scheme for the survey + + + fr.insee + lhggp2vt + 1 + + GEO + + + GEO label + + + fr.insee + lhggytud-QOP-lhggjlfa + 1 + OutParameter + + + fr.insee + lhggytud + 1 + QuestionItem + + + + + + + fr.insee + lhggj3ch + 1 + + PCS + + + PCS label + + + fr.insee + lgm7ka9g-QOP-lgm78onu + 1 + OutParameter + + + fr.insee + lgm7ka9g + 1 + QuestionItem + + + + + + + fr.insee + INSEE-Instrument-lgl1kmol-vg + 1 + + + fr.insee + Instrument-lgl1kmol + 1 + Instrument + + + Questionnaire + + REFLUNATIC + + + fr.insee + lhggp2vt + 1 + Variable + + + fr.insee + lhggj3ch + 1 + Variable + + + + + fr.insee + INSEE-SIMPSONS-PIS-1 + 1 + + SIMPSONS + + + Processing instructions of the Simpsons questionnaire + + + + fr.insee + INSEE-SIMPSONS-MRS + 1 + + Liste de formats numériques et dates de + l'enquête + Numeric and DateTime list for the survey + + + + + fr.insee + StudyUnit-lgl1kmol + 1 + + + fr.insee + DataCollection-lgl1kmol + 1 + + fr.insee + QuestionScheme-lgl1kmol + 1 + QuestionScheme + + + fr.insee + ControlConstructScheme-lgl1kmol + 1 + ControlConstructScheme + + + fr.insee + InterviewerInstructionScheme-lgl1kmol + 1 + InterviewerInstructionScheme + + + fr.insee + InstrumentScheme-lgl1kmol + 1 + + fr.insee + Instrument-lgl1kmol + 1 + + REFLUNATIC + + + Suggester - plusieurs types questionnaire + + A définir + + fr.insee + Sequence-lgl1kmol + 1 + Sequence + + + + + + diff --git a/eno-treatments/src/test/resources/suggester-treatment/pogues-lgl1kmol.json b/eno-treatments/src/test/resources/suggester-treatment/pogues-lgl1kmol.json new file mode 100644 index 000000000..92642e4b8 --- /dev/null +++ b/eno-treatments/src/test/resources/suggester-treatment/pogues-lgl1kmol.json @@ -0,0 +1,191 @@ +{ + "owner": "QTEST-LUNATIC-V2", + "FlowControl": [], + "ComponentGroup": [ + { + "MemberReference": [ + "jfaz9kv9", + "lhggytud", + "lgm7ka9g", + "idendquest" + ], + "Label": [ + "Components for page 1" + ], + "id": "kvl6f0ou", + "Name": "PAGE_1" + } + ], + "agency": "fr.insee", + "genericName": "QUESTIONNAIRE", + "Label": [ + "Suggester - plusieurs types" + ], + "childQuestionnaireRef": [], + "Name": "REFLUNATIC", + "Variables": { + "Variable": [ + { + "Label": "PCS label", + "id": "lhggj3ch", + "type": "CollectedVariableType", + "Name": "PCS", + "Datatype": { + "Pattern": "", + "typeName": "TEXT", + "type": "TextDatatypeType", + "MaxLength": "10" + } + }, + { + "Label": "GEO label", + "id": "lhggp2vt", + "type": "CollectedVariableType", + "Name": "GEO", + "Datatype": { + "Pattern": "", + "typeName": "TEXT", + "type": "TextDatatypeType", + "MaxLength": "10" + } + } + ] + }, + "lastUpdatedDate": "Tue May 09 2023 18:10:03 GMT+0200 (heure d’été d’Europe centrale)", + "DataCollection": [ + { + "id": "esa-dc-2018", + "uri": "http://ddi:fr.insee:DataCollection.esa-dc-2018" + } + ], + "final": false, + "flowLogic": "FILTER", + "id": "lgl1kmol", + "TargetMode": [ + "CAWI", + "CAPI", + "CATI", + "PAPI" + ], + "CodeLists": { + "CodeList": [] + }, + "formulasLanguage": "VTL", + "Child": [ + { + "Control": [], + "depth": 1, + "FlowControl": [], + "genericName": "MODULE", + "Label": [ + "S1" + ], + "id": "jfaz9kv9", + "TargetMode": [ + "CAWI" + ], + "Declaration": [ + { + "declarationType": "HELP", + "Text": "Cette séquence comprend les questions ouvertes (ceci est une déclaration de type aide, associée au titre de séquence).", + "id": "l1uiu9hq", + "position": "AFTER_QUESTION_TEXT", + "DeclarationMode": [ + "CAPI", + "CATI", + "CAWI", + "PAPI" + ] + } + ], + "type": "SequenceType", + "Child": [ + { + "Response": [ + { + "CollectedVariableReference": "lhggp2vt", + "id": "lhggjlfa", + "mandatory": false, + "Datatype": { + "Pattern": "", + "typeName": "TEXT", + "type": "TextDatatypeType", + "MaxLength": "10" + } + } + ], + "Control": [], + "depth": 2, + "FlowControl": [], + "Label": [ + "\"Suggester géo\"" + ], + "id": "lhggytud", + "TargetMode": [ + "CAWI", + "CAPI", + "CATI", + "PAPI" + ], + "Declaration": [], + "type": "QuestionType", + "questionType": "SIMPLE", + "Name": "GEO" + }, + { + "Response": [ + { + "CollectedVariableReference": "lhggj3ch", + "id": "lgm78onu", + "mandatory": false, + "Datatype": { + "Pattern": "", + "typeName": "TEXT", + "type": "TextDatatypeType", + "MaxLength": "10" + } + } + ], + "Control": [], + "depth": 2, + "FlowControl": [], + "Label": [ + "\"Suggester pcs\"" + ], + "id": "lgm7ka9g", + "TargetMode": [ + "CAWI", + "CAPI", + "CATI", + "PAPI" + ], + "Declaration": [], + "type": "QuestionType", + "questionType": "SIMPLE", + "Name": "PCS" + } + ], + "Name": "SIMPLE" + }, + { + "Control": [], + "depth": 1, + "FlowControl": [], + "genericName": "MODULE", + "Label": [ + "QUESTIONNAIRE_END" + ], + "id": "idendquest", + "TargetMode": [ + "CAWI", + "CAPI", + "CATI", + "PAPI" + ], + "Declaration": [], + "type": "SequenceType", + "Child": [], + "Name": "QUESTIONNAIRE_END" + } + ] +} \ No newline at end of file diff --git a/eno-treatments/src/test/resources/suggester-treatment/suggesters.json b/eno-treatments/src/test/resources/suggester-treatment/suggesters.json new file mode 100644 index 000000000..e7e898f47 --- /dev/null +++ b/eno-treatments/src/test/resources/suggester-treatment/suggesters.json @@ -0,0 +1,73 @@ +{ + "suggesters": [ + { + "responseNames": ["PCS"], + "name": "L_PCS_HOMMES-1-3-0", + "fields": [ + { + "name": "label", + "rules": ["[\\w]+"], + "language": "French", + "min": 2 + }, + { + "name": "id", + "rules": "soft" + } + ], + "stopWords": [ + "a", + "au", + "dans", + "de", + "des", + "du", + "en", + "er", + "la", + "le", + "ou", + "sur", + "d", + "l", + "aux", + "dans", + "un", + "une", + "pour", + "avec", + "chez", + "par", + "les" + ], + "queryParser": { + "type": "tokenized", + "params": { + "language": "French", + "pattern": "pattern", + "min": 1 + } + }, + "version": "1" + }, + { + "responseNames": ["GEO"], + "name": "L_GEO", + "fields": [ + { + "name": "label", + "rules": "soft", + "min": 2 + }, + { + "name": "id", + "rules": "soft" + } + ], + "queryParser": { + "type": "soft" + }, + "version": "1" + } + ] +} \ No newline at end of file diff --git a/eno-ws/src/test/resources/non-regression/suggester-processing/suggester-input.json b/eno-ws/src/test/resources/non-regression/suggester-processing/suggester-input.json index c768d6aaf..ec2d2611f 100644 --- a/eno-ws/src/test/resources/non-regression/suggester-processing/suggester-input.json +++ b/eno-ws/src/test/resources/non-regression/suggester-processing/suggester-input.json @@ -8,8 +8,12 @@ "fields": [ { "name": "label", - "rules": ["soft"], + "rules": "soft", "min": 2 + }, + { + "name": "id", + "rules": "soft" } ], "queryParser": { @@ -31,6 +35,10 @@ "language": "French", "min": 2, "stemmer": false + }, + { + "name": "id", + "rules": "soft" } ], "queryParser": { diff --git a/settings.gradle b/settings.gradle index 9e58de648..9ae38efea 100644 --- a/settings.gradle +++ b/settings.gradle @@ -23,7 +23,7 @@ dependencyResolutionManagement { versionCatalogs { libs { - version('lunatic-model', '3.2.2-SNAPSHOT') + version('lunatic-model', '3.2.5-SNAPSHOT') version('pogues-model', '1.0.5') library('lunatic-model', 'fr.insee.lunatic', 'lunatic-model').versionRef('lunatic-model') library('pogues-model', 'fr.insee.pogues', 'pogues-model').versionRef('pogues-model')