Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: suggester #792

Merged
merged 39 commits into from
Nov 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
bfc6800
fix: correct vtl labels in begin and end questions (#755)
nsenave Oct 18, 2023
099d11d
test: add unit tests for converters (#764)
nsenave Oct 23, 2023
246de9b
fix: invalid cast in lunatic resizing pairwise processing (#765)
nsenave Oct 23, 2023
d5236b0
fix: controls in loops and filters (#766)
nsenave Oct 24, 2023
f99ed43
fix: lunatic pagination parameter (#768)
nsenave Oct 24, 2023
981164f
build: update depencencies (#769)
nsenave Oct 25, 2023
372e97a
test: add a set of non regression tests
nsenave Oct 24, 2023
ace53d4
test: update 'simpsons' without 'other specify'
nsenave Oct 25, 2023
dba4e77
refactor: authorize pairwise resizing override
nsenave Oct 25, 2023
09503c0
test: update test since no exception
nsenave Oct 25, 2023
518b278
test: update functional tests
nsenave Oct 25, 2023
c60119d
fix: complex multiple choice questions (#773)
nsenave Oct 26, 2023
4180fb8
fix: end question labels (#774)
nsenave Oct 26, 2023
3cae00f
test(pairwise): integration test ddi to lunatic
nsenave Oct 28, 2023
a19628c
fis(suggester): missing properties (wip)
nsenave Nov 5, 2023
1f7153a
build(docker): change base image
nsenave Nov 7, 2023
fe785bc
chore: bump version
nsenave Nov 7, 2023
d177bf2
Merge pull request #781 from InseeFr/build/v3-docker-image
laurentC35 Nov 7, 2023
45494dc
build: update dependency constraints (#784)
nsenave Nov 7, 2023
ef46c49
refactor: improve suggesters json schema (wip)
nsenave Nov 9, 2023
73994a2
fix: exclude externals from cleaning (#786)
nsenave Nov 10, 2023
072c258
build(docker): non root user in image (#790)
nsenave Nov 13, 2023
51dfc14
feat: identification section (#791)
nsenave Nov 14, 2023
f82887b
chore: version 3.12.0
nsenave Nov 14, 2023
2f9d422
refactor: remove schema that has been split
nsenave Nov 14, 2023
5bc0997
fix: map synonyms as key/value pairs
nsenave Nov 14, 2023
5f64ef5
fix(suggester): edit suggesters schema
nsenave Nov 14, 2023
6b529fc
refactor: synonyms prop refactored
nsenave Nov 14, 2023
6d81129
test(suggester): update tests
nsenave Nov 14, 2023
8212728
fix: typos
nsenave Nov 14, 2023
44471c3
fix: regrouping schema
nsenave Nov 14, 2023
ce4e708
refactor: add "responseNames" in eno suggesters schema
nsenave Nov 15, 2023
7c08c23
fix(suggester): use lunatic-model fixed version
nsenave Nov 15, 2023
09b0908
refactor: update after new lunatic-model version
nsenave Nov 16, 2023
ccdc7b4
Merge branch 'v3-next' into fix/v3-suggester
nsenave Nov 16, 2023
d952a8e
Merge branch 'v3-next' into fix/v3-suggester
nsenave Nov 16, 2023
cdbf68e
chore: bump version
nsenave Nov 16, 2023
9ef16d4
test(suggester): update soft rule
nsenave Nov 17, 2023
ed4c583
refactor(suggester): rules property
nsenave Nov 22, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ plugins {

allprojects {
group = 'fr.insee.eno'
version = '3.12.0'
version = '3.12.1-SNAPSHOT'
sourceCompatibility = '17'
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,17 @@ public class SpecificTreatmentsDeserializer {
public SpecificTreatments deserialize(InputStream treatmentsStream) {
ObjectMapper mapper = new ObjectMapper();
JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012);
JsonSchema jsonSchema = factory.getSchema(
classLoader.getResourceAsStream("schema.treatments.json"));
JsonSchema suggesterSchema = factory.getSchema(
classLoader.getResourceAsStream("schema.suggesters.json"));
JsonSchema regroupingSchema = factory.getSchema(
classLoader.getResourceAsStream("schema.regrouping.json"));

try {
JsonNode jsonTreatments = mapper.readTree(treatmentsStream);
Set<ValidationMessage> errors = jsonSchema.validate(jsonTreatments);

if(!errors.isEmpty()) {
StringBuilder messageBuilder = new StringBuilder();
for(ValidationMessage errorMessage : errors) {
messageBuilder.append(errorMessage.getMessage());
messageBuilder.append("\n");
}
throw new SpecificTreatmentsValidationException(messageBuilder.toString());
if (jsonTreatments.has("suggesters"))
validateTreatmentInput(suggesterSchema, jsonTreatments.get("suggesters"));
if (jsonTreatments.has("regroupements")) {
validateTreatmentInput(regroupingSchema, jsonTreatments.get("regroupements"));
}

ObjectReader reader = mapper.readerFor(SpecificTreatments.class);
Expand All @@ -51,4 +48,18 @@ public SpecificTreatments deserialize(InputStream treatmentsStream) {
throw new SpecificTreatmentsDeserializationException(ex.getMessage());
}
}

private static void validateTreatmentInput(JsonSchema suggesterSchema, JsonNode treatmentInput) {
Set<ValidationMessage> errors = suggesterSchema.validate(treatmentInput);

if(!errors.isEmpty()) {
StringBuilder messageBuilder = new StringBuilder();
for(ValidationMessage errorMessage : errors) {
messageBuilder.append(errorMessage.getMessage());
messageBuilder.append("\n");
}
throw new SpecificTreatmentsValidationException(messageBuilder.toString());
}
}

}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonProperty;
import fr.insee.lunatic.model.flat.FieldRules;
import fr.insee.lunatic.model.flat.SuggesterField;
import lombok.*;
import lombok.Data;

import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

@Data
public class EnoSuggesterField {
Expand All @@ -19,15 +21,15 @@ public class EnoSuggesterField {
private String language;
private BigInteger min;
private Boolean stemmer;
private List<EnoFieldSynonym> synonyms;
private Map<String, List<String>> synonyms;

@JsonCreator
public EnoSuggesterField(@JsonProperty(value = "name", required = true) String name,
@JsonProperty("rules") List<String> rules,
@JsonProperty("language") String language,
@JsonProperty("min") BigInteger min,
@JsonProperty("stemmer") Boolean stemmer,
@JsonProperty("synonyms") List<EnoFieldSynonym> synonyms) {
@JsonProperty("synonyms") Map<String, List<String>> synonyms) {
this.name = name;
this.rules = rules;
this.language = language;
Expand All @@ -53,16 +55,26 @@ public static SuggesterField toLunaticModel(EnoSuggesterField suggesterField) {
lunaticField.setStemmer(suggesterField.getStemmer());

if(suggesterField.getRules() != null) {
lunaticField.getRules().addAll(suggesterField.getRules());
lunaticField.setRules(new FieldRules());
if (isSoftRule(suggesterField.getRules())) {
lunaticField.getRules().setRule(FieldRules.SOFT_RULE);
} else {
suggesterField.getRules().forEach(pattern -> lunaticField.getRules().addPattern(pattern));
}
}

if(suggesterField.getSynonyms() != null) {
lunaticField.getSynonyms().addAll(EnoFieldSynonym.toLunaticModelList(suggesterField.getSynonyms()));
suggesterField.getSynonyms().forEach((stringKey, stringsValue) ->
lunaticField.getSynonyms().put(stringKey, stringsValue));
}

return lunaticField;
}

private static boolean isSoftRule(List<String> enoFieldRules) {
return enoFieldRules.size() == 1 && FieldRules.SOFT_RULE.equals(enoFieldRules.get(0));
}

/**
* @param enoFields EnoSuggesterField list object to convert
* @return the corresponding lunatic model list object
Expand Down
15 changes: 15 additions & 0 deletions eno-treatments/src/main/resources/schema.regrouping.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://insee.fr/eno/regrouping.schema.json",
"title": "Regrouping specific treatments schema",
"description": "List of variable names to be grouped in the same page in the Lunatic questionnaire.",
"type": "array",
"items": {
"type": "array",
"items": {
"type": "string"
},
"minItems": 2,
"uniqueItems": true
}
}
Original file line number Diff line number Diff line change
@@ -1,29 +1,20 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://insee.fr/suggesters.schema.json",
"title": "Specific treatments schema",
"description": "Specific treatments schema",
"type": "object",
"properties": {
"suggesters": {
"type": "array",
"items": {
"$ref": "#/$defs/suggester"
}
},
"regroupements": {
"type": "array",
"items": {
"$ref": "#/$defs/regroupement"
}
}
"$id": "https://insee.fr/eno/suggesters.schema.json",
"title": "Lunatic suggesters' schema for Eno specific treatment",
"description": "Fits Lunatic suggesters property's schema, except a 'responseName' attribute is added here.",
"type": "array",
"items": {
"$ref": "#/$defs/suggester"
},

"$defs": {

"suggester": {
"type": "object",
"properties": {
"responseNames": {
"description": "Property that is not in Lunatic schema, used to associate suggester with components.",
"type": "array",
"items": {
"type": "string"
Expand All @@ -33,15 +24,18 @@
"name": {
"type": "string"
},
"queryParser": {
"$ref": "#/$defs/query_parser"
},
"fields": {
"type": "array",
"items": {
"$ref": "#/$defs/suggester_field"
}
},
"queryParser": {
"$ref": "#/$defs/query_parser"
},
"meloto": {
"type": "boolean"
},
"stopWords": {
"type": "array",
"items": {
Expand All @@ -61,7 +55,7 @@
"$ref": "#/$defs/suggester_order"
}
},
"required": ["responseNames", "name", "fields", "queryParser", "version"]
"required": ["name", "fields", "queryParser", "version"]
},

"suggester_field": {
Expand All @@ -71,32 +65,30 @@
"type": "string"
},
"rules": {
"type": "array"
"type": ["string", "array"],
"pattern": "^soft$"
},
"min": {
"type": "number"
},
"language": {
"type": "string"
"enum": ["French", "English"]
},
"stemmer": {
"type": "boolean"
},
"synonyms": {
"type" : "array",
"items" : {
"$ref": "#/$defs/field_synonym"
}
"$ref": "#/$defs/field_synonym"
}
},
"required": ["name", "rules"]
"required": ["name"]
},

"query_parser": {
"type": "object",
"properties": {
"type": {
"type": "string"
"enum": ["soft", "tokenized"]
},
"params": {
"$ref": "#/$defs/query_parser_params"
Expand All @@ -109,7 +101,7 @@
"type": "object",
"properties": {
"type": {
"type": "string"
"enum": ["ascending", "desceding"]
},
"field": {
"type": "string"
Expand All @@ -121,8 +113,11 @@
"query_parser_params": {
"type": "object",
"properties": {
"stemmer": {
"type": "boolean"
},
"language": {
"type": "string"
"enum": ["French", "English"]
},
"min": {
"type": "number"
Expand All @@ -132,25 +127,19 @@
}
}
},

"field_synonym": {
"type": "object",
"properties": {
"source": {
"type": "string"
},
"target": {
"patternProperties": {
".*" : {
"type": "array",
"items": {
"type": "string"
},
"minItems": 1
},
"required": ["source", "target"]
}
}
},
"regroupement": {
"type": "array",
"minItems": 2
}

}
}
Loading