feat(ddi to lunatic): dynamic units

build.gradle.kts

@@ -16,7 +16,7 @@ java {
 
 allprojects {
     group = "fr.insee.eno"
-    version = "3.30.0"
+    version = "3.31.0-SNAPSHOT.2"
 }
 
 subprojects {

eno-core/src/main/java/fr/insee/eno/core/model/question/NumericQuestion.java

@@ -4,6 +4,7 @@
 import fr.insee.eno.core.annotations.Contexts.Context;
 import fr.insee.eno.core.annotations.DDI;
 import fr.insee.eno.core.annotations.Lunatic;
+import fr.insee.eno.core.model.label.DynamicLabel;
 import fr.insee.eno.core.parameter.Format;
 import fr.insee.lunatic.model.flat.InputNumber;
 import lombok.Getter;
@@ -52,7 +53,7 @@ public class NumericQuestion extends SingleResponseQuestion {
      * In DDI, this information is not available (even through references) in 'QuestionItem' object.
      * It is mapped in Variable class and moved here by a processing. */
     @Lunatic("setUnit(#param)")
-    String unit;
+    DynamicLabel unit;
 
     /** Lunatic component type property.
      * This should be inserted by Lunatic-Model serializer later on. */

eno-core/src/main/java/fr/insee/eno/core/model/question/table/NumericCell.java

@@ -4,6 +4,7 @@
 import fr.insee.eno.core.annotations.Contexts.Context;
 import fr.insee.eno.core.annotations.DDI;
 import fr.insee.eno.core.annotations.Lunatic;
+import fr.insee.eno.core.model.label.DynamicLabel;
 import fr.insee.eno.core.parameter.Format;
 import fr.insee.lunatic.model.flat.BodyCell;
 import lombok.Getter;
@@ -36,6 +37,6 @@ public class NumericCell extends ResponseCell {
 
     /** Unit not accessible here in DDI, the value is set there during a processing. */
     @Lunatic("setUnit(#param)")
-    String unit;
+    DynamicLabel unit;
 
 }

eno-core/src/main/java/fr/insee/eno/core/model/variable/Variable.java

@@ -39,6 +39,11 @@ public enum CollectionType {COLLECTED, CALCULATED, EXTERNAL}
     @DDI("getVariableRepresentation()?.getValueRepresentation()?.getMeasurementUnit()?.getStringValue()")
     private String unit;
 
+    /** A unit can be either fixed or dynamic. */
+    @DDI("getVariableRepresentation()?.getValueRepresentation()?.getMeasurementUnit()?.getControlledVocabularyName() != null ? " +
+            "getVariableRepresentation().getValueRepresentation().getMeasurementUnit().getControlledVocabularyName() == 'personalizedUnit' : false")
+    private Boolean isUnitDynamic;
+
     /** Method to convert an Eno-model CollectionType object (from the enum class)
      * to the value expected in Lunatic-Model. */
     public static VariableTypeEnum lunaticCollectionType(CollectionType enoCollectionType) {

eno-core/src/main/java/fr/insee/eno/core/processing/common/steps/EnoAddResponseTimeSection.java

@@ -14,6 +14,8 @@
 
 import java.math.BigInteger;
 
+import static fr.insee.eno.core.processing.in.steps.ddi.DDIMoveUnitInQuestions.createUnit;
+
 public class EnoAddResponseTimeSection implements ProcessingStep<EnoQuestionnaire> {
 
     public static final String HOURS_VARIABLE_NAME = "HEURE_REMPL";
@@ -79,7 +81,7 @@ public void apply(EnoQuestionnaire enoQuestionnaire) {
         hoursQuestion.setMinValue(HOURS_QUESTION_MIN_VALUE);
         hoursQuestion.setMaxValue(HOURS_QUESTION_MAX_VALUE);
         hoursQuestion.setNumberOfDecimals(HOURS_QUESTION_DECIMALS);
-        hoursQuestion.setUnit(HOURS_VARIABLE_UNIT);
+        hoursQuestion.setUnit(createUnit(HOURS_VARIABLE_UNIT));
         hoursQuestion.setResponse(new Response());
         hoursQuestion.getResponse().setVariableName(HOURS_VARIABLE_NAME);
         enoQuestionnaire.getSingleResponseQuestions().add(hoursQuestion);
@@ -94,7 +96,7 @@ public void apply(EnoQuestionnaire enoQuestionnaire) {
         minutesQuestion.setMinValue(TIME_MINUTES_QUESTION_MIN_VALUE);
         minutesQuestion.setMaxValue(RESPONSE_TIME_MINUTES_QUESTION_MAX_VALUE);
         minutesQuestion.setNumberOfDecimals(RESPONSE_TIME_MINUTES_QUESTION_DECIMALS);
-        minutesQuestion.setUnit(MINUTES_VARIABLE_UNIT);
+        minutesQuestion.setUnit(createUnit(MINUTES_VARIABLE_UNIT));
         minutesQuestion.setResponse(new Response());
         minutesQuestion.getResponse().setVariableName(MINUTES_VARIABLE_NAME);
         enoQuestionnaire.getSingleResponseQuestions().add(minutesQuestion);

eno-core/src/main/java/fr/insee/eno/core/processing/in/DDIInProcessing.java

@@ -16,7 +16,7 @@ public void applyProcessing(EnoQuestionnaire enoQuestionnaire) {
                 .then(new DDICleanUpQuestionnaireId())
                 .then(new DDIMarkRowControls())
                 .then(new DDIMarkRoundaboutFilters())
-                .then(new DDIMoveUnitInQuestions())
+                .then(new DDIMoveUnitInQuestions(enoIndex))
                 .then(new DDIInsertResponseInTableCells())
                 .then(new DDIInsertDetailResponses())
                 .then(new DDIInsertMultipleChoiceLabels())

eno-core/src/main/java/fr/insee/eno/core/processing/in/steps/ddi/DDIMoveUnitInQuestions.java

@@ -1,39 +1,42 @@
 package fr.insee.eno.core.processing.in.steps.ddi;
 
 import fr.insee.eno.core.model.EnoQuestionnaire;
+import fr.insee.eno.core.model.label.DynamicLabel;
 import fr.insee.eno.core.model.navigation.Binding;
 import fr.insee.eno.core.model.question.*;
 import fr.insee.eno.core.model.question.table.NumericCell;
 import fr.insee.eno.core.model.question.table.ResponseCell;
-import fr.insee.eno.core.model.question.table.TableCell;
 import fr.insee.eno.core.model.variable.CollectedVariable;
 import fr.insee.eno.core.model.variable.Variable;
 import fr.insee.eno.core.processing.ProcessingStep;
+import fr.insee.eno.core.reference.EnoIndex;
 import lombok.extern.slf4j.Slf4j;
 
 import java.util.List;
 import java.util.Optional;
 
+/** Processing step to move units which are mapped in variable objects to numeric questions. */
 @Slf4j
 public class DDIMoveUnitInQuestions implements ProcessingStep<EnoQuestionnaire> {
 
-    // TODO: JavaDoc on method or on class?
+    private final EnoIndex enoIndex;
+
+    public DDIMoveUnitInQuestions(EnoIndex enoIndex) {
+        this.enoIndex = enoIndex;
+    }
 
     /** In DDI, the 'unit' information is accessible in variables.
      * This information must also belong in concerned questions in the Eno model.
      * In Lunatic, this information is required in some numeric questions. */
     public void apply(EnoQuestionnaire enoQuestionnaire) {
-        // TODO: assert or proper log + exception?
-        assert enoQuestionnaire.getIndex() != null;
-        //
         enoQuestionnaire.getVariables().stream()
                 .filter(variable -> Variable.CollectionType.COLLECTED.equals(variable.getCollectionType()))
                 .map(CollectedVariable.class::cast)
                 .filter(variable -> variable.getUnit() != null)
                 .forEach(variable -> {
-                    Question question = (Question) enoQuestionnaire.get(variable.getQuestionReference());
+                    Question question = (Question) enoIndex.get(variable.getQuestionReference());
                     if (question instanceof NumericQuestion numericQuestion) {
-                        numericQuestion.setUnit(variable.getUnit());
+                        numericQuestion.setUnit(createUnit(variable.getUnit()));
                         return;
                     }
 
@@ -86,6 +89,16 @@ private void applyUnitOnNumericCell(Variable variable, List<Binding> bindings, L
             return;
         }
 
-        numericCell.get().setUnit(variable.getUnit());
+        numericCell.get().setUnit(createUnit(variable.getUnit()));
     }
+
+    public static DynamicLabel createUnit(String value) {
+        if (value == null)
+            return null;
+        DynamicLabel label = new DynamicLabel();
+        label.setValue(value);
+        // (type is left at its default value)
+        return label;
+    }
+
 }

eno-core/src/main/java/fr/insee/eno/core/processing/out/steps/lunatic/LunaticEditLabelTypes.java

@@ -35,6 +35,31 @@ private void editLabels(List<ComponentType> lunaticComponents) {
         lunaticComponents.stream()
                 .filter(Sequence.class::isInstance).map(Sequence.class::cast)
                 .forEach(this::editSequenceLabel);
+        lunaticComponents.stream()
+                .filter(InputNumber.class::isInstance).map(InputNumber.class::cast)
+                .forEach(this::editInputNumberUnit);
+        lunaticComponents.stream()
+                .filter(Table.class::isInstance).map(Table.class::cast)
+                .map(Table::getBodyLines)
+                .forEach(bodyLines -> bodyLines.stream()
+                        .map(BodyLine::getBodyCells)
+                        .forEach(this::editCellUnits));
+        lunaticComponents.stream()
+                .filter(RosterForLoop.class::isInstance).map(RosterForLoop.class::cast)
+                .map(RosterForLoop::getComponents)
+                .forEach(this::editCellUnits);
+    }
+
+    private void editCellUnits(List<BodyCell> bodyCells) {
+        bodyCells.stream()
+                .filter(bodyCell -> bodyCell.getUnitLabel() != null)
+                .forEach(bodyCell -> bodyCell.getUnitLabel().setType(LabelTypeEnum.VTL));
+    }
+
+    private void editInputNumberUnit(InputNumber inputNumber) {
+        if (inputNumber.getUnitLabel() == null)
+            return;
+        inputNumber.getUnitLabel().setType(LabelTypeEnum.VTL);
     }
 
     private void editSequenceLabel(Sequence sequence) {

eno-core/src/main/java/fr/insee/eno/core/processing/out/steps/lunatic/LunaticInputNumberDescription.java

@@ -20,8 +20,8 @@ public class LunaticInputNumberDescription implements ProcessingStep<Questionnai
             EnoParameters.Language.FR, "Format attendu : un nombre%s entre %s et %s",
             EnoParameters.Language.EN, "Expected format: a number%s between %s and %s");
     private static final Map<EnoParameters.Language, String> UNIT_DESCRIPTION_PREFIX = Map.of(
-            EnoParameters.Language.FR, " en ",
-            EnoParameters.Language.EN, " in ");
+            EnoParameters.Language.FR, "en",
+            EnoParameters.Language.EN, "in");
 
     private final EnoParameters.Language language;
     private final Locale locale;
@@ -54,21 +54,30 @@ private void generateInputNumberDescription(List<ComponentType> lunaticComponent
 
     private void generateInputNumberDescription(InputNumber inputNumber) {
         //
-        String unit = inputNumber.getUnit();
+        LabelType unitLabel = inputNumber.getUnitLabel();
+        boolean noUnit = unitLabel == null;
         int decimals = inputNumber.getDecimals() != null ? inputNumber.getDecimals().intValue() : 0;
         DecimalFormat decimalFormat = (DecimalFormat) NumberFormat.getNumberInstance(locale);
         decimalFormat.setMinimumFractionDigits(decimals);
         String minDescription = decimalFormat.format(inputNumber.getMin());
         String maxDescription = decimalFormat.format(inputNumber.getMax());
         //
-        String unitDescription = unit != null ? UNIT_DESCRIPTION_PREFIX.get(language) + unit : "";
+        String unitDescription = generateUnitDescription(unitLabel);
         String generatedDescription = String.format(INPUT_NUMBER_DESCRIPTION_CANVAS.get(language),
                 unitDescription, minDescription, maxDescription);
+        if (!noUnit)
+            generatedDescription = "\"" + generatedDescription + "\"";
         //
         LabelType description = new LabelType();
         description.setValue(generatedDescription);
-        description.setType(LabelTypeEnum.TXT);
+        description.setType(noUnit ? LabelTypeEnum.TXT : LabelTypeEnum.VTL);
         inputNumber.setDescription(description);
     }
 
+    private String generateUnitDescription(LabelType unitLabel) {
+        if (unitLabel == null)
+            return "";
+        return " " + UNIT_DESCRIPTION_PREFIX.get(language) + " \" || " + unitLabel.getValue() + " || \"";
+    }
+
 }

eno-core/src/main/java/fr/insee/eno/core/reference/EnoCatalog.java

@@ -8,11 +8,9 @@
 import fr.insee.eno.core.model.declaration.Instruction;
 import fr.insee.eno.core.model.label.EnoLabel;
 import fr.insee.eno.core.model.navigation.Control;
-import fr.insee.eno.core.model.question.DynamicTableQuestion;
-import fr.insee.eno.core.model.question.EnoTable;
-import fr.insee.eno.core.model.question.Question;
-import fr.insee.eno.core.model.question.TableQuestion;
+import fr.insee.eno.core.model.question.*;
 import fr.insee.eno.core.model.question.table.NoDataCell;
+import fr.insee.eno.core.model.question.table.NumericCell;
 import fr.insee.eno.core.model.sequence.AbstractSequence;
 import fr.insee.eno.core.model.sequence.RoundaboutSequence;
 import fr.insee.eno.core.model.sequence.Sequence;
@@ -102,6 +100,18 @@ private void gatherLabels(EnoQuestionnaire enoQuestionnaire) {
             labels.addAll(enoComponent.getDeclarations().stream().map(Declaration::getLabel).toList());
             labels.addAll(enoComponent.getInstructions().stream().map(Instruction::getLabel).toList());
         });
+        // Units
+        enoQuestionnaire.getSingleResponseQuestions().stream()
+                .filter(NumericQuestion.class::isInstance).map(NumericQuestion.class::cast)
+                .map(NumericQuestion::getUnit).filter(Objects::nonNull)
+                .forEach(labels::add);
+        enoQuestionnaire.getMultipleResponseQuestions().stream()
+                .filter(question -> question instanceof TableQuestion || question instanceof DynamicTableQuestion)
+                .map(EnoTable.class::cast)
+                .map(EnoTable::getResponseCells).forEach(responseCells -> responseCells.stream()
+                        .filter(NumericCell.class::isInstance).map(NumericCell.class::cast)
+                        .map(NumericCell::getUnit).filter(Objects::nonNull)
+                        .forEach(labels::add));
         // Controls
         this.getQuestions().forEach(enoQuestion ->
                 labels.addAll(enoQuestion.getControls().stream().map(Control::getMessage).toList()));

eno-core/src/test/java/fr/insee/eno/core/mapping/in/ddi/VariableTest.java

@@ -0,0 +1,45 @@
+package fr.insee.eno.core.mapping.in.ddi;
+
+import fr.insee.eno.core.exceptions.business.DDIParsingException;
+import fr.insee.eno.core.mappers.DDIMapper;
+import fr.insee.eno.core.model.EnoQuestionnaire;
+import fr.insee.eno.core.model.variable.Variable;
+import fr.insee.eno.core.serialize.DDIDeserializer;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class VariableTest {
+
+    @Test
+    void variableUnitIntegrationTest() throws DDIParsingException {
+        // Given + When
+        EnoQuestionnaire enoQuestionnaire = new EnoQuestionnaire();
+        DDIMapper ddiMapper = new DDIMapper();
+        ddiMapper.mapDDI(
+                DDIDeserializer.deserialize(this.getClass().getClassLoader().getResourceAsStream(
+                        "integration/ddi/ddi-dynamic-unit.xml")),
+                enoQuestionnaire);
+
+        // Then
+        // Index variables by name to ease testing
+        Map<String, Variable> variableMap = enoQuestionnaire.getVariables().stream()
+                .collect(Collectors.toMap(Variable::getName, variable -> variable));
+        //
+        assertEquals("€", variableMap.get("NUMBER_FIXED_UNIT").getUnit());
+        assertEquals("%", variableMap.get("TABLE11").getUnit());
+        assertEquals("%", variableMap.get("TABLE21").getUnit());
+        assertEquals("€", variableMap.get("DYNAMIC_TABLE1").getUnit());
+        List.of("NUMBER_DYNAMIC_UNIT", "TABLE12", "TABLE22", "DYNAMIC_TABLE2").forEach(variableName ->
+                assertTrue(variableMap.get(variableName).getUnit().contains("¤")));
+        //
+        List.of("NUMBER_FIXED_UNIT", "TABLE11", "TABLE21", "DYNAMIC_TABLE1").forEach(variableName ->
+                assertFalse(variableMap.get(variableName).getIsUnitDynamic()));
+        List.of("NUMBER_DYNAMIC_UNIT", "TABLE12", "TABLE22", "DYNAMIC_TABLE2").forEach(variableName ->
+                assertTrue(variableMap.get(variableName).getIsUnitDynamic()));
+    }
+}

eno-core/src/test/java/fr/insee/eno/core/mapping/out/lunatic/NumberQuestionTest.java

@@ -1,12 +1,16 @@
 package fr.insee.eno.core.mapping.out.lunatic;
 
+import fr.insee.eno.core.DDIToEno;
 import fr.insee.eno.core.DDIToLunatic;
 import fr.insee.eno.core.exceptions.business.DDIParsingException;
+import fr.insee.eno.core.mappers.LunaticMapper;
+import fr.insee.eno.core.model.EnoQuestionnaire;
 import fr.insee.eno.core.parameter.EnoParameters;
 import fr.insee.eno.core.parameter.Format;
-import fr.insee.lunatic.model.flat.ComponentTypeEnum;
-import fr.insee.lunatic.model.flat.InputNumber;
-import fr.insee.lunatic.model.flat.Questionnaire;
+import fr.insee.eno.core.processing.out.steps.lunatic.LunaticEditLabelTypes;
+import fr.insee.eno.core.processing.out.steps.lunatic.LunaticSortComponents;
+import fr.insee.eno.core.processing.out.steps.lunatic.table.LunaticTableProcessing;
+import fr.insee.lunatic.model.flat.*;
 import org.junit.jupiter.api.Test;
 
 import java.util.List;
@@ -36,10 +40,45 @@ void integrationTest_unit() throws DDIParsingException {
                 .findAny();
         // assertions
         assertEquals(3, inputNumbersNoUnit.size());
-        inputNumbersNoUnit.forEach(inputNumber -> assertNull(inputNumber.getUnit()));
+        inputNumbersNoUnit.forEach(inputNumber -> assertNull(inputNumber.getUnitLabel()));
         assertTrue(inputNumberWithUnit.isPresent());
-        assertNotNull(inputNumberWithUnit.get().getUnit());
-        assertEquals("kg", inputNumberWithUnit.get().getUnit());
+        assertNotNull(inputNumberWithUnit.get().getUnitLabel());
+        assertEquals("\"kg\"", inputNumberWithUnit.get().getUnitLabel().getValue());
+    }
+
+    @Test
+    void integrationTest_dynamicUnit() throws DDIParsingException {
+        // Given + When
+        EnoQuestionnaire enoQuestionnaire = DDIToEno.transform(
+                this.getClass().getClassLoader().getResourceAsStream("integration/ddi/ddi-dynamic-unit.xml"),
+                EnoParameters.of(EnoParameters.Context.DEFAULT, EnoParameters.ModeParameter.CAWI));
+        Questionnaire lunaticQuestionnaire = new Questionnaire();
+        new LunaticMapper().mapQuestionnaire(enoQuestionnaire, lunaticQuestionnaire);
+        new LunaticSortComponents(enoQuestionnaire).apply(lunaticQuestionnaire);
+        new LunaticTableProcessing(enoQuestionnaire).apply(lunaticQuestionnaire);
+        new LunaticEditLabelTypes().apply(lunaticQuestionnaire); // to set unit type to VTL
+
+        // Then
+        InputNumber inputNumber1 = (InputNumber) lunaticQuestionnaire.getComponents().get(1);
+        InputNumber inputNumber2 = (InputNumber) lunaticQuestionnaire.getComponents().get(3);
+        Table table = (Table) lunaticQuestionnaire.getComponents().get(4);
+        RosterForLoop rosterForLoop = (RosterForLoop) lunaticQuestionnaire.getComponents().get(5);
+
+        testUnitContent("\"€\"", inputNumber1.getUnitLabel());
+        testUnitContent("WHICH_UNIT", inputNumber2.getUnitLabel());
+
+        // Reminder: in Lunatic tables there is a left column so responses cells start at column of index 1
+        testUnitContent("\"%\"", table.getBodyLines().get(0).getBodyCells().get(1).getUnitLabel());
+        testUnitContent("\"%\"", table.getBodyLines().get(1).getBodyCells().get(1).getUnitLabel());
+        testUnitContent("WHICH_UNIT", table.getBodyLines().get(0).getBodyCells().get(2).getUnitLabel());
+        testUnitContent("WHICH_UNIT", table.getBodyLines().get(1).getBodyCells().get(2).getUnitLabel());
+
+        testUnitContent("\"€\"", rosterForLoop.getComponents().get(0).getUnitLabel());
+        testUnitContent("WHICH_UNIT", rosterForLoop.getComponents().get(1).getUnitLabel());
+    }
+    private void testUnitContent(String expectedValue, LabelType unitLabel) {
+        assertEquals(expectedValue, unitLabel.getValue().trim());
+        assertEquals(LabelTypeEnum.VTL, unitLabel.getType());
     }
 
 }