refactor: remove lunatic patches (#913)

* test(dynamic table): clean up expected lunatic

* test: unit test for lunatic scalar variable

* build: bump lunatic model to 3.3.4

* refactor: remove patches for lunatic cleaning

* refactor: remove patches for lunatic resizing

* refactor: remove patches for lunatic missing block

* chore: remove unused mockito lib

Note: using it shows a warning since java 21.

* test(label): remove usage of deprecated lunatic methods

* refactor: remove patches in lunatic serializer

* chore: maven local at first for dependency resolution

So that maven local is checked first to resolve dependencies.

* chore: version 3.17.2

* test(variables): remove redundant test

This test exists in Lunatic-Model.

build.gradle

@@ -16,7 +16,7 @@ tasks.withType(JavaCompile).configureEach {
 
 allprojects {
     group = 'fr.insee.eno'
-    version = '3.17.1'
+    version = '3.17.2'
 }
 
 subprojects {

eno-core/build.gradle

@@ -46,8 +46,6 @@ dependencies {
     testImplementation 'org.assertj:assertj-core'
     // JSONAssert
     testImplementation 'org.skyscreamer:jsonassert'
-    // Mockito
-    testImplementation 'org.mockito:mockito-core'
 
     // Lombok
     compileOnly 'org.projectlombok:lombok'

eno-core/src/main/java/fr/insee/eno/core/processing/out/steps/lunatic/LunaticAddCleaningVariables.java

@@ -1,7 +1,5 @@
 package fr.insee.eno.core.processing.out.steps.lunatic;
 
-import fr.insee.eno.core.model.lunatic.CleaningConcernedVariable;
-import fr.insee.eno.core.model.lunatic.CleaningEntry;
 import fr.insee.eno.core.processing.ProcessingStep;
 import fr.insee.lunatic.model.flat.*;
 import lombok.extern.slf4j.Slf4j;
@@ -28,16 +26,17 @@ public class LunaticAddCleaningVariables implements ProcessingStep<Questionnaire
     @Override
     public void apply(Questionnaire lunaticQuestionnaire) {
         List<ComponentType> components = lunaticQuestionnaire.getComponents();
-        List<CleaningEntry> cleaningVariables = createCleaningVariables(components, lunaticQuestionnaire);
+        List<CleaningVariableEntry> cleaningVariables = createCleaningVariables(components, lunaticQuestionnaire);
 
         if(cleaningVariables.isEmpty()) {
             return;
         }
 
         CleaningType cleaningType = new CleaningType();
-        lunaticQuestionnaire.setCleaning(cleaningType);
 
-        cleaningType.getAny().addAll(groupCleaningVariables(cleaningVariables));
+        cleaningVariables.forEach(cleaningType::addCleaningEntry);
+
+        lunaticQuestionnaire.setCleaning(cleaningType);
     }
 
     /**
@@ -47,8 +46,8 @@ public void apply(Questionnaire lunaticQuestionnaire) {
      * @param lunaticQuestionnaire Lunatic questionnaire to process.
      * @return All cleaning entries for the questionnaire's components.
      */
-    private List<CleaningEntry> createCleaningVariables(List<ComponentType> components, Questionnaire lunaticQuestionnaire) {
-        List<CleaningEntry> cleaningEntries = new ArrayList<>();
+    private List<CleaningVariableEntry> createCleaningVariables(List<ComponentType> components, Questionnaire lunaticQuestionnaire) {
+        List<CleaningVariableEntry> cleaningEntries = new ArrayList<>();
 
         components.stream()
                 .filter(component -> component.getConditionFilter() != null)
@@ -69,15 +68,15 @@ private List<CleaningEntry> createCleaningVariables(List<ComponentType> componen
                 .map(ComponentNestingType.class::cast)
                 .forEach(nestingComponent -> cleaningEntries.addAll(createCleaningVariables(nestingComponent.getComponents(), lunaticQuestionnaire)));
 
-        return cleaningEntries;
+        return groupCleaningVariables(cleaningEntries);
     }
 
     /**
      * Create cleaning variables for a simple response type component
      * @param componentType component to process (must be a simple response type)
      * @return cleaning variables for this component
      */
-    private List<CleaningEntry> createCleaningVariablesFromSimpleResponseComponent(
+    private List<CleaningVariableEntry> createCleaningVariablesFromSimpleResponseComponent(
             ComponentType componentType, Questionnaire lunaticQuestionnaire) {
         if(!(componentType instanceof ComponentSimpleResponseType simpleResponseType)) {
             throw new IllegalArgumentException(String.format(
@@ -92,13 +91,17 @@ private List<CleaningEntry> createCleaningVariablesFromSimpleResponseComponent(
         }
 
         // Cleaning value (here one value per key, objects with same key are grouped together afterward)
-        CleaningConcernedVariable concernedVariable = new CleaningConcernedVariable(
+        CleanedVariableEntry cleanedVariableEntry = new CleanedVariableEntry(
                 simpleResponseType.getResponse().getName(),
                 componentType.getConditionFilter().getValue());
 
         // Cleaning entries
         return bindingDependencies.stream()
-                .map(bindingDependency -> new CleaningEntry(bindingDependency, List.of(concernedVariable)))
+                .map(bindingDependency -> {
+                    CleaningVariableEntry cleaningVariableEntry = new CleaningVariableEntry(bindingDependency);
+                    cleaningVariableEntry.addCleanedVariable(cleanedVariableEntry);
+                    return cleaningVariableEntry;
+                })
                 .toList();
     }
 
@@ -121,7 +124,7 @@ private static List<String> filterNonCollectedVariables(List<String> variableNam
      * @param componentType component to process (must be a multiple response type)
      * @return cleaning variables for this component
      */
-    private List<CleaningEntry> createCleaningVariablesFromMultipleResponseComponent(ComponentType componentType) {
+    private List<CleaningVariableEntry> createCleaningVariablesFromMultipleResponseComponent(ComponentType componentType) {
         if(!(componentType instanceof ComponentMultipleResponseType)) {
             throw new IllegalArgumentException(String.format("Cannot create cleaning variable from this multiple response component %s", componentType.getId()));
         }
@@ -131,36 +134,40 @@ private List<CleaningEntry> createCleaningVariablesFromMultipleResponseComponent
         }
         String conditionFilter = componentType.getConditionFilter().getValue();
 
-        List<CleaningConcernedVariable> concernedVariables;
+        List<CleanedVariableEntry> cleanedVariableEntries;
 
         switch(componentType.getComponentType()) {
-            case TABLE -> concernedVariables = ((Table) componentType).getBodyLines().stream()
+            case TABLE -> cleanedVariableEntries = ((Table) componentType).getBodyLines().stream()
                     .map(BodyLine::getBodyCells)
                     .flatMap(Collection::stream)
                     .map(BodyCell::getResponse)
                     .filter(Objects::nonNull)
                     .map(ResponseType::getName)
-                    .map(name -> new CleaningConcernedVariable(name, conditionFilter))
+                    .map(name -> new CleanedVariableEntry(name, conditionFilter))
                     .toList();
 
-            case ROSTER_FOR_LOOP -> concernedVariables = ((RosterForLoop) componentType).getComponents().stream()
+            case ROSTER_FOR_LOOP -> cleanedVariableEntries = ((RosterForLoop) componentType).getComponents().stream()
                     .map(BodyCell::getResponse)
                     .filter(Objects::nonNull)
                     .map(ResponseType::getName)
-                    .map(name -> new CleaningConcernedVariable(name, conditionFilter))
+                    .map(name -> new CleanedVariableEntry(name, conditionFilter))
                     .toList();
 
-            case CHECKBOX_GROUP -> concernedVariables = ((CheckboxGroup) componentType).getResponses().stream()
+            case CHECKBOX_GROUP -> cleanedVariableEntries = ((CheckboxGroup) componentType).getResponses().stream()
                     .map(ResponsesCheckboxGroup::getResponse)
                     .map(ResponseType::getName)
-                    .map(name -> new CleaningConcernedVariable(name, conditionFilter))
+                    .map(name -> new CleanedVariableEntry(name, conditionFilter))
                     .toList();
 
             default -> throw new IllegalArgumentException(String.format("Cannot create cleaning variable from this multiple response component %s, componentType not defined", componentType.getId()));
         }
 
         return bindingDependencies.stream()
-                .map(bindingDependency -> new CleaningEntry(bindingDependency, concernedVariables))
+                .map(bindingDependency -> {
+                    CleaningVariableEntry cleaningVariableEntry = new CleaningVariableEntry(bindingDependency);
+                    cleanedVariableEntries.forEach(cleaningVariableEntry::addCleanedVariable);
+                    return cleaningVariableEntry;
+                })
                 .toList();
     }
 
@@ -169,19 +176,21 @@ private List<CleaningEntry> createCleaningVariablesFromMultipleResponseComponent
      * @param cleaningEntries cleaning variables to regroup
      * @return grouped variables
      */
-    private List<CleaningEntry> groupCleaningVariables(List<CleaningEntry> cleaningEntries) {
-        Map<String, CleaningEntry> groupCleaningVariables = new LinkedHashMap<>();
-        for (CleaningEntry cleaningEntry : cleaningEntries) {
-            if(!groupCleaningVariables.containsKey(cleaningEntry.getVariableName())) {
-                groupCleaningVariables.put(cleaningEntry.getVariableName(), cleaningEntry);
+    private List<CleaningVariableEntry> groupCleaningVariables(List<CleaningVariableEntry> cleaningEntries) {
+
+        Map<String, CleaningVariableEntry> groupCleaningVariables = new LinkedHashMap<>();
+
+        for (CleaningVariableEntry cleaningEntry : cleaningEntries) {
+            if(!groupCleaningVariables.containsKey(cleaningEntry.getCleaningVariableName())) {
+                groupCleaningVariables.put(cleaningEntry.getCleaningVariableName(), cleaningEntry);
                 continue;
             }
 
-            CleaningEntry groupVariable = groupCleaningVariables.get(cleaningEntry.getVariableName());
-            List<CleaningConcernedVariable> concernedVariables = new ArrayList<>(groupVariable.getConcernedVariables());
-            concernedVariables.addAll(cleaningEntry.getConcernedVariables());
-            groupVariable.setConcernedVariables(concernedVariables);
+            CleaningVariableEntry groupVariable = groupCleaningVariables.get(cleaningEntry.getCleaningVariableName());
+            cleaningEntry.getCleanedVariableNames().forEach(cleanedVariableName ->
+                    groupVariable.addCleanedVariable(cleaningEntry.getCleanedVariable(cleanedVariableName)));
         }
+
         return groupCleaningVariables.values().stream()
                 .toList();
     }

eno-core/src/main/java/fr/insee/eno/core/processing/out/steps/lunatic/LunaticAddMissingVariables.java

@@ -1,7 +1,5 @@
 package fr.insee.eno.core.processing.out.steps.lunatic;
 
-import fr.insee.eno.core.exceptions.business.LunaticLoopException;
-import fr.insee.eno.core.model.lunatic.MissingBlock;
 import fr.insee.eno.core.model.question.Question;
 import fr.insee.eno.core.processing.ProcessingStep;
 import fr.insee.eno.core.reference.EnoCatalog;
@@ -58,17 +56,17 @@ public void apply(Questionnaire lunaticQuestionnaire) {
 
         components.forEach(component -> processComponentsMissingResponse(component, lunaticQuestionnaire));
 
-        List<MissingBlock> missingBlocks = createMissingBlocks(components);
+        List<MissingEntry> missingBlocks = createMissingBlocks(components);
         // New list so that we put missing blocks next to the corresponding reversed missing block
-        List<MissingBlock> allMissingBlocks = new ArrayList<>();
+        List<MissingEntry> allMissingBlocks = new ArrayList<>();
         missingBlocks.forEach(missingBlock -> {
             allMissingBlocks.add(missingBlock);
             allMissingBlocks.addAll(createReversedMissingBlocks(missingBlock));
         });
 
         if (!allMissingBlocks.isEmpty()) {
             MissingType missingType = new MissingType();
-            missingType.getAny().addAll(allMissingBlocks);
+            allMissingBlocks.forEach(missingType::addMissingEntry);
             lunaticQuestionnaire.setMissingBlock(missingType);
         }
     }
@@ -139,15 +137,17 @@ private void addMissingVariable(IVariableType variable, String missingResponseNa
      * @param components List of components used to generate missing block entries.
      * @return list of missing blocks
      */
-    private List<MissingBlock> createMissingBlocks(List<ComponentType> components) {
-        List<MissingBlock> missingBlocks = new ArrayList<>();
+    private List<MissingEntry> createMissingBlocks(List<ComponentType> components) {
+        List<MissingEntry> missingBlocks = new ArrayList<>();
         // generate blocks on components with missing response attribute (included main loop)
         missingBlocks.addAll(components.stream()
                 .filter(componentType -> componentType.getMissingResponse() != null)
                 .map(component -> {
                     String missingResponseName = component.getMissingResponse().getName();
                     List<String> names = LunaticUtils.getResponseNames(component);
-                    return new MissingBlock(missingResponseName, names);
+                    MissingEntry missingEntry = new MissingEntry(missingResponseName);
+                    names.forEach(name -> missingEntry.getCorrespondingVariables().add(name));
+                    return missingEntry;
                 })
                 .toList());
 
@@ -176,9 +176,13 @@ private List<MissingBlock> createMissingBlocks(List<ComponentType> components) {
      * @param missingBlock Missing block from which we need to create reversed missing blocks.
      * @return List of reversed missing blocks.
      */
-    private List<MissingBlock> createReversedMissingBlocks(MissingBlock missingBlock) {
-        return missingBlock.getNames().stream()
-                .map(name -> new MissingBlock(name, List.of(missingBlock.getMissingName())))
+    private List<MissingEntry> createReversedMissingBlocks(MissingEntry missingBlock) {
+        return missingBlock.getCorrespondingVariables().stream()
+                .map(name -> {
+                    MissingEntry missingEntry = new MissingEntry(name);
+                    missingEntry.getCorrespondingVariables().add(missingBlock.getVariableName());
+                    return missingEntry;
+                })
                 .toList();
     }