Merge pull request #98 from InseeFr/devLunaticReader

Refactor of lunatic specification reader

kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java

@@ -107,7 +107,7 @@ public void init() throws KraftwerkException {
 		userInputsFile = controlInputSequence.getUserInputs(inDirectory);
 		if (withDDI) metadataModels = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap());
 		if (!withDDI) metadataModels = MetadataUtils.getMetadataFromLunatic(userInputsFile.getModeInputsMap());
-	
+
 		if (fileByFile) userInputsFileList = getUserInputsFile(userInputsFile);
 
 		// Check size of data files and throw an exception if it is too big .Limit is 400 Mo for one processing (one file or data folder if not file by file).

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java

@@ -51,6 +51,7 @@ private Constants() {}
 	// ----- Fixed parameters
 	public static final String ROOT_GROUP_NAME = "RACINE";
 	public static final String ROOT_IDENTIFIER_NAME = "IdUE";
+	public static final String LOOP_NAME_PREFIX = "BOUCLE";
 	public static final String MULTIMODE_DATASET_NAME = "MULTIMODE";
 	public static final String REPORTING_DATA_GROUP_NAME = "REPORTINGDATA";
 	public static final String REPORTING_DATA_INTERVIEWER_ID_NULL_PLACEHOLDER = "NON_AFFECTE_";

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/ComponentLunatic.java

@@ -0,0 +1,41 @@
+package fr.insee.kraftwerk.core.metadata;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import java.util.Arrays;
+
+@Log4j2
+@Getter
+public enum ComponentLunatic {
+
+    DATE_PICKER("Datepicker", VariableType.DATE),
+    CHECKBOX_BOOLEAN("CheckboxBoolean", VariableType.BOOLEAN),
+    INPUT_NUMBER("InputNumber", null),
+    INPUT("Input", VariableType.STRING),
+    TEXT_AREA("Textarea", VariableType.STRING),
+    RADIO("Radio", VariableType.STRING),
+    CHECKBOX_ONE("CheckboxOne", VariableType.STRING),
+    DROPDOWN("Dropdown", VariableType.STRING),
+    CHECKBOX_GROUP("CheckboxGroup", VariableType.BOOLEAN),
+    SUGGESTER("Suggester", VariableType.STRING),
+    PAIRWISE_LINKS("PairwiseLinks", null),
+    TABLE("Table", null);
+
+    private String jsonName;
+    // Represents the type of the variable expected with this component type
+    // If null, the type is not unique
+    private VariableType type;
+
+    ComponentLunatic(String jsonName, VariableType type) {
+        this.jsonName=jsonName;
+        this.type = type;
+    }
+
+    public static ComponentLunatic fromJsonName(String jsonName) {
+        return Arrays.stream(ComponentLunatic.values())
+                .filter(component -> component.getJsonName().equals(jsonName))
+                .findFirst()
+                .orElse(null);
+    }
+}

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataModel.java

@@ -21,6 +21,8 @@ public class MetadataModel {
     private LinkedHashMap<String, Group> groups  = new LinkedHashMap<>();
 
     private List<Sequence> sequences = new ArrayList<>();
+
+    private HashMap<SpecType,String> specVersions = new HashMap<>();
 
 
     /** The root group is created when creating a MetadataModel instance. */
@@ -82,6 +84,9 @@ public boolean hasGroup(String groupName) {
         return groups.containsKey(groupName);
     }
 
+    public void putSpecVersions(SpecType specType, String lunaticModelVersion) {
+        specVersions.put(specType, lunaticModelVersion);
+    }
 
     /** Return the fully qualified name of a variable, that is
      * - the name of the variable if it is in the root group.

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtils.java

@@ -32,6 +32,8 @@ private static void putToMetadataModels(String dataMode, ModeInputs modeInputs,
 		}
 		// Step 2 : we add the variables that are only present in the Lunatic file
 		if (modeInputs.getLunaticFile() != null) {
+			// We read and store lunaticModelVersion
+            metadataModel.putSpecVersions(SpecType.LUNATIC,LunaticReader.getLunaticModelVersion(modeInputs.getLunaticFile()));
 			// First we add the collected _MISSING variables
 			List<String> missingVars = LunaticReader.getMissingVariablesFromLunatic(modeInputs.getLunaticFile());
 			for (String missingVar : missingVars) {
@@ -83,6 +85,20 @@ public static Map<String, MetadataModel> getMetadataFromLunatic(Map<String, Mode
 
 	private static void putToMetadataVariableFromLunatic(String dataMode, ModeInputs modeInputs, Map<String, MetadataModel> metadataModels ) {
 		MetadataModel metadataModel = LunaticReader.getMetadataFromLunatic(modeInputs.getLunaticFile());
+		// We add the variables for pairwise links
+		if (metadataModel.getVariables().getVariable(Constants.LIENS) != null) {
+			// We identify the group containing the individuals
+			// The solution is not pretty (hoping that the group name contains "PRENOM")
+			// It is meant to be temporary until we have a better way to identify the group containing the individuals
+			String groupContainingIndividuals = metadataModel.getGroupNames().stream()
+					.filter(g -> g.contains("PRENOM"))
+					.findFirst()
+					.orElse(metadataModel.getGroupNames().getFirst());
+			for (int k=1;k<Constants.MAX_LINKS_ALLOWED;k++) {
+				Variable varLien = new Variable(Constants.LIEN+k, metadataModel.getGroup(groupContainingIndividuals), VariableType.INTEGER);
+				metadataModel.getVariables().putVariable(varLien);
+			}
+		}
 		metadataModels.put(dataMode, metadataModel);
 	}
 }

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/SpecType.java

@@ -0,0 +1,6 @@
+package fr.insee.kraftwerk.core.metadata;
+
+public enum SpecType {
+
+    DDI,LUNATIC;
+}

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/JsonFileReader.java

@@ -24,24 +24,4 @@ public static JsonNode read(Path filePath) throws IOException {
         return mapper.readTree(filePath.toFile());
     }
 
-    public static int compareVersions(String version1, String version2) {
-        int comparisonResult = 0;
-
-        String[] version1Splits = version1.split("\\.");
-        String[] version2Splits = version2.split("\\.");
-        int maxLengthOfVersionSplits = Math.max(version1Splits.length, version2Splits.length);
-
-        for (int i = 0; i < maxLengthOfVersionSplits; i++){
-            Integer v1 = i < version1Splits.length ? Integer.parseInt(version1Splits[i]) : 0;
-            Integer v2 = i < version2Splits.length ? Integer.parseInt(version2Splits[i]) : 0;
-            int compare = v1.compareTo(v2);
-            if (compare != 0) {
-                comparisonResult = compare;
-                break;
-            }
-        }
-        return comparisonResult;
-    }
-
-
 }

kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/LunaticReaderTest.java

@@ -59,8 +59,9 @@ void readVariablesFromLogX21WebLunaticFile() {
 
         //
         assertNotNull(variables);
-        assertEquals(11,variables.getGroupsCount());
+        assertEquals(2,variables.getGroupsCount());
         assertEquals(683, variables.getVariables().getVariables().size());
 
     }
+
 }

kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java

@@ -10,6 +10,7 @@
 import fr.insee.kraftwerk.core.inputs.UserInputsFile;
 import fr.insee.kraftwerk.core.metadata.MetadataModel;
 import fr.insee.kraftwerk.core.metadata.MetadataUtils;
+import fr.insee.kraftwerk.core.metadata.VariableType;
 import fr.insee.kraftwerk.core.outputs.OutputFiles;
 import fr.insee.kraftwerk.core.outputs.csv.CsvOutputFiles;
 import fr.insee.kraftwerk.core.sequence.*;
@@ -18,6 +19,7 @@
 import fr.insee.kraftwerk.core.vtl.VtlBindings;
 import fr.insee.kraftwerk.core.vtl.VtlExecute;
 import io.cucumber.java.Before;
+import io.cucumber.java.en.And;
 import io.cucumber.java.en.Given;
 import io.cucumber.java.en.Then;
 import io.cucumber.java.en.When;
@@ -48,7 +50,7 @@ public class MainDefinitions {
 	static Path outDirectory = Paths.get(FUNCTIONAL_TESTS_OUTPUT_DIRECTORY);
 	Path tempDirectory = Paths.get(FUNCTIONAL_TESTS_TEMP_DIRECTORY);
 	String userInputFileName = Constants.USER_INPUT_FILE;
-	UserInputsFile userInputs;
+	UserInputsFile userInputsFile;
 	String campaignName = "";
 	VtlBindings vtlBindings = new VtlBindings();
 	OutputFiles outputFiles;
@@ -94,16 +96,32 @@ public void init_VTL_script(String vtlScriptName, String variableToCreate, Strin
 	@When("Step 1 : We initialize the input files")
 	public void initialize_input_files() throws KraftwerkException {
 		System.out.println("InDirectory value : " + inDirectory);
-		userInputs = controlInputSequence.getUserInputs(inDirectory);
+		userInputsFile = controlInputSequence.getUserInputs(inDirectory);
 		vtlBindings = new VtlBindings();
 	}
 
 	@When("Step 1 : We initialize with input file {string}")
 	public void initialize_with_specific_input(String inputFileName) throws KraftwerkException {
-		userInputs = new UserInputsFile(inDirectory.resolve(inputFileName), inDirectory);
+		userInputsFile = new UserInputsFile(inDirectory.resolve(inputFileName), inDirectory);
 		vtlBindings = new VtlBindings();
 	}
 
+	@When("Step 1 : We initialize metadata model with lunatic specification only")
+	public void initialize_metadata_model_with_lunatic() throws KraftwerkException {
+		MainProcessing mp = new MainProcessing(inDirectory.toString(), false,false,false, "defaultDirectory", 419430400L);
+		mp.init();
+		userInputsFile=mp.getUserInputsFile();
+		metadataModelMap=mp.getMetadataModels();
+	}
+
+	@When("Step 1 : We initialize metadata model with DDI specification only")
+	public void initialize_metadata_model_with_DDI() throws KraftwerkException {
+		MainProcessing mp = new MainProcessing(inDirectory.toString(), false,false,true, "defaultDirectory", 419430400L);
+		mp.init();
+		userInputsFile=mp.getUserInputsFile();
+		metadataModelMap=mp.getMetadataModels();
+	}
+
 	@When("Step 1 : We launch main service")
 	public void launch_main() throws KraftwerkException {
 		// We clean the output and the temp directory
@@ -137,29 +155,29 @@ public void launch_main_filebyfile() throws KraftwerkException {
 
 	@When("Step 2 : We get each unimodal dataset")
 	public void unimodal_treatments() throws NullException {
-		metadataModelMap = MetadataUtils.getMetadata(userInputs.getModeInputsMap());
+		metadataModelMap = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap());
 		BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(true);
-		for (String dataMode : userInputs.getModeInputsMap().keySet()) {
+		for (String dataMode : userInputsFile.getModeInputsMap().keySet()) {
 			boolean withDDI = true;
-			buildBindingsSequence.buildVtlBindings(userInputs, dataMode, vtlBindings, metadataModelMap.get(dataMode), withDDI,null);
+			buildBindingsSequence.buildVtlBindings(userInputsFile, dataMode, vtlBindings, metadataModelMap.get(dataMode), withDDI,null);
 			UnimodalSequence unimodal = new UnimodalSequence();
-			unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, errors, metadataModelMap);
+			unimodal.applyUnimodalSequence(userInputsFile, dataMode, vtlBindings, errors, metadataModelMap);
 		}
 	}
 
 	@When("Step 3 : We aggregate each unimodal dataset into a multimodal dataset")
 	public void aggregate_datasets() {
 		MultimodalSequence multimodalSequence = new MultimodalSequence();
-		multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModelMap);
+		multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModelMap);
 	}
 
 	@When("Step 4 : We export the final version")
 	public void export_results() throws KraftwerkException {
 		WriterSequence writerSequence = new WriterSequence();
 		LocalDateTime localDateTime = LocalDateTime.now();
-		writerSequence.writeOutputFiles(inDirectory, localDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModelMap, errors);
+		writerSequence.writeOutputFiles(inDirectory, localDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModelMap, errors);
 		writeErrorsFile(inDirectory, localDateTime, errors);
-		outputFiles = new CsvOutputFiles(outDirectory, vtlBindings, userInputs.getModes());
+		outputFiles = new CsvOutputFiles(outDirectory, vtlBindings, userInputsFile.getModes());
 	}
 
 	@Then("Step 5 : We check if we have {int} lines")
@@ -317,4 +335,18 @@ public void clean_vtl(String vtlScriptName) throws IOException {
 
 		Files.deleteIfExists(bkpPath);
 	}
+
+	@Then("We should have a metadata model with {int} variables")
+	public void check_variables_count(int nbVariablesExpected) throws IOException, CsvValidationException {
+		String mode = userInputsFile.getModes().getFirst();
+		int nbVariables = metadataModelMap.get(mode).getVariables().getVariables().size();
+		assertThat(nbVariables).isEqualTo(nbVariablesExpected);
+	}
+
+	@And("We should have {int} of type STRING")
+	public void check_string_variables_count(int nbStringVariablesExpected) throws IOException, CsvValidationException {
+		String mode = userInputsFile.getModes().getFirst();
+		int nbStringVariables = metadataModelMap.get(mode).getVariables().getVariables().values().stream().filter(v -> v.getType()== VariableType.STRING).toArray().length;
+		assertThat(nbStringVariables).isEqualTo(nbStringVariablesExpected);
+	}
 }

kraftwerk-functional-tests/src/test/resources/features/do_we_get_metadata_correctly.feature

@@ -0,0 +1,31 @@
+Feature: Do we retrieve correctly all metadata from specifications ?
+
+  Scenario Outline: Does the metadata model is correct with Lunatic?
+    Given Step 0 : We have some survey in directory "<Directory>"
+    When Step 1 : We initialize metadata model with lunatic specification only
+    Then We should have a metadata model with <NumberOfVariables> variables
+    And We should have <NumberOfStringVariables> of type STRING
+
+    Examples:
+    # Parameters :
+    # - Directory : Directory of test campaigns
+    # - NumberOfVariables : Expected number of variables identified in Lunatic Json File
+    # - NumberOfStringVariables : Expected number of variables of String Type
+
+      |Directory                        |NumberOfVariables              | NumberOfStringVariables    |
+      |SAMPLETEST-METADATA              |370                            | 174                        |
+
+  Scenario Outline: Does the metadata model is correct with DDI?
+    Given Step 0 : We have some survey in directory "<Directory>"
+    When Step 1 : We initialize metadata model with DDI specification only
+    Then We should have a metadata model with <NumberOfVariables> variables
+    And We should have <NumberOfStringVariables> of type STRING
+
+    Examples:
+    # Parameters :
+    # - Directory : Directory of test campaigns
+    # - NumberOfVariables : Expected number of variables identified in Lunatic Json File
+    # - NumberOfStringVariables : Expected number of variables of String Type
+
+      |Directory                        |NumberOfVariables              | NumberOfStringVariables    |
+      |SAMPLETEST-METADATA              |370                            | 163                        |