diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index fbeb74eb..47790a3c 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -28,8 +28,20 @@ jobs: java-version: '21' distribution: 'temurin' cache: maven + + - name: Clone BPM + uses: actions/checkout@master + with: + repository: InseeFr/BPM + path: bpm + + - name: Build BPM + run: | + cd bpm + mvn clean install + cd .. - - name: Build with Maven + - name: Build Kraftwerk with Maven run: mvn -B clean package --file pom.xml # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive diff --git a/.gitignore b/.gitignore index 7bb2ce1a..b082d7bf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,8 +8,6 @@ local_files/logs/* # Config src/main/resources/application-*.properties -!src/main/resources/application-local.properties -src/main/resources/kraftwerk.properties # Compiled files target/ diff --git a/kraftwerk-api/.gitignore b/kraftwerk-api/.gitignore new file mode 100644 index 00000000..dd88f8b4 --- /dev/null +++ b/kraftwerk-api/.gitignore @@ -0,0 +1,2 @@ +# Config +src/main/resources/kraftwerk.properties \ No newline at end of file diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java index 9721429a..ca8116b8 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java @@ -4,7 +4,6 @@ import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.ModeInputs; import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.MetadataUtils; import fr.insee.kraftwerk.core.sequence.BuildBindingsSequence; import fr.insee.kraftwerk.core.sequence.ControlInputSequence; @@ -17,6 +16,9 @@ import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; import fr.insee.kraftwerk.core.vtl.VtlBindings; + +import fr.insee.bpm.metadata.model.MetadataModel; + import lombok.Getter; import lombok.extern.log4j.Log4j2; diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java index 13224c3c..cfdea9c2 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java @@ -1,5 +1,8 @@ package fr.insee.kraftwerk.api.process; +import fr.insee.bpm.exceptions.MetadataParserException; +import fr.insee.bpm.metadata.model.MetadataModel; + import fr.insee.kraftwerk.api.client.GenesisClient; import fr.insee.kraftwerk.api.configuration.ConfigProperties; import fr.insee.kraftwerk.core.KraftwerkError; @@ -7,7 +10,6 @@ import fr.insee.kraftwerk.core.data.model.SurveyUnitUpdateLatest; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.UserInputsGenesis; -import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.MetadataUtilsGenesis; import fr.insee.kraftwerk.core.sequence.BuildBindingsSequenceGenesis; import fr.insee.kraftwerk.core.sequence.ControlInputSequenceGenesis; @@ -74,8 +76,12 @@ public void init(String idCampaign) throws KraftwerkException { //We build userInputs for the given questionnaire userInputs = new UserInputsGenesis(controlInputSequenceGenesis.isHasConfigFile(), inDirectory, client.getModes(idCampaign), fileUtilsInterface); if (!userInputs.getModes().isEmpty()) { - metadataModels = MetadataUtilsGenesis.getMetadata(userInputs.getModeInputsMap(), fileUtilsInterface); - } else { + try { + metadataModels = MetadataUtilsGenesis.getMetadata(userInputs.getModeInputsMap(), fileUtilsInterface); + } catch (MetadataParserException e) { + throw new KraftwerkException(500, e.getMessage()); + } + } else { log.error("No source found for campaign " + idCampaign); } } diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/MainService.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/MainService.java index c287520a..7bf34e7b 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/MainService.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/MainService.java @@ -1,12 +1,12 @@ package fr.insee.kraftwerk.api.services; +import fr.insee.bpm.exceptions.MetadataParserException; import fr.insee.kraftwerk.api.configuration.ConfigProperties; import fr.insee.kraftwerk.api.configuration.MinioConfig; import fr.insee.kraftwerk.api.process.MainProcessing; import fr.insee.kraftwerk.api.process.MainProcessingGenesis; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.sequence.ControlInputSequenceGenesis; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.files.MinioImpl; diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java index be3bfce1..7723ac28 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java @@ -1,12 +1,12 @@ package fr.insee.kraftwerk.api.services; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.api.configuration.MinioConfig; import fr.insee.kraftwerk.api.process.MainProcessing; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.StepEnum; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.MetadataUtils; import fr.insee.kraftwerk.core.sequence.*; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; diff --git a/kraftwerk-api/src/main/resources/kraftwerk.properties b/kraftwerk-api/src/main/resources/kraftwerk.properties deleted file mode 100644 index 734ae64a..00000000 --- a/kraftwerk-api/src/main/resources/kraftwerk.properties +++ /dev/null @@ -1,21 +0,0 @@ -########################################## -## Properties that are fixed by OPS ## -########################################## - -# Folders for in and out -fr.insee.postcollecte.files = *** -# Minio for kube datas -fr.insee.postcollecte.minio.endpoint = *** -fr.insee.postcollecte.minio.access_key = *** -fr.insee.postcollecte.minio.secret_key = *** -fr.insee.postcollecte.minio.enable = false -fr.insee.postcollecte.minio.bucket_name = *** - -# LOGS -logging.level.root= INFO -logging.level.fr.insee.kraftwerk=INFO -logging.file.name= ***\\kraftwerk.log -logging.pattern.rolling-file-name= ***\\kraftwerk-%d{yyyy-MM-dd}.%i.log - -# Genesis API -fr.insee.postcollecte.genesis.api.url=*** diff --git a/kraftwerk-core/pom.xml b/kraftwerk-core/pom.xml index ff33faf7..6dff929a 100644 --- a/kraftwerk-core/pom.xml +++ b/kraftwerk-core/pom.xml @@ -112,5 +112,12 @@ duckdb_jdbc ${duckdb.version} + + + + fr.insee.bpm + bpm + 1.0.0 + \ No newline at end of file diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java index f6046afe..95c1b52d 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java @@ -1,7 +1,8 @@ package fr.insee.kraftwerk.core.dataprocessing; +import fr.insee.bpm.metadata.model.CalculatedVariables; + import fr.insee.kraftwerk.core.KraftwerkError; -import fr.insee.kraftwerk.core.metadata.CalculatedVariables; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java index 1317e6d7..9e5fb717 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java @@ -1,9 +1,9 @@ package fr.insee.kraftwerk.core.dataprocessing; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.PaperUcq; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.kraftwerk.core.KraftwerkError; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.PaperUcq; -import fr.insee.kraftwerk.core.metadata.VariablesMap; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessingManager.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessingManager.java index a8257fe3..e813bcee 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessingManager.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessingManager.java @@ -1,6 +1,7 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; + import fr.insee.kraftwerk.core.parsers.DataFormat; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessing.java index cda4868c..cd2fe2d0 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessing.java @@ -1,7 +1,8 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.VariablesMap; +import fr.insee.bpm.metadata.model.MetadataModel; + +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsProcessing.java index 3848d713..b39822c2 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsProcessing.java @@ -1,7 +1,7 @@ package fr.insee.kraftwerk.core.dataprocessing; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlMacros; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/LunaticDataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/LunaticDataProcessing.java index 40d17c09..2906e450 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/LunaticDataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/LunaticDataProcessing.java @@ -1,6 +1,7 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.metadata.MetadataModel; + +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessing.java index 1df415ef..f753a31a 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessing.java @@ -1,8 +1,8 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.UcqModality; -import fr.insee.kraftwerk.core.metadata.UcqVariable; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.UcqModality; +import fr.insee.bpm.metadata.model.UcqVariable; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java index 17839cd1..a815b899 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java @@ -1,7 +1,7 @@ package fr.insee.kraftwerk.core.dataprocessing; import fr.insee.kraftwerk.core.KraftwerkError; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.TextFileReader; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/UnimodalDataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/UnimodalDataProcessing.java index f20f0873..9c475d70 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/UnimodalDataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/UnimodalDataProcessing.java @@ -1,6 +1,6 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/XformsDataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/XformsDataProcessing.java index 81bb30ad..acd4b50f 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/XformsDataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/XformsDataProcessing.java @@ -1,6 +1,6 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParadataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParadataParser.java index 475764de..6e0a125f 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParadataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParadataParser.java @@ -2,10 +2,10 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.NullException; -import fr.insee.kraftwerk.core.metadata.Group; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; -import fr.insee.kraftwerk.core.metadata.VariablesMap; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; @@ -13,12 +13,10 @@ import org.json.simple.JSONArray; import org.json.simple.JSONObject; -import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; import java.time.LocalDateTime; import java.util.*; -import java.util.stream.Stream; @Log4j2 public class ParadataParser { diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java index 3ed7d107..a7516d34 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java @@ -2,9 +2,9 @@ import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.Group; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.DateUtils; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/CalculatedVariables.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/CalculatedVariables.java deleted file mode 100644 index f673bfe1..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/CalculatedVariables.java +++ /dev/null @@ -1,63 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import fr.insee.kraftwerk.core.metadata.CalculatedVariables.CalculatedVariable; -import lombok.Getter; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; - -/** - * Map to store specific information concerning the calculated variables. - * Keys: a variable name - * Values: CalculatedVariable objects. - * */ -public class CalculatedVariables extends LinkedHashMap { - - private static final long serialVersionUID = -2884154114234943033L; - - /** Register a calculated variable in the map. - * The object name attribute is the key of the entry in the map. */ - public void putVariable(CalculatedVariable calculatedVariable) { - this.put(calculatedVariable.getName(), calculatedVariable); - } - - /** Get the VTL expression of a registered variable. */ - public String getVtlExpression(String calculatedName) { - return this.get(calculatedName).getVtlExpression(); - } - - /** Get the dependant variables of a calculated variable. */ - public List getDependantVariables(String calculatedName) { - return this.get(calculatedName).getDependantVariables(); - } - - - /** POJO class to store specific information of a calculated variable. */ - @Getter - public static class CalculatedVariable { - - /** Variable name (should be the same as in the DDI) */ - String name; - /** VTL expression that defines the variable (read in the Lunatic questionnaire). */ - String vtlExpression; - /** Variables needed to perform the calculation. */ - List dependantVariables; - - public CalculatedVariable(String name, String vtlExpression) { - this.name = name; - this.vtlExpression = vtlExpression; - this.dependantVariables = new ArrayList<>(); - } - - public CalculatedVariable(String name, String vtlExpression, List dependantVariables) { - this.name = name; - this.vtlExpression = vtlExpression; - this.dependantVariables = dependantVariables; - } - - public void addDependantVariable(String name) { - dependantVariables.add(name); - } - } -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/ComponentLunatic.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/ComponentLunatic.java deleted file mode 100644 index f4d6f3ac..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/ComponentLunatic.java +++ /dev/null @@ -1,41 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import lombok.Getter; -import lombok.extern.log4j.Log4j2; - -import java.util.Arrays; - -@Log4j2 -@Getter -public enum ComponentLunatic { - - DATE_PICKER("Datepicker", VariableType.DATE), - CHECKBOX_BOOLEAN("CheckboxBoolean", VariableType.BOOLEAN), - INPUT_NUMBER("InputNumber", null), - INPUT("Input", VariableType.STRING), - TEXT_AREA("Textarea", VariableType.STRING), - RADIO("Radio", VariableType.STRING), - CHECKBOX_ONE("CheckboxOne", VariableType.STRING), - DROPDOWN("Dropdown", VariableType.STRING), - CHECKBOX_GROUP("CheckboxGroup", VariableType.BOOLEAN), - SUGGESTER("Suggester", VariableType.STRING), - PAIRWISE_LINKS("PairwiseLinks", null), - TABLE("Table", null); - - private String jsonName; - // Represents the type of the variable expected with this component type - // If null, the type is not unique - private VariableType type; - - ComponentLunatic(String jsonName, VariableType type) { - this.jsonName=jsonName; - this.type = type; - } - - public static ComponentLunatic fromJsonName(String jsonName) { - return Arrays.stream(ComponentLunatic.values()) - .filter(component -> component.getJsonName().equals(jsonName)) - .findFirst() - .orElse(null); - } -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/DDIReader.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/DDIReader.java deleted file mode 100644 index bfa5beb2..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/DDIReader.java +++ /dev/null @@ -1,275 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import fr.insee.kraftwerk.core.utils.xsl.SaxonTransformer; -import lombok.extern.log4j.Log4j2; -import org.apache.commons.lang3.StringUtils; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; - -import javax.xml.XMLConstants; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; - -@Log4j2 -public class DDIReader { - - private DDIReader() { - //Utility class - } - - // DDI should stay in KW and use DOM (not Jaxb) - - /** - * This method apply the XSLT_STRUCTURED_VARIABLES transformation to the DDI, - * then reads the output xml to return the variables. The XML file generated is - * written in the system temporary folder with the name 'variables.xml'. This file is - * deleted after being used or when the virtual machine terminates. - * - * @param ddiUrlString : Path to the DDI file. - * - * @return The variables found in the DDI. - * @throws KraftwerkException - */ - public static MetadataModel getMetadataFromDDI(String ddiUrlString, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { - - try { - // Path of the output 'variables.xml' temp file - File variablesFile = File.createTempFile("variables", ".xml"); - variablesFile.deleteOnExit(); - Path variablesTempFilePath = variablesFile.toPath(); - - transformDDI(ddiUrlString, variablesTempFilePath, fileUtilsInterface); - - MetadataModel metadataModel = readVariables(variablesTempFilePath); - Files.delete(variablesFile.toPath()); - return metadataModel; - } - - catch (MalformedURLException e) { - log.error(String.format("Error when converting file path '%s' to an URL.", ddiUrlString), e); - return null; - } catch (IOException e) { - log.error("Unable to write temp file.", e); - return null; - } catch (SAXException | ParserConfigurationException e) { - log.error("Unable to read Variables in DDI file.", e); - return null; - } - } - - /** - * Apply the XSLT_STRUCTURED_VARIABLES transformation. - * - * @param ddiUrlString : URL of the DDI file. - * @param variablesFilePath : Path of the 'variables.xml' file to be generated. - */ - private static void transformDDI(String ddiUrlString, Path variablesFilePath, FileUtilsInterface fileUtilsInterface) throws IOException { - SaxonTransformer saxonTransformer = new SaxonTransformer(fileUtilsInterface); - saxonTransformer.xslTransform(ddiUrlString, Constants.XSLT_STRUCTURED_VARIABLES, variablesFilePath); - } - - /** - * Parse the transformed xml file (using XmlFileReader), and read the - * transformed xml to return a VariablesMap. - * - * @param variablesFilePath Path to the transformed xml file. - * @return The variables described in the file. - * @throws KraftwerkException - * @throws IOException - * @throws SAXException - * @throws ParserConfigurationException - */ - private static MetadataModel readVariables(Path variablesFilePath) - throws KraftwerkException, SAXException, IOException, ParserConfigurationException { - MetadataModel metadataModel = new MetadataModel(); - - // Parse - Element root = readXmlFile(variablesFilePath); - - // Get XML groups - NodeList groupElements = root.getChildNodes(); - - // temporary save the root group name to normalize it - String rootGroupName = null; - - for (int i = 0; i < groupElements.getLength(); i++) { - try { - Node groupNode = groupElements.item(i); - if (nodeIsElementWithName(groupNode, "Group")) { - // Get the group name - Element groupElement = (Element) groupNode; - - String groupName = groupElement.getAttribute("name"); - String parentGroupName = groupElement.getAttribute("parent"); - // To be normalized later : we don't want to create a group of variables for pairwise links. - // Pairwise links scope are the individuals - if (groupName.equals(Constants.PAIRWISE_GROUP_NAME)){ - // Instead we assign temporarily the variables inside pairwise links group to the root group - getVariablesInGroup(metadataModel.getVariables(), groupNode, metadataModel.getRootGroup(), metadataModel.getSequences()); - continue; - } - - // Store the group - Group group; - if (StringUtils.isEmpty(parentGroupName)) { - rootGroupName = groupName; - group = metadataModel.getRootGroup(); - } else { - group = new Group(groupName, parentGroupName); - metadataModel.putGroup(group); - } - - // Variables in the group - getVariablesInGroup(metadataModel.getVariables(), groupNode, group, metadataModel.getSequences()); - - } - } catch (NullPointerException e) { - log.error(String.format("Missing field in mandatory information for variable %s", - ((Element) groupElements.item(i)).getAttribute("name"))); - } - - for (String groupName : metadataModel.getSubGroupNames()) { - Group group = metadataModel.getGroup(groupName); - if (group.getParentName().equals(rootGroupName)) { - group.setParentName(Constants.ROOT_GROUP_NAME); - } - } - - } - // Normalize the root group name - if (rootGroupName == null) { - log.debug("Failed to identify the root group while reading variables files: " + variablesFilePath); - } - return metadataModel; - } - - - - private static void getVariablesInGroup(VariablesMap variablesMap, Node groupNode, Group group, List sequences) { - NodeList variableNodes = groupNode.getChildNodes(); - for (int j = 0; j < variableNodes.getLength(); j++) { - Node variableNode = variableNodes.item(j); - if (nodeIsElementWithName(variableNode, "Variable")) { - addVariableToVariablesMap(variablesMap, group, variableNode, sequences); - } - } - } - - private static void addVariableToVariablesMap(VariablesMap variablesMap, Group group, Node variableNode, List sequences) { - Element variableElement = (Element) variableNode; - - // Variable name, type and size - String variableName = getFirstChildValue(variableElement, "Name"); - VariableType variableType = VariableType.valueOf(getFirstChildValue(variableElement, "Format")); - String variableLength = getFirstChildValue(variableElement, "Size"); - String sequenceName= getFirstChildAttribute(variableElement, "Sequence","name"); - - Node questionItemName = getFirstChildNode(variableElement, "QuestionItemName"); - Node valuesElement = getFirstChildNode(variableElement, "Values"); - Node mcqElement = getFirstChildNode(variableElement, "QGrid"); - - if (sequenceName != null){ - Sequence sequence = new Sequence(sequenceName); - if (sequences.isEmpty() || !sequences.contains(sequence)){ - sequences.add(sequence); - } - } - - if (valuesElement != null) { - UcqVariable variable = new UcqVariable(variableName, group, variableType, variableLength); - if (questionItemName != null) { - variable.setQuestionItemName(questionItemName.getTextContent()); - } else if (mcqElement != null) { - variable.setQuestionItemName(mcqElement.getTextContent()); - variable.setInQuestionGrid(true); - } - NodeList valueElements = valuesElement.getChildNodes(); - addValues(variable, valueElements); - variablesMap.putVariable(variable); - } else if (mcqElement != null) { - McqVariable variable = new McqVariable(variableName, group, variableType, variableLength); - variable.setQuestionItemName(mcqElement.getTextContent()); - variable.setInQuestionGrid(true); - variable.setText(getFirstChildValue(variableElement, "Label")); - variablesMap.putVariable(variable); - } else { - Variable variable = new Variable(variableName, group, variableType, variableLength); - if (questionItemName != null) { - variable.setQuestionItemName(questionItemName.getTextContent()); - } else { - variable.setQuestionItemName(variableName); - } - variablesMap.putVariable(variable); - } - } - - private static void addValues(UcqVariable variable, NodeList valueElements) { - for (int k = 0; k < valueElements.getLength(); k++) { - Node valueElement = valueElements.item(k); - if (nodeIsElementWithName(valueElement, "Value")) { - variable.addModality(valueElement.getTextContent(), - ((Element) valueElement).getAttribute("label")); - - } - } - } - - private static boolean nodeIsElementWithName(Node groupNode, String name) { - return name.equals(groupNode.getNodeName()) && groupNode.getNodeType() == Node.ELEMENT_NODE; - } - private static Element readXmlFile(Path variablesFilePath) - throws ParserConfigurationException, SAXException, IOException, KraftwerkException { - System.setProperty("javax.xml.parsers.DocumentBuilderFactory", - "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - - factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); - factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); - DocumentBuilder builder = factory.newDocumentBuilder(); - try(InputStream inputStream = Files.newInputStream(variablesFilePath)){ - Document document = builder.parse(inputStream); - if (document == null) - throw new KraftwerkException(500, "Can't read DDI - DDI is null"); - return document.getDocumentElement(); - } - } - - private static String getFirstChildValue(Element variableElement, String childTagName) { - Node child = getFirstChildNode(variableElement, childTagName); - if (child == null){ - return null; - } - return child.getTextContent(); - } - - private static String getFirstChildAttribute(Element variableElement, String childTagName, String attribute){ - Node child = getFirstChildNode(variableElement, childTagName); - if (child == null) { - return null; - } - return child.hasAttributes() ? child.getAttributes().getNamedItem(attribute).getTextContent() : null; - } - - private static Node getFirstChildNode(Element variableElement, String childTagName) { - NodeList children = variableElement.getElementsByTagName(childTagName); - if (children == null){ - return null; - } - return children.item(0); - } - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/ErrorVariableLength.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/ErrorVariableLength.java index 838a5efd..969f098c 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/ErrorVariableLength.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/ErrorVariableLength.java @@ -1,5 +1,6 @@ package fr.insee.kraftwerk.core.metadata; +import fr.insee.bpm.metadata.model.Variable; import fr.insee.kraftwerk.core.KraftwerkError; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/Group.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/Group.java deleted file mode 100644 index 426711d6..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/Group.java +++ /dev/null @@ -1,44 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import lombok.Getter; -import lombok.NonNull; -import lombok.Setter; -import lombok.extern.log4j.Log4j2; - -/** - * Object class to represent a group of variables. - */ -@Log4j2 -@Getter -@Setter -public class Group { - - protected String name; - protected String parentName; - - /** Protected constructor designed to create the root group. - * Root group is (only) created in VariablesMap constructor. */ - protected Group(String name) { - this.name = name; - this.parentName = null; - } - - public Group(@NonNull String name, @NonNull String parentName){ - this.name = name; - this.parentName = parentName; - if (parentName.isEmpty()) { - String msg = "Parent group name must be provided, \"\" name is invalid."; - log.debug(msg); - throw new IllegalArgumentException(msg); - } - } - - - private boolean hasParent() { - return parentName != null && !(parentName.isEmpty()); - } - public boolean isRoot() { - return !hasParent(); - } - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/LunaticReader.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/LunaticReader.java deleted file mode 100644 index 3e6b2127..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/LunaticReader.java +++ /dev/null @@ -1,492 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import com.fasterxml.jackson.databind.JsonNode; -import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.CalculatedVariables.CalculatedVariable; -import fr.insee.kraftwerk.core.utils.JsonReader; -import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import lombok.extern.log4j.Log4j2; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.*; - -import static fr.insee.kraftwerk.core.Constants.FILTER_RESULT_PREFIX; -import static fr.insee.kraftwerk.core.Constants.MISSING_SUFFIX; - -@Log4j2 -public class LunaticReader { - - private static final String BINDING_DEPENDENCIES = "bindingDependencies"; - private static final String VARIABLES = "variables"; - private static final String EXCEPTION_MESSAGE = "Unable to read Lunatic questionnaire file: "; - private static final String RESPONSE = "response"; - private static final String COMPONENTS = "components"; - private static final String COMPONENT_TYPE = "componentType"; - private static final String VALUE = "value"; - private static final String LABEL = "label"; - private static final String MISSING_RESPONSE = "missingResponse"; - private static final String LUNATIC_MODEL_VERSION= "lunaticModelVersion"; - - private LunaticReader() { - throw new IllegalStateException("Utility class"); - } - - /** - * Read the lunatic questionnaire given to get VTL expression of calculated - * variables. - * - * @param lunaticFile Path to a lunatic questionnaire file. - * @return A CalculatedVariables map. - */ - public static CalculatedVariables getCalculatedFromLunatic(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { - try { - JsonNode rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); - String lunaticModelVersion = rootNode.get(LUNATIC_MODEL_VERSION).asText(); - boolean isLunaticV2 = compareVersions(lunaticModelVersion, "2.3.0") > 0; - - CalculatedVariables calculatedVariables = new CalculatedVariables(); - - JsonNode variablesNode = rootNode.get(VARIABLES); - variablesNode.forEach(variableNode -> { - if (variableNode.get("variableType").asText().equals("CALCULATED")) { - String formula = isLunaticV2 ? variableNode.get("expression").get(VALUE).asText().replace("\r\n","") - : variableNode.get("expression").asText().replace("\r\n",""); - CalculatedVariable calculatedVariable = new CalculatedVariable(variableNode.get("name").asText(), - formula); - JsonNode dependantVariablesNode = variableNode.get(BINDING_DEPENDENCIES); - if (dependantVariablesNode != null) { - dependantVariablesNode.forEach(name -> calculatedVariable.addDependantVariable(name.asText())); - } - calculatedVariables.putVariable(calculatedVariable); - } - }); - - return calculatedVariables; - - } catch (IOException e) { - log.error(EXCEPTION_MESSAGE + lunaticFile); - return new CalculatedVariables(); - } - } - - /** - * Read the lunatic file to get the names of the _MISSING variables and the - * collected variables added by Eno which are not present in the DDI - * - * @param lunaticFile Path to a lunatic questionnaire file. - * @return A List of String. - */ - public static List getMissingVariablesFromLunatic(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { - try { - JsonNode rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); - List variables = new ArrayList<>(); - List varsEno = Arrays.asList(Constants.getEnoVariables()); - - JsonNode variablesNode = rootNode.get(VARIABLES); - variablesNode.forEach(variableNode -> variables.add(variableNode.get("name").asText())); - return variables.stream() - .filter(varToRead -> varToRead.endsWith(MISSING_SUFFIX) || varsEno.contains(varToRead)).toList(); - - } catch (IOException e) { - log.error(EXCEPTION_MESSAGE + lunaticFile); - return Collections.emptyList(); - } - } - - /** - * Read the lunatic file to get the names of the FILTER_RESULT variables which - * are not present in the DDI - * - * @param lunaticFile Path to a lunatic questionnaire file. - * @return A List of String. - */ - public static List getFilterResultFromLunatic(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { - try { - JsonNode rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); - List variables = new ArrayList<>(); - - JsonNode variablesNode = rootNode.get(VARIABLES); - variablesNode.forEach(variableNode -> variables.add(variableNode.get("name").asText())); - return variables.stream().filter(variable -> variable.startsWith(FILTER_RESULT_PREFIX)).toList(); - - } catch (IOException e) { - log.error(EXCEPTION_MESSAGE + lunaticFile); - return Collections.emptyList(); - } - } - - public static String getLunaticModelVersion(Path lunaticFile, FileUtilsInterface fileUtilsInterface){ - try { - JsonNode rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); - return rootNode.get(LUNATIC_MODEL_VERSION).toString(); - - } catch (IOException e) { - log.error(EXCEPTION_MESSAGE + lunaticFile); - return ""; - } - } - - /** - * This method extracts return the variables of a questionnaire without reading - * a DDI file. It should be used only when the DDI is not available. - * - * @param lunaticFile : Path to a Lunatic specification file. - * @return The variables found in the Lunatic specification. - */ - public static MetadataModel getMetadataFromLunatic(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { - log.info("Mode Lunatic-only : start reading file {} ",lunaticFile.getFileName()); - JsonNode rootNode; - try { - rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); - List variables = new ArrayList<>(); - JsonNode variablesNode = rootNode.get(VARIABLES); - variablesNode.forEach(newVar -> variables.add(newVar.get("name").asText())); - // Root group is created in VariablesMap constructor - MetadataModel metadataModel = new MetadataModel(); - metadataModel.putSpecVersions(SpecType.LUNATIC,rootNode.get(LUNATIC_MODEL_VERSION).asText()); - Group rootGroup = metadataModel.getRootGroup(); - iterateOnComponents(rootNode, variables, metadataModel, rootGroup); - - // We iterate on root components to identify variables belonging to root group - JsonNode rootComponents = rootNode.get(COMPONENTS); - for (JsonNode comp : rootComponents) { - addResponsesAndMissing(comp, rootGroup, variables, metadataModel); - } - // We add the remaining (not identified in any loops nor root) variables to the root group - variables.forEach( - varName -> metadataModel.getVariables().putVariable(new Variable(varName, rootGroup, VariableType.STRING))); - return metadataModel; - } catch (IOException e) { - log.error(EXCEPTION_MESSAGE + lunaticFile); - return null; - } - } - - /** - * This method iterates on an array of components to extract the variables present in loops and get their group. - * @param rootNode : node containing the components we want to iterate on - * @param variables : variables list to be completed - * @param metadataModel : metadata model of the questionnaire to be completed - * @param parentGroup : group of rootNode - */ - private static void iterateOnComponents(JsonNode rootNode, List variables, MetadataModel metadataModel, Group parentGroup) { - JsonNode componentsNode = rootNode.get(COMPONENTS); - if (componentsNode.isArray()) { - for (JsonNode component : componentsNode) { - if (component.get(COMPONENT_TYPE).asText().equals("Loop")) { - if (component.has("lines")) { - processPrimaryLoop(variables, metadataModel, parentGroup, component); - } - if (component.has("iterations")) { - Group group = processDependingLoop(variables, metadataModel, parentGroup, component); - iterateOnComponents(component,variables, metadataModel,group); - } - } - } - } - } - - /** - * This method processes the primary loop and creates a group with the name of the first response - * @param variables : variables list to be completed - * @param metadataModel : metadata model of the questionnaire to be completed - * @param parentGroup : parent group of the loop - * @param component : loop component - */ - private static void processPrimaryLoop(List variables, MetadataModel metadataModel, Group parentGroup, JsonNode component) { - JsonNode primaryComponents = component.get(COMPONENTS); - //We create a group only with the name of the first response - //Then we add all the variables found in response to the newly created group - String groupName = getFirstResponseName(primaryComponents); - log.info("Creation of group :" + groupName); - Group group = getNewGroup(metadataModel, groupName, parentGroup); - for (JsonNode primaryComponent : primaryComponents) { - addResponsesAndMissing(primaryComponent, group, variables, metadataModel); - } - } - - /** - * This method processes the loop depending on variables and creates a group with the name of loop dependencies - * @param variables : variables list to be completed - * @param metadataModel : metadata model of the questionnaire to be completed - * @param parentGroup : parent group of the loop - * @param component : loop component - * @return the group corresponding to the loop - */ - private static Group processDependingLoop(List variables, MetadataModel metadataModel, Group parentGroup, JsonNode component) { - JsonNode loopDependencies = component.get("loopDependencies"); - String groupName; - if (!loopDependencies.isEmpty()) { - StringBuilder groupNameBuilder = new StringBuilder(loopDependencies.get(0).asText()); - for (int i = 1; i < loopDependencies.size(); i++) { - groupNameBuilder.append("_").append(loopDependencies.get(i).asText()); - } - groupName = groupNameBuilder.toString(); - } else { - int i = 1; - groupName = "UNNAMED_" + i; - List groups = metadataModel.getGroupNames(); - while (groups.contains(groupName)) { - i++; - groupName = "UNNAMED_" + i; - } - } - log.info("Creation of group :" + groupName); - Group group = getNewGroup(metadataModel, groupName, parentGroup); - iterateOnComponentsToFindResponses(component, variables, metadataModel, group); - return group; - } - - private static String getFirstResponseName(JsonNode components){ - for(JsonNode component : components){ - if (component.has(RESPONSE)){ - return component.get(RESPONSE).get("name").asText(); - } - } - return null; - } - - /** - * Adds variables to the metadata model (it infers type of variables from the component type) - * Checks Lunatic version to adapt to the different ways of writing the JSON - * - * @param primaryComponent : a component of the questionnaire - * @param group : the group to which the variables belong - * @param variables : list of variables to be completed - * @param metadataModel : metadata model of the questionnaire to be completed - */ - private static void addResponsesAndMissing(JsonNode primaryComponent, Group group, List variables, MetadataModel metadataModel) { - //We read the name of the collected variables in response(s) - //And we deduce the variable type by looking at the component that encapsulate the variable - ComponentLunatic componentType = ComponentLunatic.fromJsonName(primaryComponent.get(COMPONENT_TYPE).asText()); - String variableName; - boolean isLunaticV2 = compareVersions(metadataModel.getSpecVersions().get(SpecType.LUNATIC), "2.3.0") > 0; - switch(componentType){ - case ComponentLunatic.DATE_PICKER, ComponentLunatic.CHECKBOX_BOOLEAN, ComponentLunatic.INPUT, ComponentLunatic.TEXT_AREA, ComponentLunatic.SUGGESTER: - variableName = getVariableName(primaryComponent); - metadataModel.getVariables().putVariable(new Variable(variableName, group, componentType.getType())); - variables.remove(variableName); - break; - case ComponentLunatic.INPUT_NUMBER: - variableName = getVariableName(primaryComponent); - if (primaryComponent.get("decimals").asInt()==0){ - metadataModel.getVariables().putVariable(new Variable(variableName, group, VariableType.INTEGER)); - variables.remove(variableName); - break; - } - metadataModel.getVariables().putVariable(new Variable(variableName, group, VariableType.NUMBER)); - variables.remove(variableName); - break; - case ComponentLunatic.DROPDOWN: - variableName = getVariableName(primaryComponent); - UcqVariable ucqVar = new UcqVariable(variableName, group, VariableType.STRING); - JsonNode modalities = primaryComponent.get("options"); - for (JsonNode modality : modalities){ - ucqVar.addModality(modality.get(VALUE).asText(), modality.get(LABEL).asText()); - } - metadataModel.getVariables().putVariable(ucqVar); - variables.remove(variableName); - break; - case ComponentLunatic.RADIO, ComponentLunatic.CHECKBOX_ONE: - variableName = getVariableName(primaryComponent); - UcqVariable ucqVarOne = new UcqVariable(variableName, group, VariableType.STRING); - JsonNode modalitiesOne = primaryComponent.get("options"); - for (JsonNode modality : modalitiesOne){ - if (isLunaticV2) { - ucqVarOne.addModality(modality.get(VALUE).asText(), modality.get(LABEL).get(VALUE).asText()); - continue; - } - ucqVarOne.addModality(modality.get(VALUE).asText(), modality.get(LABEL).asText()); - } - metadataModel.getVariables().putVariable(ucqVarOne); - variables.remove(variableName); - break; - case ComponentLunatic.CHECKBOX_GROUP: - processCheckboxGroup(primaryComponent, group, variables, metadataModel, isLunaticV2); - break; - case ComponentLunatic.PAIRWISE_LINKS: - // In we case of a pairwiseLinks component we have to iterate on the components to find the responses - // It is a nested component, but we treat it differently than the loops because it does not create a new level of information - iterateOnComponentsToFindResponses(primaryComponent, variables, metadataModel, group); - break; - case ComponentLunatic.TABLE: - iterateOnTableBody(primaryComponent, group, variables, metadataModel, isLunaticV2); - break; - case null: - log.warn(String.format("%s component type not recognized", primaryComponent.get(COMPONENT_TYPE).asText())); - break; - } - //We also had the missing variable if it exists (only one missing variable even if multiple responses) - addMissingVariable(primaryComponent, group, variables, metadataModel); - } - - /** - * Process a checkbox group to create a boolean variable for each response - * @param checkboxComponent : component representing a checkbox group - * @param group : group to which the variables belong - * @param variables : list of variables to be completed - * @param metadataModel : metadata model of the questionnaire to be completed - * @param isLunaticV2 : true if the Lunatic version is 2.3 or higher - */ - private static void processCheckboxGroup(JsonNode checkboxComponent, Group group, List variables, MetadataModel metadataModel, boolean isLunaticV2) { - String variableName; - JsonNode responses = checkboxComponent.get("responses"); - List responsesName= new ArrayList<>(); - for (JsonNode response : responses){ - responsesName.add(getVariableName(response)); - } - String questionName = findLongestCommonPrefix(responsesName); - for (JsonNode response : responses){ - variableName = getVariableName(response); - McqVariable mcqVariable = new McqVariable(variableName, group, VariableType.BOOLEAN); - if (isLunaticV2) mcqVariable.setText(response.get(LABEL).get(VALUE).asText()); - if (!isLunaticV2) mcqVariable.setText(response.get(LABEL).asText()); - mcqVariable.setInQuestionGrid(true); - mcqVariable.setQuestionItemName(questionName); - metadataModel.getVariables().putVariable(mcqVariable); - variables.remove(variableName); - } - } - - /** - * Iterate on the components in the body of a table to find the responses - * @param tableComponent : component representing a table - * @param group : group to which the variables belong - * @param variables : list of variables to be completed - * @param metadataModel : metadata model of the questionnaire to be completed - * @param isLunaticV2 : true if the Lunatic version is 2.3 or higher - */ - private static void iterateOnTableBody(JsonNode tableComponent, Group group, List variables, MetadataModel metadataModel, boolean isLunaticV2) { - // In we case of a table component we have to iterate on the body components to find the responses - // The body is a nested array of arrays - // In Lunatic 2.2 and lower the body is called cells - JsonNode body = isLunaticV2 ? tableComponent.get("body") : tableComponent.get("cells"); - for(JsonNode arr : body){ - if (arr.isArray()){ - for (JsonNode cell : arr){ - if (cell.has(COMPONENT_TYPE)) { - addResponsesAndMissing(cell, group, variables, metadataModel); - } - } - } - } - } - - /** - * Add the missing variable defined in the component if present - * @param component : a questionnaire component - * @param group : group to which the variables belong - * @param variables : list of variables to be completed - * @param metadataModel : metadata model of the questionnaire to be completed - */ - private static void addMissingVariable(JsonNode component, Group group, List variables, MetadataModel metadataModel) { - if (component.has(MISSING_RESPONSE)){ - String missingVariable = component.get(MISSING_RESPONSE).get("name").asText(); - metadataModel.getVariables().putVariable(new Variable(missingVariable, group, VariableType.STRING)); - variables.remove(missingVariable); - } - } - - /** - * Get the name of the variable collected by a component - * @param component : a questionnaire component - * @return the name of the variable - */ - private static String getVariableName(JsonNode component) { - return component.get(RESPONSE).get("name").asText(); - } - - private static void iterateOnComponentsToFindResponses(JsonNode node, List variables, MetadataModel metadataModel, Group group) { - JsonNode components = node.get(COMPONENTS); - if (components.isArray()){ - for (JsonNode component : components) { - addResponsesAndMissing(component, group, variables, metadataModel); - } - } - } - - private static Group getNewGroup(MetadataModel metadataModel, String newName, Group parentGroup) { - Group group = new Group(String.format("%s_%s",Constants.LOOP_NAME_PREFIX,newName), parentGroup.getName()); - metadataModel.putGroup(group); - return group; - } - - /** - * Read the lunatic file and returns a String containing the questionnaire model - * id - * - * @param lunaticFile : Path to a Lunatic specification file. - * @return the questionnaire model id - */ - public static String getQuestionnaireModelId(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { - JsonNode rootNode; - try { - rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); - return rootNode.get("id").asText(); - } catch (IOException e) { - log.error(EXCEPTION_MESSAGE + lunaticFile); - return null; - } - } - - /** - * Find the common part of a list of strings that differs only at the end - * - * @param similarStrings : list of strings - * @return the common prefix - */ - public static String findLongestCommonPrefix(List similarStrings) { - int minLength = similarStrings.getFirst().length(); - for(String str : similarStrings){ - if (str.length() groups = new LinkedHashMap<>(); - - private List sequences = new ArrayList<>(); - - private HashMap specVersions = new HashMap<>(); - - - /** The root group is created when creating a MetadataModel instance. */ - public MetadataModel() { - groups.put(Constants.ROOT_GROUP_NAME, new Group(Constants.ROOT_GROUP_NAME)); - } - - - /** Register a group in the map. */ - public void putGroup(Group group) { - groups.put(group.getName(), group); - } - /** Return the group with given name. */ - public Group getGroup(String groupName) { - return groups.get(groupName); - } - /** Return the root group. */ - public Group getRootGroup() { - if (! groups.containsKey(Constants.ROOT_GROUP_NAME)) { - log.debug("Root group not in the variables map."); - } - return groups.get(Constants.ROOT_GROUP_NAME); - } - /** Return the reporting data group. */ - public Group getReportingDataGroup() { - if (! groups.containsKey(Constants.REPORTING_DATA_GROUP_NAME)) { - log.debug("Reporting data group not in the variables map."); - } - return groups.get(Constants.REPORTING_DATA_GROUP_NAME); - } - /** Return the name of all groups registered in the map, including the root group. */ - public List getGroupNames() { - return new ArrayList<>(groups.keySet()); - } - /** Return the names of all groups registered in the map, except the root group. */ - public List getSubGroupNames() { - return groups.keySet() - .stream().filter(name -> ! groups.get(name).isRoot()) - .toList(); - } - - /** Return the number of groups in the map (including the root group). */ - public int getGroupsCount() { - return groups.size(); - } - - /** Identifiers are not represented by Variable objects, they are: - * - the root identifier (fixed value), - * - each subgroup name is also an identifier name. - * @return The list of all identifiers associated to the variables map. */ - public List getIdentifierNames() { - List res = new ArrayList<>(List.of(Constants.ROOT_IDENTIFIER_NAME)); - res.addAll(getSubGroupNames()); - return res; - } - - /** Return true if there is a group under the given name. */ - public boolean hasGroup(String groupName) { - return groups.containsKey(groupName); - } - - public void putSpecVersions(SpecType specType, String lunaticModelVersion) { - specVersions.put(specType, lunaticModelVersion); - } - - /** Return the fully qualified name of a variable, that is - * - the name of the variable if it is in the root group. - * - the variable name prefixed with its group and parent group names, otherwise. - * In the second case, the separator use is defined by Constants.METADATA_SEPARATOR. */ - public String getFullyQualifiedName(String variableName) { - if (getVariables().hasVariable(variableName) && StringUtils.isNotEmpty(variableName)) { - - /* done using StringBuilder, maybe concatenate a list of strings is better - https://stackoverflow.com/a/523913/13425151 */ - - StringBuilder res = new StringBuilder(variableName); - Variable variable = getVariables().getVariables().get(variableName); - Group group = variable.getGroup(); - while(! group.isRoot()) { - res.insert(0, group.getName() + Constants.METADATA_SEPARATOR); - group = groups.get(group.getParentName()); - } - return res.toString(); - } - else { - log.debug(String.format( "Trying to get fully qualified name for unknown variable \"%s\". null returned.", - variableName)); - return null; - } - } - - public Set getFullyQualifiedNames() { - return getVariables().getVariables().keySet().stream().map(this::getFullyQualifiedName).collect(Collectors.toSet()); - } - - public Set getDistinctVariableNamesAndFullyQualifiedNames(){ - Set set = new HashSet<>(); - set.addAll(getFullyQualifiedNames()); - set.addAll(getVariables().getVariableNames()); - return set; - } - - public List getSequencesName(){ - return sequences.stream().map(Sequence::getName).toList(); - } - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtils.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtils.java index 5dbf7cd0..2c1e8837 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtils.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtils.java @@ -1,7 +1,16 @@ package fr.insee.kraftwerk.core.metadata; +import fr.insee.bpm.exceptions.MetadataParserException; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.MetadataModel; + +import fr.insee.bpm.metadata.model.SpecType; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.bpm.metadata.reader.ddi.DDIReader; +import fr.insee.bpm.metadata.reader.lunatic.LunaticReader; + import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.ModeInputs; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; @@ -27,22 +36,22 @@ private static void putToMetadataModels(String dataMode, ModeInputs modeInputs, // Step 1 : we add the variables read in the DDI MetadataModel metadataModel = new MetadataModel(); try { - metadataModel = DDIReader.getMetadataFromDDI(modeInputs.getDdiUrl(), fileUtilsInterface); - } catch (KraftwerkException e) { + metadataModel = DDIReader.getMetadataFromDDI(modeInputs.getDdiUrl(), fileUtilsInterface.readFile(modeInputs.getDdiUrl())); + } catch (MetadataParserException e) { log.error(e.getMessage()); } // Step 2 : we add the variables that are only present in the Lunatic file if (modeInputs.getLunaticFile() != null) { log.info("Adding variables from Lunatic file : {}", modeInputs.getLunaticFile().getFileName()); // We read and store lunaticModelVersion - metadataModel.putSpecVersions(SpecType.LUNATIC,LunaticReader.getLunaticModelVersion(modeInputs.getLunaticFile(), fileUtilsInterface)); + metadataModel.putSpecVersions(SpecType.LUNATIC,LunaticReader.getLunaticModelVersion(fileUtilsInterface.readFile(modeInputs.getLunaticFile().toString()))); // First we add the collected _MISSING variables - List missingVars = LunaticReader.getMissingVariablesFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); + List missingVars = LunaticReader.getMissingVariablesFromLunatic(fileUtilsInterface.readFile(modeInputs.getLunaticFile().toString())); for (String missingVar : missingVars) { addLunaticVariable(metadataModel, missingVar, Constants.MISSING_SUFFIX, VariableType.STRING); } // Then we add calculated FILTER_RESULT_ variables - List filterResults = LunaticReader.getFilterResultFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); + List filterResults = LunaticReader.getFilterResultFromLunatic(fileUtilsInterface.readFile(modeInputs.getLunaticFile().toString())); for (String filterResult : filterResults) { addLunaticVariable(metadataModel, filterResult, Constants.FILTER_RESULT_PREFIX, VariableType.BOOLEAN); } @@ -81,7 +90,7 @@ public static Map getMetadataFromLunatic(Map metadataModels, FileUtilsInterface fileUtilsInterface) { - MetadataModel metadataModel = LunaticReader.getMetadataFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); + MetadataModel metadataModel = LunaticReader.getMetadataFromLunatic(fileUtilsInterface.readFile(modeInputs.getLunaticFile().toString())); // We add the variables for pairwise links if (metadataModel.getVariables().getVariable(Constants.LIENS) != null) { // We identify the group containing the individuals diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsGenesis.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsGenesis.java index 5279a71d..b08a6573 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsGenesis.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsGenesis.java @@ -1,7 +1,14 @@ package fr.insee.kraftwerk.core.metadata; +import fr.insee.bpm.exceptions.MetadataParserException; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.bpm.metadata.reader.ddi.DDIReader; +import fr.insee.bpm.metadata.reader.lunatic.LunaticReader; + import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.ModeInputs; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; @@ -17,7 +24,7 @@ private MetadataUtilsGenesis() { throw new IllegalStateException("Utility class"); } - public static Map getMetadata(Map modeInputsMap, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { + public static Map getMetadata(Map modeInputsMap, FileUtilsInterface fileUtilsInterface) throws MetadataParserException { Map metadataModels = new LinkedHashMap<>(); for (Map.Entry entry : modeInputsMap.entrySet()) { String k = entry.getKey(); @@ -27,19 +34,19 @@ public static Map getMetadata(Map mod return metadataModels; } - private static void putToMetadataVariable(String dataMode, ModeInputs modeInputsGenesis, Map metadataModels, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { + private static void putToMetadataVariable(String dataMode, ModeInputs modeInputsGenesis, Map metadataModels, FileUtilsInterface fileUtilsInterface) throws MetadataParserException { // Step 1 : we add the variables read in the DDI - MetadataModel metadataModel = DDIReader.getMetadataFromDDI(modeInputsGenesis.getDdiUrl(), fileUtilsInterface); + MetadataModel metadataModel = DDIReader.getMetadataFromDDI(modeInputsGenesis.getDdiUrl(), fileUtilsInterface.readFile(modeInputsGenesis.getDdiUrl())); // Step 2 : we add the variables that are only present in the Lunatic file if (modeInputsGenesis.getLunaticFile() != null) { // First we add the collected _MISSING variables - List missingVars = LunaticReader.getMissingVariablesFromLunatic(modeInputsGenesis.getLunaticFile(), fileUtilsInterface); + List missingVars = LunaticReader.getMissingVariablesFromLunatic(fileUtilsInterface.readFile(modeInputsGenesis.getLunaticFile().toString())); for (String missingVar : missingVars) { addLunaticVariable(metadataModel, missingVar, Constants.MISSING_SUFFIX, VariableType.STRING); } // Then we add calculated FILTER_RESULT_ variables - List filterResults = LunaticReader.getFilterResultFromLunatic(modeInputsGenesis.getLunaticFile(), fileUtilsInterface); + List filterResults = LunaticReader.getFilterResultFromLunatic(fileUtilsInterface.readFile(modeInputsGenesis.getLunaticFile().toString())); for (String filterResult : filterResults) { addLunaticVariable(metadataModel, filterResult, Constants.FILTER_RESULT_PREFIX, VariableType.BOOLEAN); } @@ -73,7 +80,7 @@ public static Map getMetadataFromLunatic(Map metadataModels, FileUtilsInterface fileUtilsInterface) { - MetadataModel metadataModel = LunaticReader.getMetadataFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); + MetadataModel metadataModel = LunaticReader.getMetadataFromLunatic(fileUtilsInterface.readFile(modeInputs.getLunaticFile().toString())); metadataModels.put(dataMode, metadataModel); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/PaperUcq.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/PaperUcq.java deleted file mode 100644 index 07f1f92a..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/PaperUcq.java +++ /dev/null @@ -1,28 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import lombok.Getter; - -/** In paper data files, UCQ variables are split into several indicator variables, - * that are not defined in the DDI. - * This class is meant to store this type of variable, - * and link them to the corresponding UCQ variable defined in the DDI. */ -@Getter -public class PaperUcq extends Variable { - - String ucqName; - - /** - * Variable's group is set using the UCQ variable given. - * Variable's type is always BOOLEAN. - * The name of this variable is registered in the UCQ variable given. - * @param name Indicator variable name. - * @param ucqVariable Instance of UCQ variable associated with this indicator variable. - * @param correspondingValue Value associated with the modality that this indicator variable represents. - */ - public PaperUcq(String name, UcqVariable ucqVariable, String correspondingValue) { - super(name, ucqVariable.getGroup(), VariableType.BOOLEAN); - this.ucqName = ucqVariable.getName(); - ucqVariable.getModalityFromValue(correspondingValue).setVariableName(name); - } - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/Sequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/Sequence.java deleted file mode 100644 index 0dd72923..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/Sequence.java +++ /dev/null @@ -1,13 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import lombok.Data; -import lombok.NonNull; - - -@Data -public class Sequence { - - @NonNull - String name; - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/SpecType.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/SpecType.java deleted file mode 100644 index 6ec0f264..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/SpecType.java +++ /dev/null @@ -1,6 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -public enum SpecType { - - DDI,LUNATIC -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/UcqModality.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/UcqModality.java deleted file mode 100644 index f1f07802..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/UcqModality.java +++ /dev/null @@ -1,42 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import lombok.Getter; -import lombok.Setter; - -/** - * POJO class to store information about a UQC modality. - */ -@Getter -public class UcqModality { - - /** - * Value associated to the modality in survey data. - */ - @Setter - String value; - /** - * Text associated to the modality - */ - @Setter - String text; - /** - * If an indicator variable is associated to the modality (in a paper data - * files). - */ - @Setter - String variableName; - - UcqModality() { - } - - UcqModality(String value, String text) { - this.value = value; - this.text = text; - } - - UcqModality(String value, String text, String variableName) { - this.value = value; - this.text = text; - this.variableName = variableName; - } -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/UcqVariable.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/UcqVariable.java deleted file mode 100644 index 8e5fe470..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/UcqVariable.java +++ /dev/null @@ -1,129 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.log4j.Log4j2; - -/** One unique choice question = one variable. */ -@Getter -@Log4j2 -public class UcqVariable extends Variable { - - /** - * Map to store the UCQ modalities. Keys: possible values. Values: text - * associated. - */ - List modalities = new ArrayList<>(); - - - /** Common name of the different modalities of the UCQ variable, if this variable is itself a modality of a Mcq variable. */ - @Setter - String mcqName; - - public UcqVariable(String name, Group group, VariableType type) { - super(name, group, type); - if (type != VariableType.STRING) { - log.warn(String.format("%s type given when creating UCQ \"%s\"", type, name)); - log.warn("Type of a UCQ variable should be STRING."); - } - } - - public UcqVariable(String name, Group group, VariableType type, String variableLength) { - super(name, group, type, variableLength); - if (type != VariableType.STRING) { - log.warn(String.format("%s type given when creating UCQ \"%s\"", type, name)); - log.warn("Type of a UCQ variable should be STRING."); - } - } - - /** Add a modality using the object. */ - public void addModality(UcqModality modality) { - modalities.add(modality); - } - - /** - * Method to register a modality of the UCQ. - * - * @param value A possible value of the variable. - * @param text The text associated with the value. - */ - public void addModality(String value, String text) { - modalities.add(new UcqModality(value, text)); - } - - /** - * Method to register a modality of the UCQ. - * - * @param value A possible value of the UCQ variable. - * @param text The text associated with the value. - * @param variableName The name of the variable associated with the modality - * (when carrying out paper data). - */ - public void addModality(String value, String text, String variableName) { - modalities.add(new UcqModality(value, text, variableName)); - } - - /** Return the possible values of the variable. */ - public Set getValues() { - return modalities.stream().map(ucqModality -> ucqModality.value).collect(Collectors.toSet()); - } - - /** Return the names of eventual variables associated with modalities. */ - public Set getModalityNames() { - return modalities.stream().map(ucqModality -> ucqModality.variableName).collect(Collectors.toSet()); - } - - /** - * Get a modality from its associated value. - * - * @param value One of the possible values of the variable. - * @return The modality object that corresponds to this value. If there is no - * modality with this value, a null is returned. If several modalities - * correspond to the value, the first modality that corresponds is - * returned. - */ - public UcqModality getModalityFromValue(String value) { - List res = modalities.stream().filter(ucqModality -> ucqModality.getValue().equals(value)) - .toList(); - if (res.size() == 1) { - return res.get(0); - } else if (res.isEmpty()) { - log.debug(String.format("Value \"%s\" not registered in UCQ named \"%s\".", value, name)); - return null; - } else { - log.debug(String.format("Several modalities with value \"%s\" are registered in UCQ named \"%s\".", value, - name)); - return res.get(0); - } - } - - /** - * Get a modality from its name (that may be filled when carrying out paper - * data). - * - * @param modalityName The variable name of a modality of the UCQ. - * @return The modality object that corresponds to this name. If there is no - * modality with this name, a null is returned. If several modalities - * correspond to the name, the first modality that corresponds is - * returned. - */ - public UcqModality getModalityFromName(String modalityName) { - List res = modalities.stream() - .filter(ucqModality -> ucqModality.getVariableName().equals(modalityName)).toList(); - if (res.size() == 1) { - return res.get(0); - } else if (res.isEmpty()) { - log.debug(String.format("Modality name \"%s\" not registered in UCQ named \"%s\".", modalityName, name)); - return null; - } else { - log.debug(String.format("Several modality variables named \"%s\" are registered in UCQ named \"%s\".", - modalityName, name)); - return res.get(0); - } - } -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/Variable.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/Variable.java deleted file mode 100644 index 4426bf35..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/Variable.java +++ /dev/null @@ -1,74 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -/** - * Object class to represent a variable. - * - */ -@Slf4j -@Getter -public class Variable { - - /** Variable name. */ - protected String name; - - /** Group reference */ - protected Group group; - - /** Variable type from the enum class (STRING, INTEGER, DATE, ...) */ - protected VariableType type; - - /** Format for SAS script import */ - protected String sasFormat; - - /** Maximum length received in input for the variable. */ - @Setter - protected int maxLengthData; - - /** Name of the item used to collect the answer. */ - @Setter - protected String questionItemName; - - /** Identifies if the variable is part a question grid */ - @Setter - protected boolean isInQuestionGrid; - - public Variable(String name, Group group, VariableType type) { - this.name = name; - this.group = group; - this.type = type; - } - - public Variable(String name, Group group, VariableType type, String sasFormat) { - this.name = name; - this.group = group; - this.type = type; - if (!"".equals(sasFormat)) this.sasFormat = sasFormat; - } - - public String getGroupName() { - return group.getName(); - } - - public int getExpectedLength(){ - if (this.sasFormat != null && this.sasFormat.contains(".")){ - String[] sasFormatPart = this.sasFormat.split("\\."); - return Integer.parseInt(sasFormatPart[0]); - } - if (this.sasFormat != null){ - try { - return Integer.parseInt(this.sasFormat); - } - catch (NumberFormatException e){ - log.error("Variable {} expected length is not a number : {} ", name, sasFormat); - return 1; - } - } - // Not sure about that return - return 1; - } - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/VariableType.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/VariableType.java deleted file mode 100644 index 3ea03895..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/VariableType.java +++ /dev/null @@ -1,56 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import java.time.Instant; -import java.util.Date; - -import lombok.Getter; -import lombok.extern.log4j.Log4j2; - -/** - * Enum class for variable types. - */ -@Log4j2 -@Getter -public enum VariableType { - STRING ("character","character", "STRING", "VARCHAR"), - INTEGER ("integer","integer", "INTEGER", "BIGINT"), - NUMBER ("number","numeric", "NUMBER", "DOUBLE"), - BOOLEAN ("logical","logical", "BOOLEAN", "BOOLEAN"), - DATE ("Date","Date", "STRING", "DATE"); - - private final String dataTableType; - private final String formatR; - private final String vtlType; - private final String sqlType; - - - VariableType(String dataTableType,String formatR, String vtlType, String sqlType) { - this.dataTableType =dataTableType; - this.formatR=formatR; - this.vtlType=vtlType; - this.sqlType=sqlType; - } - - public static VariableType getTypeFromJavaClass(Class clazz){ - // memo: https://www.w3schools.com/java/java_data_types.asp - if(clazz.isAssignableFrom(Float.class) - || clazz.isAssignableFrom(Double.class)){ - return NUMBER; - } else if (clazz.isAssignableFrom(Integer.class) - || clazz.isAssignableFrom(Long.class) - || clazz.isAssignableFrom(Short.class)){ - return INTEGER; - } else if (clazz.isAssignableFrom(String.class) - || clazz.isAssignableFrom(Character.class)){ - return STRING; - } else if (clazz.isAssignableFrom(Boolean.class)){ - return BOOLEAN; - } else if (clazz.isAssignableFrom(Date.class)||clazz.isAssignableFrom(Instant.class)){ - return DATE; - } else { - log.warn(String.format("Unrecognized type for class %s ", clazz)); - return null; - } - } - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/VariablesMap.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/VariablesMap.java deleted file mode 100644 index ed676c41..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/VariablesMap.java +++ /dev/null @@ -1,158 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - - -import lombok.Getter; -import lombok.extern.log4j.Log4j2; -import org.apache.commons.lang3.StringUtils; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Object class to represent a set of variables. - * Contains a flat map and its structured equivalent. - */ -@Log4j2 -@Getter -public class VariablesMap { - - /** Map containing the variables. - * Keys: a variable name. - * Values: Variable. */ - protected final LinkedHashMap variables = new LinkedHashMap<>(); - - - - /** Register a variable in the map. */ - public void putVariable(Variable variable) { - if (StringUtils.isEmpty(variable.getName())){return;} - variables.put(variable.getName(), variable); - } - - /** Remove the variable with given name from the map. */ - public void removeVariable(String name){ - if (variables.get(name) != null) { - variables.remove(name); - } else { - log.debug(String.format("Variable named \"%s\" is not in the variables map", name)); - } - } - - /** Return the variable with given name. */ - public Variable getVariable(String variableName){ - Variable variable = variables.get(variableName); - if(variable == null) { - log.debug(String.format("Variable named \"%s\" is unknown", variableName)); - } - return variable; - } - /** Return the names of all variables in the map. */ - public Set getVariableNames() { - return variables.keySet(); - } - - - /** Return the variables names that belongs to the group. */ - public Set getGroupVariableNames(String groupName) { - return variables.keySet() - .stream().filter(name -> variables.get(name).getGroupName().equals(groupName)) - .collect(Collectors.toSet()); - } - - - /** Return true if there is a variable under the given name. */ - public boolean hasVariable(String variableName) { - return variables.containsKey(variableName); - } - - /** Return true if there is a McqVariable that has the given question name in its mcqName attribute. */ - public boolean hasMcq(String questionName) { - return variables.values().stream() - .filter(McqVariable.class::isInstance) - .anyMatch(mcqVariable -> // (FILTER_RESULT variables are upper case) - mcqVariable.getQuestionItemName().equals(questionName) - || mcqVariable.getQuestionItemName().equalsIgnoreCase(questionName)); - } - - public Group getMcqGroup(String questionName) { - return variables.values().stream() - .filter(McqVariable.class::isInstance) - .filter(mcqVariable -> mcqVariable.getQuestionItemName().equals(questionName)) - .map(Variable::getGroup) - .findFirst().orElse(null); - } - - /** Return true if there is a variable from a question grid that has the given question name in its questionName attribute. */ - public boolean isInQuestionGrid(String questionName){ - return variables.values().stream() - .filter(Variable::isInQuestionGrid) - .anyMatch(varInQuestionGrid -> // (FILTER_RESULT variables are upper case) - (varInQuestionGrid.getQuestionItemName().equals(questionName) - || varInQuestionGrid.getQuestionItemName().equalsIgnoreCase(questionName))); - } - - public Group getQuestionGridGroup(String questionName) { - return variables.values().stream() - .filter(Variable::isInQuestionGrid) - .filter(varInQuestionGrid -> varInQuestionGrid.getQuestionItemName().equals(questionName)) - .map(Variable::getGroup) - .findFirst().orElse(null); - } - - /** Return true if there is a UCQ variable with the name given. */ - public boolean hasUcq(String variableName) { - Variable variable = variables.get(variableName); - if (variable != null) { - return variable instanceof UcqVariable; - } - return false; - } - - /** Return true if there is a UCQ variable depending on a MCQ variable with the name given. */ - public boolean hasUcqMcq(String variableName) { - Variable variable = variables.get(variableName); - if (variable != null) { - return variable instanceof UcqVariable ucqVariable && !ucqVariable.getQuestionItemName().isEmpty(); - } - return false; - } - /** Return the list of all UCQ variables registered in the map. */ - public List getUcqVariables() { - return variables.values().stream() - .filter(UcqVariable.class::isInstance) - .map(UcqVariable.class::cast) - .toList(); - } - /** Return the list of all names of UCQ variables registered in the map. */ - public List getUcqVariablesNames() { - return variables.values().stream() - .filter(UcqVariable.class::isInstance) - .map(Variable::getQuestionItemName) - .filter(Objects::nonNull) - .distinct() - .toList(); - } - /** Return the list of all names of MCQ variables registered in the map. */ - public List getMcqVariablesNames() { - return variables.values().stream() - .filter(McqVariable.class::isInstance) - .map(Variable::getQuestionItemName) - .filter(Objects::nonNull) - .distinct() - .toList(); - } - /** Return the list of all paper UCQ indicators registered in the map. */ - public List getPaperUcq() { - return variables.values().stream() - .filter(PaperUcq.class::isInstance) - .map(PaperUcq.class::cast) - .toList(); - } - - - - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/ImportScript.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/ImportScript.java index 3e6fddb1..b63e80be 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/ImportScript.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/ImportScript.java @@ -1,9 +1,9 @@ package fr.insee.kraftwerk.core.outputs; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.vtl.model.Structured; import java.util.LinkedHashMap; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java index 53b3b7a5..9019250b 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java @@ -3,7 +3,7 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.Getter; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/TableScriptInfo.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/TableScriptInfo.java index cd296905..a127bcbf 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/TableScriptInfo.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/TableScriptInfo.java @@ -1,6 +1,6 @@ package fr.insee.kraftwerk.core.outputs; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.vtl.model.Structured.DataStructure; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java index 6b63cb63..208e762f 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java @@ -3,8 +3,8 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.outputs.OutputFiles; import fr.insee.kraftwerk.core.outputs.TableScriptInfo; import fr.insee.kraftwerk.core.utils.SqlUtils; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/RImportScript.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/RImportScript.java index 96aa61d1..045376d2 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/RImportScript.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/RImportScript.java @@ -1,9 +1,9 @@ package fr.insee.kraftwerk.core.outputs.csv; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.outputs.ImportScript; import fr.insee.kraftwerk.core.outputs.TableScriptInfo; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/SASImportScript.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/SASImportScript.java index b360b62d..7a0deeb0 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/SASImportScript.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/SASImportScript.java @@ -3,9 +3,9 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.metadata.ErrorVariableLength; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.outputs.ImportScript; import fr.insee.kraftwerk.core.outputs.TableScriptInfo; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/parquet/ParquetOutputFiles.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/parquet/ParquetOutputFiles.java index 3baa68b4..5bd77934 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/parquet/ParquetOutputFiles.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/parquet/ParquetOutputFiles.java @@ -2,7 +2,7 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.outputs.OutputFiles; import fr.insee.kraftwerk.core.outputs.TableScriptInfo; import fr.insee.kraftwerk.core.utils.TextFileWriter; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParser.java index f759b2d5..be71b43e 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParser.java @@ -1,7 +1,11 @@ package fr.insee.kraftwerk.core.parsers; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.bpm.metadata.reader.lunatic.LunaticReader; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.*; import fr.insee.kraftwerk.core.rawdata.GroupData; import fr.insee.kraftwerk.core.rawdata.GroupInstance; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; @@ -62,7 +66,7 @@ private void parseDataFile(Path filePath, Path lunaticFile) { .getChildElements("SurveyUnit"); String questionnaireModelId = null; if (lunaticFile != null) { - questionnaireModelId = LunaticReader.getQuestionnaireModelId(lunaticFile, fileUtilsInterface); + questionnaireModelId = LunaticReader.getQuestionnaireModelId(fileUtilsInterface.readFile(lunaticFile.toString())); } for (int i = 0; i < questionnaireNodeList.size(); i++) { @@ -325,7 +329,7 @@ private Elements getCalculatedElements(Element questionnaireNode) { * Check if the given value is a VTL expression using the 'forbiddenWords' * attribute. * - * @see fr.insee.kraftwerk.core.metadata.LunaticReader + * @see fr.insee.bpm.metadata.reader.lunatic.LunaticReader * @see fr.insee.kraftwerk.core.dataprocessing.CalculatedProcessing */ private static boolean isNotVtlExpression(String value) { diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/PaperDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/PaperDataParser.java index c421b017..5604c628 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/PaperDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/PaperDataParser.java @@ -6,10 +6,10 @@ import com.opencsv.CSVReaderBuilder; import com.opencsv.exceptions.CsvValidationException; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.PaperUcq; -import fr.insee.kraftwerk.core.metadata.UcqVariable; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariablesMap; +import fr.insee.bpm.metadata.model.PaperUcq; +import fr.insee.bpm.metadata.model.UcqVariable; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.kraftwerk.core.rawdata.GroupInstance; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/rawdata/SurveyRawData.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/rawdata/SurveyRawData.java index 88457f5f..a73f012a 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/rawdata/SurveyRawData.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/rawdata/SurveyRawData.java @@ -1,7 +1,7 @@ package fr.insee.kraftwerk.core.rawdata; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; import lombok.Getter; import lombok.Setter; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java index 237bb2e6..850a9cd7 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java @@ -9,7 +9,7 @@ import fr.insee.kraftwerk.core.extradata.reportingdata.XMLReportingDataParser; import fr.insee.kraftwerk.core.inputs.ModeInputs; import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.parsers.DataParser; import fr.insee.kraftwerk.core.parsers.DataParserManager; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java index f473f490..e0f401e2 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java @@ -11,7 +11,7 @@ import fr.insee.kraftwerk.core.extradata.reportingdata.CSVReportingDataParser; import fr.insee.kraftwerk.core.extradata.reportingdata.ReportingData; import fr.insee.kraftwerk.core.extradata.reportingdata.XMLReportingDataParser; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.rawdata.GroupData; import fr.insee.kraftwerk.core.rawdata.GroupInstance; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java index 4e88876d..62f41eff 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java @@ -11,7 +11,7 @@ import fr.insee.kraftwerk.core.dataprocessing.MultimodeTransformations; import fr.insee.kraftwerk.core.dataprocessing.ReconciliationProcessing; import fr.insee.kraftwerk.core.inputs.UserInputs; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.vtl.VtlBindings; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java index 330e22b7..5636f1c5 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java @@ -1,5 +1,12 @@ package fr.insee.kraftwerk.core.sequence; +import fr.insee.bpm.metadata.model.CalculatedVariables; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.bpm.metadata.model.VariablesMap; + +import fr.insee.bpm.metadata.reader.lunatic.LunaticReader; import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.*; @@ -41,7 +48,7 @@ public void applyUnimodalSequence(UserInputs userInputs, String dataMode, VtlBin /* Step 2.4b : Apply VTL expression for calculated variables (if any) */ if (modeInputs.getLunaticFile() != null) { CalculatedVariables calculatedVariables = LunaticReader - .getCalculatedFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); + .getCalculatedFromLunatic(fileUtilsInterface.readFile(modeInputs.getLunaticFile().toString())); CalculatedProcessing calculatedProcessing = new CalculatedProcessing(vtlBindings, calculatedVariables, fileUtilsInterface); vtlGenerate = calculatedProcessing.applyCalculatedVtlTransformations(dataMode, modeInputs.getModeVtlFile(), errors); TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "CalculatedProcessing", dataMode), diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java index 807cc96b..b055c285 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java @@ -3,7 +3,7 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.ModeInputs; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.outputs.OutputFiles; import fr.insee.kraftwerk.core.outputs.csv.CsvOutputFiles; import fr.insee.kraftwerk.core.outputs.parquet.ParquetOutputFiles; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/SqlUtils.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/SqlUtils.java index 8cd6508d..2e8e1f7c 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/SqlUtils.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/SqlUtils.java @@ -1,7 +1,7 @@ package fr.insee.kraftwerk.core.utils; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Structured; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/ClasspathURIResolver.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/ClasspathURIResolver.java deleted file mode 100644 index 774200db..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/ClasspathURIResolver.java +++ /dev/null @@ -1,38 +0,0 @@ -package fr.insee.kraftwerk.core.utils.xsl; - -import javax.xml.transform.Source; -import javax.xml.transform.TransformerException; -import javax.xml.transform.URIResolver; -import javax.xml.transform.stream.StreamSource; - -import lombok.extern.log4j.Log4j2; - -/** - * Use for controlling the resolution of includes - * FIXME we need to urgently change the includes to match a simpler scheme - * i.e. import statements href are equal to /path/to/resources/directory - * */ - -@Log4j2 -public class ClasspathURIResolver implements URIResolver { - - @Override - public Source resolve(String href, String base) throws TransformerException { - log.debug("Resolving URI with href: " + href + " and base: " + base); - String resolvedHref; - if (href.startsWith("..")) { - if (href.startsWith("../..")) { - resolvedHref = href.replaceFirst("../..", "/xslt"); - log.debug("Resolved URI is: " + resolvedHref); - } else { - resolvedHref = href.replaceFirst("..", "/xslt"); - log.debug("Resolved XSLT URI is: " + resolvedHref); - } - } else { - resolvedHref = href; - log.debug("Resolved URI href is: " + resolvedHref); - } - return new StreamSource(ClasspathURIResolver.class.getResourceAsStream(resolvedHref)); - } - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformer.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformer.java deleted file mode 100644 index 29135f3a..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformer.java +++ /dev/null @@ -1,91 +0,0 @@ -package fr.insee.kraftwerk.core.utils.xsl; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URL; -import java.nio.file.Path; - -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.stream.StreamResult; -import javax.xml.transform.stream.StreamSource; - -import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import lombok.AllArgsConstructor; -import lombok.extern.log4j.Log4j2; - -@Log4j2 -@AllArgsConstructor -public class SaxonTransformer { - private FileUtilsInterface fileUtilsInterface; - - /** - * XSL transformation method using Saxon. - * - * @param inputXmlPath : Path of the XML input file - * @param inputXslPath : Path to the XSL file from the resources folder of the - * application - * @param outputXmlPath : Path to the XML output file which will be created - */ - public void xslTransform(String inputXmlPath, String inputXslPath, Path outputXmlPath) throws IOException { - log.info("About to transform the file: " + inputXmlPath); - - // Get the XML input file - StreamSource xmlSource; - InputStream xmlInput; - if(inputXmlPath.startsWith("http")){ - xmlInput = URI.create(inputXmlPath).toURL().openStream(); - }else{ - xmlInput = fileUtilsInterface.readFile(inputXmlPath); - } - xmlSource = new StreamSource(xmlInput); - xmlSource.setSystemId(inputXmlPath); - - // Get the XSL file - StreamSource xslSource; - InputStream xslInput; - xslInput = SaxonTransformer.class.getClassLoader().getResourceAsStream(inputXslPath); - xslSource = new StreamSource(xslInput); - xslSource.setSystemId(inputXslPath); - - // Instantiation of the XSL transformer factory - TransformerFactory transformerFactory = new net.sf.saxon.TransformerFactoryImpl(); // basic transformer - transformerFactory.setURIResolver(new ClasspathURIResolver()); - - // Apply the XSL transformation - try { - Transformer transformer = transformerFactory.newTransformer(xslSource); - StreamResult sr = new StreamResult(outputXmlPath.toFile()); - transformer.transform(xmlSource, sr); - } catch (TransformerConfigurationException e) { - log.error("Error when trying to configure the XSL transformer using XSL file: " + inputXslPath, e); - } catch (TransformerException e) { - log.error("Error when trying to apply the XSL transformation using XSL file: " + inputXslPath, e); - } - - try { - xmlInput.close(); - if(xslInput != null){ - xslInput.close(); - } - } catch (IOException | NullPointerException e ) { - log.error("IOException occurred when trying to close the streams after XSL transformation.", e); - } - } - - /** - * XSL transformation method using Saxon. - * - * @param inputXmlPath : Path to the XML input file - * @param inputXslPath : Path to the XSL file from the resources' folder of the - * application - * @param outputXmlPath : Path to the XML output file which will be created - */ - public void xslTransform(Path inputXmlPath, String inputXslPath, Path outputXmlPath) throws IOException { - xslTransform(inputXmlPath.toString(), inputXslPath, outputXmlPath); - } -} \ No newline at end of file diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlBindings.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlBindings.java index c0467c8d..e1f41cc1 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlBindings.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlBindings.java @@ -1,7 +1,11 @@ package fr.insee.kraftwerk.core.vtl; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; + import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.*; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Structured; import fr.insee.vtl.model.Structured.Component; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriter.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriter.java index 6b336b62..26949a7b 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriter.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriter.java @@ -1,9 +1,9 @@ package fr.insee.kraftwerk.core.vtl; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.rawdata.GroupData; import fr.insee.kraftwerk.core.rawdata.GroupInstance; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java index ba51b4c5..50b46cac 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java @@ -1,11 +1,10 @@ package fr.insee.kraftwerk.core.dataprocessing; import fr.insee.kraftwerk.core.KraftwerkError; -import fr.insee.kraftwerk.core.metadata.CalculatedVariables; -import fr.insee.kraftwerk.core.metadata.CalculatedVariables.CalculatedVariable; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.CalculatedVariables; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; @@ -36,11 +35,11 @@ static void setFooCalculated() { // fooCalculated = new CalculatedVariables(); fooCalculated.putVariable( - new CalculatedVariable("FOO1", "FOO2 + FOO3", List.of("FOO3", "FOO2"))); + new CalculatedVariables.CalculatedVariable("FOO1", "FOO2 + FOO3", List.of("FOO3", "FOO2"))); fooCalculated.putVariable( - new CalculatedVariable("FOO2", "FOO3", List.of("FOO3"))); + new CalculatedVariables.CalculatedVariable("FOO2", "FOO3", List.of("FOO3"))); fooCalculated.putVariable( - new CalculatedVariable("FOO3", "1")); + new CalculatedVariables.CalculatedVariable("FOO3", "1")); // fooMetadataModel = new MetadataModel(); fooMetadataModel.getVariables().putVariable(new Variable("FOO", fooMetadataModel.getRootGroup(), VariableType.STRING)); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java index 223b0728..c534848e 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java @@ -1,8 +1,12 @@ package fr.insee.kraftwerk.core.dataprocessing; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.PaperUcq; +import fr.insee.bpm.metadata.model.UcqVariable; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; -import fr.insee.kraftwerk.core.metadata.*; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java index bc7a3eed..1fab0e55 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java @@ -1,7 +1,10 @@ package fr.insee.kraftwerk.core.dataprocessing; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.KraftwerkError; -import fr.insee.kraftwerk.core.metadata.*; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java index d118a392..bd0962ac 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java @@ -1,7 +1,12 @@ package fr.insee.kraftwerk.core.dataprocessing; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.bpm.metadata.model.VariablesMap; + import fr.insee.kraftwerk.core.KraftwerkError; -import fr.insee.kraftwerk.core.metadata.*; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; @@ -15,8 +20,6 @@ import java.util.List; import java.util.Map; -import static org.junit.Assert.assertNotNull; - class PaperDataProcessingTest { private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); Dataset paperDataset = new InMemoryDataset( diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java index 9e40b6f4..2b27902c 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java @@ -2,8 +2,8 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.TestConstants; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Sequence; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Sequence; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/ParaDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/ParaDataParserTest.java index 61c9fb67..8a9538b7 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/ParaDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/ParaDataParserTest.java @@ -5,7 +5,7 @@ import fr.insee.kraftwerk.core.exceptions.NullException; import fr.insee.kraftwerk.core.extradata.paradata.Paradata; import fr.insee.kraftwerk.core.extradata.paradata.ParadataParser; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/DDIReaderTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/DDIReaderTest.java deleted file mode 100644 index fbc72ae3..00000000 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/DDIReaderTest.java +++ /dev/null @@ -1,170 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.TestConstants; -import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; -import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import org.junit.jupiter.api.Test; - -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.*; - -class DDIReaderTest { - - static final String DDI_FOLDER = TestConstants.UNIT_TESTS_DIRECTORY + "/ddi"; - - static final String DDI_SIMPSONS_V1 = "ddi-simpsons-v1.xml"; - static final String DDI_SIMPSONS_V2 = "ddi-simpsons-v2.xml"; - static final String DDI_VQS_WEB = "vqs-2021-x00-xforms-ddi.xml"; - static final String DDI_VQS_PAP = "vqs-2021-x00-fo-ddi.xml"; - static final String DDI_TIC_WEB = "tic-2021-a00-xforms-ddi.xml"; - static final String DDI_TIC_PAP = "tic-2021-a00-fo-ddi.xml"; - static final String DDI_LOG_T01 = "S1logement13juil_ddi.xml"; - static final String DDI_LOG_X01 = "ddi-log-2021-x01.xml"; - static final String DDI_LOG_X12 = "ddi-log-2021-x12-web.xml"; - static final String DDI_LOG_X21 = "ddi-log-2021-x21-web.xml"; - static final String DDI_LOG_X22 = "S2_WEB.xml"; - - static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); - - @Test - void readSimpsonsV1Variables() throws MalformedURLException, KraftwerkException, URISyntaxException { - - Set expectedVariables = Set.of( - // - "FAVOURITE_CHARACTERS11", "FAVOURITE_CHARACTERS102", - // - "SUM_EXPENSES", "LAST_BROADCAST", "COMMENT", "READY", "PRODUCER", "SEASON_NUMBER", "DATEFIRST", - "AUDIENCE_SHARE", "CITY", "MAYOR", "STATE", "PET1", "PET4", "ICE_FLAVOUR1", "ICE_FLAVOUR4", - "NUCLEAR_CHARACTER1", "NUCLEAR_CHARACTER4", "BIRTH_CHARACTER1", "BIRTH_CHARACTER5", - "PERCENTAGE_EXPENSES11", "PERCENTAGE_EXPENSES101", "CLOWNING11", "CLOWNING42", "TRAVEL11", "TRAVEL46", - "SURVEY_COMMENT"); - - MetadataModel simpsonsMetadata = DDIReader - .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_SIMPSONS_V1, fileUtilsInterface); - - // - assertNotNull(simpsonsMetadata); - // - assertTrue(simpsonsMetadata.hasGroup(Constants.ROOT_GROUP_NAME)); - assertTrue(simpsonsMetadata.hasGroup("FAVOURITE_CHARACTERS")); - // - for (String variableName : expectedVariables) { - assertTrue(simpsonsMetadata.getVariables().hasVariable(variableName)); - } - // - assertEquals(Constants.ROOT_GROUP_NAME, simpsonsMetadata.getVariables().getVariable("SUM_EXPENSES").getGroup().getName()); - assertEquals(Constants.ROOT_GROUP_NAME, simpsonsMetadata.getVariables().getVariable("SURVEY_COMMENT").getGroup().getName()); - assertEquals("FAVOURITE_CHARACTERS", - simpsonsMetadata.getVariables().getVariable("FAVOURITE_CHARACTERS11").getGroup().getName()); - assertEquals("FAVOURITE_CHARACTERS", - simpsonsMetadata.getVariables().getVariable("FAVOURITE_CHARACTERS102").getGroup().getName()); - } - - @Test - void readSimpsonsV2Variables() throws MalformedURLException, KraftwerkException, URISyntaxException { - - Set expectedVariables = Set.of( - // - "SUM_EXPENSES", "YEAR", "FAVOURITE_CHAR1", "FAVOURITE_CHAR2", "FAVOURITE_CHAR3", "FAVOURITE_CHAR33CL", - // - "NAME_CHAR", "AGE_CHAR", "FAVCHAR", "MEMORY_CHAR", - // - "LAST_BROADCAST", "COMMENT", "READY", "PRODUCER", "SEASON_NUMBER", "DATEFIRST", "DATEYYYYMM", - "DATEYYYY", "DURATIONH", "DURATIONHH", "AUDIENCE_SHARE", "CITY", "MAYOR", "MAYOROTCL", "STATE", "PET1", - "PET4", "PETOCL", "ICE_FLAVOUR1", "ICE_FLAVOUR4", "ICE_FLAVOUROTCL", "NUCLEAR_CHARACTER1", - "NUCLEAR_CHARACTER4", "BIRTH_CHARACTER1", "BIRTH_CHARACTER5", "PERCENTAGE_EXPENSES11", - "PERCENTAGE_EXPENSES71", "LAST_FOOD_SHOPPING11", "LAST_FOOD_SHOPPING81", "LAST_FOOD_SHOPPING113CL", - "LAST_FOOD_SHOPPING813CL", "CLOWNING11", "CLOWNING42", "TRAVEL11", "TRAVEL46", "FEELCHAREV1", - "FEELCHAREV4", "LEAVDURATION11", "LEAVDURATION52", "NB_CHAR", "SURVEY_COMMENT"); - - MetadataModel simpsonsMetadata = DDIReader - .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_SIMPSONS_V2, fileUtilsInterface); - - // - assertNotNull(simpsonsMetadata); - // - assertTrue(simpsonsMetadata.hasGroup(Constants.ROOT_GROUP_NAME)); - assertTrue(simpsonsMetadata.hasGroup("FAVOURITE_CHAR")); - assertTrue(simpsonsMetadata.hasGroup("Loop1")); - // - for (String variableName : expectedVariables) { - assertTrue(simpsonsMetadata.getVariables().hasVariable(variableName)); - } - // - assertEquals(Constants.ROOT_GROUP_NAME, simpsonsMetadata.getVariables().getVariable("LAST_BROADCAST").getGroup().getName()); - assertEquals(Constants.ROOT_GROUP_NAME, simpsonsMetadata.getVariables().getVariable("SURVEY_COMMENT").getGroup().getName()); - assertEquals("FAVOURITE_CHAR", simpsonsMetadata.getVariables().getVariable("SUM_EXPENSES").getGroup().getName()); - assertEquals("FAVOURITE_CHAR", simpsonsMetadata.getVariables().getVariable("FAVOURITE_CHAR1").getGroup().getName()); - assertEquals("Loop1", simpsonsMetadata.getVariables().getVariable("NAME_CHAR").getGroup().getName()); - } - - @Test - void readVqsWebVariables() throws MalformedURLException, KraftwerkException, URISyntaxException { - - Set expectedVariables = Set.of("prenom", "NOM", "SEXE", "DTNAIS", "ETAT_SANT", "APPRENT", "AIDREG_A", - "AIDREG_B", "AIDREG_C", "AIDREG_D", "RELATION1", "RELATION2", "RELATION3", "RELATION4", "ADRESSE", - "RESIDM", "NHAB"); - - MetadataModel vqsMetadata = DDIReader - .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_VQS_WEB.toString(), fileUtilsInterface); - - // - assertNotNull(vqsMetadata); - // - assertTrue(vqsMetadata.hasGroup(Constants.ROOT_GROUP_NAME)); - assertTrue(vqsMetadata.hasGroup("BOUCLEINDIV")); - // - for (String variableName : expectedVariables) { - assertTrue(vqsMetadata.getVariables().hasVariable(variableName)); - } - // - assertEquals(Constants.ROOT_GROUP_NAME, vqsMetadata.getVariables().getVariable("ADRESSE").getGroup().getName()); - assertEquals("BOUCLEINDIV", vqsMetadata.getVariables().getVariable("prenom").getGroup().getName()); - // UCQ - assertInstanceOf(UcqVariable.class, vqsMetadata.getVariables().getVariable("ETAT_SANT")); - UcqVariable etatSant = (UcqVariable) vqsMetadata.getVariables().getVariable("ETAT_SANT"); - assertEquals(5, etatSant.getModalities().size()); - // MCQ - assertInstanceOf(McqVariable.class, vqsMetadata.getVariables().getVariable("AIDREG_A")); - assertInstanceOf(McqVariable.class, vqsMetadata.getVariables().getVariable("AIDREG_D")); - assertInstanceOf(McqVariable.class, vqsMetadata.getVariables().getVariable("RELATION1")); - assertInstanceOf(McqVariable.class, vqsMetadata.getVariables().getVariable("RELATION4")); - McqVariable aidregA = (McqVariable) vqsMetadata.getVariables().getVariable("AIDREG_A"); - assertEquals("AIDREG", aidregA.getQuestionItemName()); - assertEquals("1 - Oui, une aide aux activités de la vie quotidienne", aidregA.getText()); - // - assertFalse(vqsMetadata.getVariables().getVariable("ADRESSE") instanceof McqVariable); - assertFalse(vqsMetadata.getVariables().getVariable("ADRESSE") instanceof UcqVariable); - assertFalse(vqsMetadata.getVariables().getVariable("PRENOM") instanceof McqVariable); - assertFalse(vqsMetadata.getVariables().getVariable("PRENOM") instanceof UcqVariable); - } - - @Test - void readVqsPapVariables() throws MalformedURLException, KraftwerkException, URISyntaxException { - - Set expectedVariables = Set.of("PRENOM", "NOM", "SEXE", "DTNAIS", "ETAT_SANT", "APPRENT", "AIDREG_A", - "AIDREG_B", "AIDREG_C", "AIDREG_D", "RESID", "RESIDANCIEN", "NBQUEST"); - - MetadataModel vqsMetadata = DDIReader - .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_VQS_PAP, fileUtilsInterface); - - // - assertNotNull(vqsMetadata); - // - assertTrue(vqsMetadata.hasGroup(Constants.ROOT_GROUP_NAME)); - assertTrue(vqsMetadata.hasGroup("BOUCLEINDIV")); - // - for (String variableName : expectedVariables) { - assertTrue(vqsMetadata.getVariables().hasVariable(variableName)); - } - // - assertEquals(Constants.ROOT_GROUP_NAME, vqsMetadata.getVariables().getVariable("NBQUEST").getGroup().getName()); - assertEquals("BOUCLEINDIV", vqsMetadata.getVariables().getVariable("PRENOM").getGroup().getName()); - } - -} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/GroupTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/GroupTest.java deleted file mode 100644 index 496ffae9..00000000 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/GroupTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertThrows; - - class GroupTest { - - @Test - // @SuppressWarnings({"null", "ConstantConditions"}) - void nullParentName() { - assertThrows(NullPointerException.class, () -> new Group("TEST", null)); - } - - @Test - void emptyParentName() { - assertThrows(IllegalArgumentException.class, () -> new Group("TEST", "")); - } -} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/LunaticReaderTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/LunaticReaderTest.java deleted file mode 100644 index ae480ce3..00000000 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/LunaticReaderTest.java +++ /dev/null @@ -1,70 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import fr.insee.kraftwerk.core.TestConstants; -import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; -import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import org.junit.jupiter.api.Test; - -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -class LunaticReaderTest { - - static final Path lunaticSamplesPath = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "lunatic"); - static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); - - @Test - void readLogX21TelLunaticFile() { - // - CalculatedVariables calculatedVariables = LunaticReader.getCalculatedFromLunatic( - lunaticSamplesPath.resolve("log2021x21_tel.json"), fileUtilsInterface); - - // - assertNotNull(calculatedVariables); - // - assertTrue(calculatedVariables.containsKey("AGE")); - assertEquals( - "if (FUTURANNIVERSAIRE) then cast((cast(AGEMILLESIME,integer) - 1),integer) else cast(AGEMILLESIME,integer)", - calculatedVariables.getVtlExpression("AGE")); - assertEquals( - List.of("FUTURANNIVERSAIRE", "AGEMILLESIME", "DATENAIS"), - calculatedVariables.getDependantVariables("AGE")); - // - assertTrue(calculatedVariables.containsKey("ANNEENQ")); - assertEquals( - "cast(current_date(),string,\"YYYY\")", - calculatedVariables.getVtlExpression("ANNEENQ")); - assertTrue(calculatedVariables.getDependantVariables("ANNEENQ").isEmpty()); - - } - - @Test - void readLogX22WebLunaticFile() { - // - CalculatedVariables calculatedVariables = LunaticReader.getCalculatedFromLunatic( - lunaticSamplesPath.resolve("log2021x22_web.json"), fileUtilsInterface); - - // - assertNotNull(calculatedVariables); - assertTrue(calculatedVariables.containsKey("S2_MAA1AT")); - - } - - - @Test - //Same test with DDI [2 groups, 463 variables] - void readVariablesFromLogX21WebLunaticFile() { - // - MetadataModel variables = LunaticReader.getMetadataFromLunatic( - lunaticSamplesPath.resolve("log2021x21_web.json"), fileUtilsInterface); - - // - assertNotNull(variables); - assertEquals(2,variables.getGroupsCount()); - assertEquals(683, variables.getVariables().getVariables().size()); - - } - -} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/MetadataModelTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/MetadataModelTest.java deleted file mode 100644 index d0eca6f0..00000000 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/MetadataModelTest.java +++ /dev/null @@ -1,294 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import fr.insee.kraftwerk.core.Constants; -import lombok.extern.log4j.Log4j2; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.*; - -@Log4j2 -public class MetadataModelTest { - - private MetadataModel metadataModel; - - @BeforeEach - public void createTestVariablesMap() { - metadataModel = new MetadataModel(); - - Group rootGroup = metadataModel.getRootGroup(); - Group individualsGroup = new Group("INDIVIDUALS_LOOP", Constants.ROOT_GROUP_NAME); - Group carsGroup = new Group("CARS_LOOP", "INDIVIDUALS_LOOP"); - - metadataModel.putGroup(individualsGroup); - metadataModel.putGroup(carsGroup); - - metadataModel.getVariables().putVariable( - new Variable("ADDRESS", rootGroup, VariableType.STRING)); - metadataModel.getVariables().putVariable( - new Variable("HOUSEHOLD_INCOME", rootGroup, VariableType.NUMBER)); - metadataModel.getVariables().putVariable( - new Variable("FIRST_NAME", individualsGroup, VariableType.STRING)); - metadataModel.getVariables().putVariable( - new Variable("LAST_NAME", individualsGroup, VariableType.STRING)); - metadataModel.getVariables().putVariable( - new Variable("GENDER", individualsGroup, VariableType.STRING)); - metadataModel.getVariables().putVariable( - new Variable("CAR_COLOR", carsGroup, VariableType.STRING)); - } - - @Test - void testRootGroup() { - assertTrue(metadataModel.hasGroup(Constants.ROOT_GROUP_NAME)); - assertEquals(Constants.ROOT_GROUP_NAME, metadataModel.getRootGroup().getName()); - assertNull(metadataModel.getRootGroup().getParentName()); - } - - @Test - void testGetVariableByName() { - - // Get - Variable rootVariable = metadataModel.getVariables().getVariable("HOUSEHOLD_INCOME"); - Variable group1Variable = metadataModel.getVariables().getVariable("FIRST_NAME"); - Variable group2Variable = metadataModel.getVariables().getVariable("CAR_COLOR"); - - // Get a variable that does not exist - log.debug("Trying to get a variable that does not exist in a test function, " + - "a second message should pop in the log."); - Variable dummyVariable = metadataModel.getVariables().getVariable("DUMMY"); - - // - assertEquals("HOUSEHOLD_INCOME", rootVariable.getName()); - assertEquals("FIRST_NAME", group1Variable.getName()); - assertEquals("CAR_COLOR", group2Variable.getName()); - assertNull(dummyVariable); - } - - @Test - void testRemoveAndHasVariable() { - - // Remove - metadataModel.getVariables().removeVariable("HOUSEHOLD_INCOME"); - metadataModel.getVariables().removeVariable("CAR_COLOR"); - - // Remove a variable that does not exist - log.debug("Trying to remove a variable that does not exist in a test function, " + - "a second message should pop in the log."); - metadataModel.getVariables().removeVariable("FOO"); - - // - assertFalse(metadataModel.getVariables().hasVariable("HOUSEHOLD_INCOME")); - assertTrue(metadataModel.getVariables().hasVariable("FIRST_NAME")); - assertFalse(metadataModel.getVariables().hasVariable("CAR_COLOR")); - } - - @Test - void getIdentifierNamesTest() { - assertEquals( - List.of(Constants.ROOT_IDENTIFIER_NAME, "INDIVIDUALS_LOOP", "CARS_LOOP"), - metadataModel.getIdentifierNames() - ); - } - - @Test - void getFullyQualifiedNameTest() { - assertEquals("HOUSEHOLD_INCOME", - metadataModel.getFullyQualifiedName("HOUSEHOLD_INCOME")); - assertEquals("INDIVIDUALS_LOOP.FIRST_NAME", - metadataModel.getFullyQualifiedName("FIRST_NAME")); - assertEquals("INDIVIDUALS_LOOP.CARS_LOOP.CAR_COLOR", - metadataModel.getFullyQualifiedName("CAR_COLOR")); - } - - @Test - void testGetGroupVariableNames() { - assertTrue(metadataModel.getVariables().getGroupVariableNames(Constants.ROOT_GROUP_NAME) - .containsAll(Set.of("ADDRESS", "HOUSEHOLD_INCOME"))); - assertTrue(metadataModel.getVariables().getGroupVariableNames("INDIVIDUALS_LOOP") - .containsAll(Set.of("FIRST_NAME", "LAST_NAME", "GENDER"))); - assertTrue(metadataModel.getVariables().getGroupVariableNames("CARS_LOOP") - .contains("CAR_COLOR")); - } - - @Test - void testMcqMethods() { - // - Group group = metadataModel.getGroup("INDIVIDUALS_LOOP"); - metadataModel.getVariables().putVariable(McqVariable.builder() - .name("RELATIONSHIP_A").group(group).questionItemName("RELATIONSHIP").text("Spouse").build()); - metadataModel.getVariables().putVariable(McqVariable.builder() - .name("RELATIONSHIP_B").group(group).questionItemName("RELATIONSHIP").text("Child").build()); - metadataModel.getVariables().putVariable(McqVariable.builder() - .name("RELATIONSHIP_C").group(group).questionItemName("RELATIONSHIP").text("Parent").build()); - metadataModel.getVariables().putVariable(McqVariable.builder() - .name("RELATIONSHIP_D").group(group).questionItemName("RELATIONSHIP").text("Other").build()); - // - assertTrue(metadataModel.getVariables().hasMcq("RELATIONSHIP")); - assertSame("RELATIONSHIP", metadataModel.getVariables().getVariable("RELATIONSHIP_A").getQuestionItemName()); - assertFalse(metadataModel.getVariables().hasMcq("ADDRESS")); - assertFalse(metadataModel.getVariables().hasMcq("FIRST_NAME")); - assertFalse(metadataModel.getVariables().hasMcq("CAR_COLOR")); - assertFalse(metadataModel.getVariables().hasMcq("UNKNOWN_QUESTION")); - // - assertSame(group, metadataModel.getVariables().getMcqGroup("RELATIONSHIP")); - assertNull(metadataModel.getVariables().getMcqGroup("ADDRESS")); - assertNull(metadataModel.getVariables().getMcqGroup("FIRST_NAME")); - assertNull(metadataModel.getVariables().getMcqGroup("CAR_COLOR")); - assertNull(metadataModel.getVariables().getMcqGroup("UNKNOWN_QUESTION")); - } - - @Test - void testGetVariablesNames() { - MetadataModel metadataMod = createCompleteFakeVariablesMap(); - // KSE et KGA à trouver, une par liste - List ucqMcqVariablesNames = metadataMod.getVariables().getUcqVariablesNames(); - List mcqVariablesNames = metadataMod.getVariables().getMcqVariablesNames(); - Set variablesNames = metadataMod.getVariables().getVariableNames(); - // Check ucq - assertTrue(ucqMcqVariablesNames.contains("VEHICLE_OWNER")); - assertFalse(ucqMcqVariablesNames.contains("CAR_OWNER")); - // Check mcq - assertFalse(mcqVariablesNames.contains("VEHICLE_OWNER")); - assertTrue(mcqVariablesNames.contains("RELATIONSHIP")); - assertFalse(mcqVariablesNames.contains("RELATIONSHIP_A")); - // Check mcq - assertFalse(variablesNames.contains("VEHICLE_OWNER")); - assertTrue(variablesNames.contains("CAR_OWNER")); - assertTrue(variablesNames.contains("MOTO_OWNER")); - - assertTrue(metadataMod.getVariables().hasMcq("RELATIONSHIP")); - assertTrue(metadataMod.getVariables().hasUcq("CAR_OWNER")); - assertTrue(metadataMod.getVariables().hasUcqMcq("CAR_OWNER")); - assertFalse(metadataMod.getVariables().hasUcqMcq("VEHICLE_OWNER")); - assertFalse(metadataMod.getVariables().hasMcq("ADDRESS")); - assertFalse(metadataMod.getVariables().hasMcq("FIRST_NAME")); - assertFalse(metadataMod.getVariables().hasMcq("CAR_COLOR")); - assertFalse(metadataMod.getVariables().hasMcq("UNKNOWN_QUESTION")); - } - - /* Variables map objects to test multimode management */ - - /** - * Return a VariablesMap object containing variables named as follows: - * - FIRST_NAME, LAST_NAME, AGE at the root - * - CAR_COLOR in a group named CARS_LOOP - */ - public static MetadataModel createCompleteFakeVariablesMap(){ - - MetadataModel metadataM = new MetadataModel(); - - // Groups - Group rootGroup = metadataM.getRootGroup(); - Group carsGroup = new Group("CARS_LOOP", Constants.ROOT_GROUP_NAME); - metadataM.putGroup(carsGroup); - - // Variables - metadataM.getVariables().putVariable(new Variable("LAST_NAME", rootGroup, VariableType.STRING, "20")); - metadataM.getVariables().putVariable(new Variable("FIRST_NAME", rootGroup, VariableType.STRING, "50")); - metadataM.getVariables().putVariable(new Variable("AGE", rootGroup, VariableType.INTEGER, "50")); - metadataM.getVariables().putVariable(new Variable("CAR_COLOR", carsGroup, VariableType.STRING, "50")); - - // unique choice question variable - UcqVariable ucq = new UcqVariable("SEXE", rootGroup, VariableType.STRING, "50"); - ucq.addModality("1", "Male"); - ucq.addModality("2", "Female"); - Variable paperUcq1 = new PaperUcq("SEXE_1", ucq, "1"); - Variable paperUcq2 = new PaperUcq("SEXE_2", ucq, "2"); - metadataM.getVariables().putVariable(ucq); - metadataM.getVariables().putVariable(paperUcq1); - metadataM.getVariables().putVariable(paperUcq2); - - // unique choice question variable related to multiple choices question - UcqVariable ucqMcq1 = new UcqVariable("CAR_OWNER", rootGroup, VariableType.STRING, "50"); - ucqMcq1.setQuestionItemName("VEHICLE_OWNER"); - ucqMcq1.addModality("1", "Yes"); - ucqMcq1.addModality("2", "No"); - UcqVariable ucqMcq2 = new UcqVariable("MOTO_OWNER", rootGroup, VariableType.STRING, "50"); - ucqMcq2.setQuestionItemName("VEHICLE_OWNER"); - ucqMcq2.addModality("1", "Yes"); - ucqMcq2.addModality("2", "No"); - metadataM.getVariables().putVariable(ucqMcq1); - metadataM.getVariables().putVariable(ucqMcq2); - - // multiple choices question variable - metadataM.getVariables().putVariable(McqVariable.builder() - .name("RELATIONSHIP_A").group(rootGroup).questionItemName("RELATIONSHIP").text("Spouse").build()); - metadataM.getVariables().putVariable(McqVariable.builder() - .name("RELATIONSHIP_B").group(rootGroup).questionItemName("RELATIONSHIP").text("Child").build()); - metadataM.getVariables().putVariable(McqVariable.builder() - .name("RELATIONSHIP_C").group(rootGroup).questionItemName("RELATIONSHIP").text("Parent").build()); - metadataM.getVariables().putVariable(McqVariable.builder() - .name("RELATIONSHIP_D").group(rootGroup).questionItemName("RELATIONSHIP").text("Other").build()); - - return metadataM; - } - - public static MetadataModel createAnotherFakeVariablesMap(){ - - MetadataModel metadataM = new MetadataModel(); - - // Groups - Group rootGroup = metadataM.getRootGroup(); - Group carsGroup = new Group("CARS_LOOP", Constants.ROOT_GROUP_NAME); - metadataM.putGroup(carsGroup); - - // Variables - metadataM.getVariables().putVariable(new Variable("LAST_NAME", rootGroup, VariableType.STRING, "50")); - metadataM.getVariables().putVariable(new Variable("FIRST_NAME", rootGroup, VariableType.STRING, "20")); - metadataM.getVariables().putVariable(new Variable("ADDRESS", rootGroup, VariableType.STRING, "50")); - metadataM.getVariables().putVariable(new Variable("CAR_COLOR", carsGroup, VariableType.STRING, "500")); - - return metadataM; - } - - /* Variables map objects to test information levels management */ - - public static MetadataModel createVariablesMap_rootOnly() { - MetadataModel metadataModel1 = new MetadataModel(); - - Group rootGroup = metadataModel1.getRootGroup(); - - metadataModel1.putGroup(rootGroup); - - metadataModel1.getVariables().putVariable( - new Variable("ADDRESS", rootGroup, VariableType.STRING)); - metadataModel1.getVariables().putVariable( - new Variable("HOUSEHOLD_INCOME", rootGroup, VariableType.NUMBER)); - - return metadataModel1; - } - - public static MetadataModel createVariablesMap_oneLevel() { - MetadataModel metadataModel1 = createVariablesMap_rootOnly(); - - Group individualsGroup = new Group("INDIVIDUALS_LOOP", Constants.ROOT_GROUP_NAME); - - metadataModel1.putGroup(individualsGroup); - - metadataModel1.getVariables().putVariable( - new Variable("FIRST_NAME", individualsGroup, VariableType.STRING)); - metadataModel1.getVariables().putVariable( - new Variable("LAST_NAME", individualsGroup, VariableType.STRING)); - metadataModel1.getVariables().putVariable( - new Variable("GENDER", individualsGroup, VariableType.STRING)); - - return metadataModel1; - } - - public static MetadataModel createVariablesMap_twoLevels() { - MetadataModel metadataModel1 = createVariablesMap_oneLevel(); - - Group carsGroup = new Group("CARS_LOOP", "INDIVIDUALS_LOOP"); - - metadataModel1.putGroup(carsGroup); - - metadataModel1.getVariables().putVariable( - new Variable("CAR_COLOR", carsGroup, VariableType.STRING)); - - return metadataModel1; - } - -} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsTest.java index 798bea0f..2ce0a71a 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsTest.java @@ -1,5 +1,10 @@ package fr.insee.kraftwerk.core.metadata; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.McqVariable; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.Constants; import org.junit.jupiter.api.Test; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/VariableTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/VariableTest.java deleted file mode 100644 index 1e2941a1..00000000 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/VariableTest.java +++ /dev/null @@ -1,162 +0,0 @@ -package fr.insee.kraftwerk.core.metadata; - -import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.TestConstants; -import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; -import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import org.junit.jupiter.api.Test; - -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.*; - -class VariableTest { - - static final String DDI_FOLDER = TestConstants.UNIT_TESTS_DIRECTORY + "/ddi"; - - static final String DDI_SIMPSONS_V1 = "ddi-simpsons-v1.xml"; - static final String DDI_SIMPSONS_V2 = "ddi-simpsons-v2.xml"; - static final String DDI_VQS_WEB = "vqs-2021-x00-xforms-ddi.xml"; - static final String DDI_VQS_PAP = "vqs-2021-x00-fo-ddi.xml"; - static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); - - @Test - void readSimpsonsV1Variables() throws MalformedURLException, KraftwerkException, URISyntaxException { - - Set expectedVariables = Set.of( - // - "FAVOURITE_CHARACTERS11", "FAVOURITE_CHARACTERS102", - // - "SUM_EXPENSES", "LAST_BROADCAST", "COMMENT", "READY", "PRODUCER", "SEASON_NUMBER", "DATEFIRST", - "AUDIENCE_SHARE", "CITY", "MAYOR", "STATE", "PET1", "PET4", "ICE_FLAVOUR1", "ICE_FLAVOUR4", - "NUCLEAR_CHARACTER1", "NUCLEAR_CHARACTER4", "BIRTH_CHARACTER1", "BIRTH_CHARACTER5", - "PERCENTAGE_EXPENSES11", "PERCENTAGE_EXPENSES101", "CLOWNING11", "CLOWNING42", "TRAVEL11", "TRAVEL46", - "SURVEY_COMMENT"); - - MetadataModel simpsonsMetadata = DDIReader - .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_SIMPSONS_V1, fileUtilsInterface); - - // - assertNotNull(simpsonsMetadata); - // - assertTrue(simpsonsMetadata.hasGroup(Constants.ROOT_GROUP_NAME)); - assertTrue(simpsonsMetadata.hasGroup("FAVOURITE_CHARACTERS")); - // - for (String variableName : expectedVariables) { - assertTrue(simpsonsMetadata.getVariables().hasVariable(variableName)); - } - // - assertEquals(Constants.ROOT_GROUP_NAME, simpsonsMetadata.getVariables().getVariable("SUM_EXPENSES").getGroup().getName()); - assertEquals(Constants.ROOT_GROUP_NAME, simpsonsMetadata.getVariables().getVariable("SURVEY_COMMENT").getGroup().getName()); - assertEquals("FAVOURITE_CHARACTERS", - simpsonsMetadata.getVariables().getVariable("FAVOURITE_CHARACTERS11").getGroup().getName()); - assertEquals("FAVOURITE_CHARACTERS", - simpsonsMetadata.getVariables().getVariable("FAVOURITE_CHARACTERS102").getGroup().getName()); - } - - @Test - void readSimpsonsV2Variables() throws MalformedURLException, KraftwerkException, URISyntaxException { - - Set expectedVariables = Set.of( - // - "SUM_EXPENSES", "YEAR", "FAVOURITE_CHAR1", "FAVOURITE_CHAR2", "FAVOURITE_CHAR3", "FAVOURITE_CHAR33CL", - // - "NAME_CHAR", "AGE_CHAR", "FAVCHAR", "MEMORY_CHAR", - // - "LAST_BROADCAST", "COMMENT", "READY", "PRODUCER", "SEASON_NUMBER", "DATEFIRST", "DATEYYYYMM", - "DATEYYYY", "DURATIONH", "DURATIONHH", "AUDIENCE_SHARE", "CITY", "MAYOR", "MAYOROTCL", "STATE", "PET1", - "PET4", "PETOCL", "ICE_FLAVOUR1", "ICE_FLAVOUR4", "ICE_FLAVOUROTCL", "NUCLEAR_CHARACTER1", - "NUCLEAR_CHARACTER4", "BIRTH_CHARACTER1", "BIRTH_CHARACTER5", "PERCENTAGE_EXPENSES11", - "PERCENTAGE_EXPENSES71", "LAST_FOOD_SHOPPING11", "LAST_FOOD_SHOPPING81", "LAST_FOOD_SHOPPING113CL", - "LAST_FOOD_SHOPPING813CL", "CLOWNING11", "CLOWNING42", "TRAVEL11", "TRAVEL46", "FEELCHAREV1", - "FEELCHAREV4", "LEAVDURATION11", "LEAVDURATION52", "NB_CHAR", "SURVEY_COMMENT"); - - MetadataModel simpsonsMetadata = DDIReader - .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_SIMPSONS_V2.toString(), fileUtilsInterface); - - // - assertNotNull(simpsonsMetadata); - // - assertTrue(simpsonsMetadata.hasGroup(Constants.ROOT_GROUP_NAME)); - assertTrue(simpsonsMetadata.hasGroup("FAVOURITE_CHAR")); - assertTrue(simpsonsMetadata.hasGroup("Loop1")); - // - for (String variableName : expectedVariables) { - assertTrue(simpsonsMetadata.getVariables().hasVariable(variableName)); - } - // - assertEquals(Constants.ROOT_GROUP_NAME, simpsonsMetadata.getVariables().getVariable("LAST_BROADCAST").getGroup().getName()); - assertEquals(Constants.ROOT_GROUP_NAME, simpsonsMetadata.getVariables().getVariable("SURVEY_COMMENT").getGroup().getName()); - assertEquals("FAVOURITE_CHAR", simpsonsMetadata.getVariables().getVariable("SUM_EXPENSES").getGroup().getName()); - assertEquals("FAVOURITE_CHAR", simpsonsMetadata.getVariables().getVariable("FAVOURITE_CHAR1").getGroup().getName()); - assertEquals("Loop1", simpsonsMetadata.getVariables().getVariable("NAME_CHAR").getGroup().getName()); - } - - @Test - void readVqsWebVariables() throws MalformedURLException, KraftwerkException, URISyntaxException { - - Set expectedVariables = Set.of("prenom", "NOM", "SEXE", "DTNAIS", "ETAT_SANT", "APPRENT", "AIDREG_A", - "AIDREG_B", "AIDREG_C", "AIDREG_D", "RELATION1", "RELATION2", "RELATION3", "RELATION4", "ADRESSE", - "RESIDM", "NHAB"); - - MetadataModel vqsMetadata = DDIReader - .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_VQS_WEB, fileUtilsInterface); - - // - assertNotNull(vqsMetadata); - // - assertTrue(vqsMetadata.hasGroup(Constants.ROOT_GROUP_NAME)); - assertTrue(vqsMetadata.hasGroup("BOUCLEINDIV")); - // - for (String variableName : expectedVariables) { - assertTrue(vqsMetadata.getVariables().hasVariable(variableName)); - } - // - assertEquals(Constants.ROOT_GROUP_NAME, vqsMetadata.getVariables().getVariable("ADRESSE").getGroup().getName()); - assertEquals("BOUCLEINDIV", vqsMetadata.getVariables().getVariable("prenom").getGroup().getName()); - // UCQ - assertInstanceOf(UcqVariable.class, vqsMetadata.getVariables().getVariable("ETAT_SANT")); - UcqVariable etatSant = (UcqVariable) vqsMetadata.getVariables().getVariable("ETAT_SANT"); - assertEquals(5, etatSant.getModalities().size()); - // MCQ - assertInstanceOf(McqVariable.class, vqsMetadata.getVariables().getVariable("AIDREG_A")); - assertInstanceOf(McqVariable.class, vqsMetadata.getVariables().getVariable("AIDREG_D")); - assertInstanceOf(McqVariable.class, vqsMetadata.getVariables().getVariable("RELATION1")); - assertInstanceOf(McqVariable.class, vqsMetadata.getVariables().getVariable("RELATION4")); - McqVariable aidregA = (McqVariable) vqsMetadata.getVariables().getVariable("AIDREG_A"); - assertEquals("AIDREG", aidregA.getQuestionItemName()); - assertEquals("1 - Oui, une aide aux activités de la vie quotidienne", aidregA.getText()); - // - assertFalse(vqsMetadata.getVariables().getVariable("ADRESSE") instanceof McqVariable); - assertFalse(vqsMetadata.getVariables().getVariable("ADRESSE") instanceof UcqVariable); - assertFalse(vqsMetadata.getVariables().getVariable("PRENOM") instanceof McqVariable); - assertFalse(vqsMetadata.getVariables().getVariable("PRENOM") instanceof UcqVariable); - } - - @Test - void readVqsPapVariables() throws MalformedURLException, KraftwerkException, URISyntaxException { - - Set expectedVariables = Set.of("PRENOM", "NOM", "SEXE", "DTNAIS", "ETAT_SANT", "APPRENT", "AIDREG_A", - "AIDREG_B", "AIDREG_C", "AIDREG_D", "RESID", "RESIDANCIEN", "NBQUEST"); - - MetadataModel vqsMetadata = DDIReader - .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_VQS_PAP, fileUtilsInterface); - - // - assertNotNull(vqsMetadata); - // - assertTrue(vqsMetadata.hasGroup(Constants.ROOT_GROUP_NAME)); - assertTrue(vqsMetadata.hasGroup("BOUCLEINDIV")); - // - for (String variableName : expectedVariables) { - assertTrue(vqsMetadata.getVariables().hasVariable(variableName)); - } - // - assertEquals(Constants.ROOT_GROUP_NAME, vqsMetadata.getVariables().getVariable("NBQUEST").getGroup().getName()); - assertEquals("BOUCLEINDIV", vqsMetadata.getVariables().getVariable("PRENOM").getGroup().getName()); - } - -} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/CsvOutputFilesTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/CsvOutputFilesTest.java index 6b5b4b93..ebeb84f5 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/CsvOutputFilesTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/CsvOutputFilesTest.java @@ -4,10 +4,10 @@ import fr.insee.kraftwerk.core.TestConstants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import fr.insee.kraftwerk.core.metadata.Group; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.outputs.csv.CsvOutputFiles; import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/ParquetOutputFilesTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/ParquetOutputFilesTest.java index 652dae42..6ffcb4c2 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/ParquetOutputFilesTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/ParquetOutputFilesTest.java @@ -4,10 +4,10 @@ import fr.insee.kraftwerk.core.TestConstants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import fr.insee.kraftwerk.core.metadata.Group; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.outputs.parquet.ParquetOutputFiles; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java index b3e48d64..fdac5a45 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java @@ -1,11 +1,15 @@ package fr.insee.kraftwerk.core.outputs.scripts; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.McqVariable; +import fr.insee.bpm.metadata.model.PaperUcq; +import fr.insee.bpm.metadata.model.UcqVariable; +import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.GroupProcessing; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.MetadataModelTest; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.outputs.ImportScript; import fr.insee.kraftwerk.core.outputs.TableScriptInfo; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; @@ -29,7 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; -class ImportScriptTest { +public class ImportScriptTest { VtlBindings vtlBindings = new VtlBindings(); @@ -48,14 +52,14 @@ public void initMetadata() { } private void instantiateMap() { - metadata.put("CAWI", MetadataModelTest.createCompleteFakeVariablesMap()); + metadata.put("CAWI", createCompleteFakeVariablesMap()); SurveyRawData srdWeb = SurveyRawDataTest.createFakeCawiSurveyRawData(); - srdWeb.setMetadataModel(MetadataModelTest.createCompleteFakeVariablesMap()); + srdWeb.setMetadataModel(createCompleteFakeVariablesMap()); vtlExecute.convertToVtlDataset(srdWeb, "CAWI", vtlBindings); - metadata.put("PAPI", MetadataModelTest.createAnotherFakeVariablesMap()); + metadata.put("PAPI", createAnotherFakeVariablesMap()); SurveyRawData srdPaper = SurveyRawDataTest.createFakePapiSurveyRawData(); - srdPaper.setMetadataModel(MetadataModelTest.createAnotherFakeVariablesMap()); + srdPaper.setMetadataModel(createAnotherFakeVariablesMap()); vtlExecute.convertToVtlDataset(srdPaper, "PAPI", vtlBindings); // add group prefixes @@ -112,4 +116,127 @@ void numberTypeInDatasets() { // => "NUMBER" type in Trevas datasets is java "Double" type } + /* Variables map objects to test multimode management */ + + /** + * Return a VariablesMap object containing variables named as follows: + * - FIRST_NAME, LAST_NAME, AGE at the root + * - CAR_COLOR in a group named CARS_LOOP + */ + public static MetadataModel createCompleteFakeVariablesMap(){ + + MetadataModel metadataM = new MetadataModel(); + + // Groups + Group rootGroup = metadataM.getRootGroup(); + Group carsGroup = new Group("CARS_LOOP", Constants.ROOT_GROUP_NAME); + metadataM.putGroup(carsGroup); + + // Variables + metadataM.getVariables().putVariable(new Variable("LAST_NAME", rootGroup, VariableType.STRING, "20")); + metadataM.getVariables().putVariable(new Variable("FIRST_NAME", rootGroup, VariableType.STRING, "50")); + metadataM.getVariables().putVariable(new Variable("AGE", rootGroup, VariableType.INTEGER, "50")); + metadataM.getVariables().putVariable(new Variable("CAR_COLOR", carsGroup, VariableType.STRING, "50")); + + // unique choice question variable + UcqVariable ucq = new UcqVariable("SEXE", rootGroup, VariableType.STRING, "50"); + ucq.addModality("1", "Male"); + ucq.addModality("2", "Female"); + Variable paperUcq1 = new PaperUcq("SEXE_1", ucq, "1"); + Variable paperUcq2 = new PaperUcq("SEXE_2", ucq, "2"); + metadataM.getVariables().putVariable(ucq); + metadataM.getVariables().putVariable(paperUcq1); + metadataM.getVariables().putVariable(paperUcq2); + + // unique choice question variable related to multiple choices question + UcqVariable ucqMcq1 = new UcqVariable("CAR_OWNER", rootGroup, VariableType.STRING, "50"); + ucqMcq1.setQuestionItemName("VEHICLE_OWNER"); + ucqMcq1.addModality("1", "Yes"); + ucqMcq1.addModality("2", "No"); + UcqVariable ucqMcq2 = new UcqVariable("MOTO_OWNER", rootGroup, VariableType.STRING, "50"); + ucqMcq2.setQuestionItemName("VEHICLE_OWNER"); + ucqMcq2.addModality("1", "Yes"); + ucqMcq2.addModality("2", "No"); + metadataM.getVariables().putVariable(ucqMcq1); + metadataM.getVariables().putVariable(ucqMcq2); + + // multiple choices question variable + metadataM.getVariables().putVariable(McqVariable.builder() + .name("RELATIONSHIP_A").group(rootGroup).questionItemName("RELATIONSHIP").text("Spouse").build()); + metadataM.getVariables().putVariable(McqVariable.builder() + .name("RELATIONSHIP_B").group(rootGroup).questionItemName("RELATIONSHIP").text("Child").build()); + metadataM.getVariables().putVariable(McqVariable.builder() + .name("RELATIONSHIP_C").group(rootGroup).questionItemName("RELATIONSHIP").text("Parent").build()); + metadataM.getVariables().putVariable(McqVariable.builder() + .name("RELATIONSHIP_D").group(rootGroup).questionItemName("RELATIONSHIP").text("Other").build()); + + return metadataM; + } + + public static MetadataModel createAnotherFakeVariablesMap(){ + + MetadataModel metadataM = new MetadataModel(); + + // Groups + Group rootGroup = metadataM.getRootGroup(); + Group carsGroup = new Group("CARS_LOOP", Constants.ROOT_GROUP_NAME); + metadataM.putGroup(carsGroup); + + // Variables + metadataM.getVariables().putVariable(new Variable("LAST_NAME", rootGroup, VariableType.STRING, "50")); + metadataM.getVariables().putVariable(new Variable("FIRST_NAME", rootGroup, VariableType.STRING, "20")); + metadataM.getVariables().putVariable(new Variable("ADDRESS", rootGroup, VariableType.STRING, "50")); + metadataM.getVariables().putVariable(new Variable("CAR_COLOR", carsGroup, VariableType.STRING, "500")); + + return metadataM; + } + + /* Variables map objects to test information levels management */ + + public static MetadataModel createVariablesMap_rootOnly() { + MetadataModel metadataModel1 = new MetadataModel(); + + Group rootGroup = metadataModel1.getRootGroup(); + + metadataModel1.putGroup(rootGroup); + + metadataModel1.getVariables().putVariable( + new Variable("ADDRESS", rootGroup, VariableType.STRING)); + metadataModel1.getVariables().putVariable( + new Variable("HOUSEHOLD_INCOME", rootGroup, VariableType.NUMBER)); + + return metadataModel1; + } + + public static MetadataModel createVariablesMap_oneLevel() { + MetadataModel metadataModel1 = createVariablesMap_rootOnly(); + + Group individualsGroup = new Group("INDIVIDUALS_LOOP", Constants.ROOT_GROUP_NAME); + + metadataModel1.putGroup(individualsGroup); + + metadataModel1.getVariables().putVariable( + new Variable("FIRST_NAME", individualsGroup, VariableType.STRING)); + metadataModel1.getVariables().putVariable( + new Variable("LAST_NAME", individualsGroup, VariableType.STRING)); + metadataModel1.getVariables().putVariable( + new Variable("GENDER", individualsGroup, VariableType.STRING)); + + return metadataModel1; + } + + public static MetadataModel createVariablesMap_twoLevels() { + MetadataModel metadataModel1 = createVariablesMap_oneLevel(); + + Group carsGroup = new Group("CARS_LOOP", "INDIVIDUALS_LOOP"); + + metadataModel1.putGroup(carsGroup); + + metadataModel1.getVariables().putVariable( + new Variable("CAR_COLOR", carsGroup, VariableType.STRING)); + + return metadataModel1; + } + + } diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/SASImportScriptTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/SASImportScriptTest.java index 4fc69595..88523615 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/SASImportScriptTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/SASImportScriptTest.java @@ -1,7 +1,9 @@ package fr.insee.kraftwerk.core.outputs.scripts; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.KraftwerkError; -import fr.insee.kraftwerk.core.metadata.*; import fr.insee.kraftwerk.core.outputs.TableScriptInfo; import fr.insee.kraftwerk.core.outputs.csv.SASImportScript; import fr.insee.vtl.model.Dataset; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParserTest.java index ae00e02b..1e1ca433 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParserTest.java @@ -2,10 +2,10 @@ import fr.insee.kraftwerk.core.TestConstants; import fr.insee.kraftwerk.core.exceptions.NullException; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.MetadataModelTest; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.kraftwerk.core.outputs.scripts.ImportScriptTest; import fr.insee.kraftwerk.core.rawdata.GroupData; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; @@ -64,7 +64,7 @@ void parseLunaticDataFolder() throws NullException { void parseLunaticXml_rootOnly() throws NullException { // SurveyRawData data = new SurveyRawData("TEST"); - data.setMetadataModel(MetadataModelTest.createVariablesMap_rootOnly()); + data.setMetadataModel(ImportScriptTest.createVariablesMap_rootOnly()); Path dataPath = Paths.get(dataSamplesFolder + "/lunatic_xml/fake-lunatic-data-root-only.xml"); LunaticXmlDataParser parser = new LunaticXmlDataParser(data, fileUtilsInterface); parser.parseSurveyData(dataPath,null); @@ -97,7 +97,7 @@ public void checkRootContent(SurveyRawData data) { void parseLunaticXml_oneLevel() throws NullException { // SurveyRawData data = new SurveyRawData("TEST"); - data.setMetadataModel(MetadataModelTest.createVariablesMap_oneLevel()); + data.setMetadataModel(ImportScriptTest.createVariablesMap_oneLevel()); Path dataPath = Paths.get(dataSamplesFolder + "/lunatic_xml/fake-lunatic-data-1.xml"); LunaticXmlDataParser parser = new LunaticXmlDataParser(data, fileUtilsInterface); parser.parseSurveyData(dataPath,null); @@ -131,7 +131,7 @@ public void checkLevelOneContent(SurveyRawData data) { void parseLunaticXml_noCollected() throws NullException { //Given SurveyRawData data = new SurveyRawData("TEST"); - data.setMetadataModel(MetadataModelTest.createVariablesMap_oneLevel()); + data.setMetadataModel(ImportScriptTest.createVariablesMap_oneLevel()); Path dataPath = Paths.get(dataSamplesFolder + "/lunatic_xml/fake-lunatic-data-3.xml"); LunaticXmlDataParser parser = new LunaticXmlDataParser(data, fileUtilsInterface); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/PaperDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/PaperDataParserTest.java index 5cfd4adc..6b9913a0 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/PaperDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/PaperDataParserTest.java @@ -2,9 +2,9 @@ import fr.insee.kraftwerk.core.TestConstants; import fr.insee.kraftwerk.core.exceptions.NullException; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/rawdata/SurveyRawDataTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/rawdata/SurveyRawDataTest.java index 3ab4e62c..617fa595 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/rawdata/SurveyRawDataTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/rawdata/SurveyRawDataTest.java @@ -1,7 +1,7 @@ package fr.insee.kraftwerk.core.rawdata; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.MetadataModelTest; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.kraftwerk.core.outputs.scripts.ImportScriptTest; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; @@ -49,14 +49,14 @@ void surveyRawDataTest(){ /** * Create a SurveyRawData object with some fake data in it. * Use the variables from the method createFakeVariablesMap. - * @see fr.insee.kraftwerk.core.metadata.MetadataModelTest + * @see fr.insee.kraftwerk.core.outputs.scripts.ImportScriptTest * */ public static SurveyRawData createFakeCawiSurveyRawData() { // Instantiate the data object SurveyRawData data = new SurveyRawData("CAWI"); // Give it a data structure - MetadataModel metMod = MetadataModelTest.createCompleteFakeVariablesMap(); + MetadataModel metMod = ImportScriptTest.createCompleteFakeVariablesMap(); data.setMetadataModel(metMod); // Homer Simpsons, aged 40, has a purple and a red car. @@ -91,7 +91,7 @@ public static SurveyRawData createFakeCapiSurveyRawData() { SurveyRawData data = new SurveyRawData("CAPI"); // Give it a data structure - MetadataModel metadataModel = MetadataModelTest.createCompleteFakeVariablesMap(); + MetadataModel metadataModel = ImportScriptTest.createCompleteFakeVariablesMap(); data.setMetadataModel(metadataModel); // Homer Simpsons, aged 40, has a purple and a red car. @@ -126,7 +126,7 @@ public static SurveyRawData createFakePapiSurveyRawData() { SurveyRawData data = new SurveyRawData("PAPI"); // Give it a data structure - MetadataModel metMod = MetadataModelTest.createAnotherFakeVariablesMap(); + MetadataModel metMod = ImportScriptTest.createAnotherFakeVariablesMap(); data.setMetadataModel(metMod); // @@ -183,7 +183,7 @@ public static SurveyRawData createFakeData_rootOnly() { SurveyRawData data = new SurveyRawData(); - data.setMetadataModel(MetadataModelTest.createVariablesMap_rootOnly()); + data.setMetadataModel(ImportScriptTest.createVariablesMap_rootOnly()); QuestionnaireData q1 = new QuestionnaireData(); q1.setIdentifier("S0000001"); @@ -204,7 +204,7 @@ public static SurveyRawData createFakeData_oneLevel() { SurveyRawData data = createFakeData_rootOnly(); - data.setMetadataModel(MetadataModelTest.createVariablesMap_oneLevel()); + data.setMetadataModel(ImportScriptTest.createVariablesMap_oneLevel()); QuestionnaireData q1 = data.getQuestionnaires().getFirst(); q1.putValue("Homer", "FIRST_NAME", Pair.of("INDIVIDUALS_LOOP", 0)); @@ -227,7 +227,7 @@ public static SurveyRawData createFakeData_twoLevels() { SurveyRawData data = createFakeData_oneLevel(); - data.setMetadataModel(MetadataModelTest.createVariablesMap_twoLevels()); + data.setMetadataModel(ImportScriptTest.createVariablesMap_twoLevels()); QuestionnaireData q1 = data.getQuestionnaires().getFirst(); q1.putValue("Purple", "CAR_COLOR", Pair.of("INDIVIDUALS_LOOP", 0), Pair.of("CARS_LOOP", 0)); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBIndingsSequenceGenesisTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBIndingsSequenceGenesisTest.java index 5472f7c0..e87fbd20 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBIndingsSequenceGenesisTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBIndingsSequenceGenesisTest.java @@ -6,10 +6,10 @@ import fr.insee.kraftwerk.core.data.model.Mode; import fr.insee.kraftwerk.core.data.model.SurveyUnitUpdateLatest; import fr.insee.kraftwerk.core.data.model.VariableState; -import fr.insee.kraftwerk.core.metadata.Group; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.vtl.VtlBindings; import org.junit.jupiter.api.BeforeAll; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceTest.java index d17b2a03..03bdbfbf 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceTest.java @@ -3,10 +3,10 @@ import fr.insee.kraftwerk.core.TestConstants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.metadata.UcqVariable; -import fr.insee.kraftwerk.core.metadata.Variable; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.UcqVariable; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/SqlUtilsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/SqlUtilsTest.java index b5fdeec4..2103dbed 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/SqlUtilsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/SqlUtilsTest.java @@ -4,7 +4,7 @@ import fr.insee.kraftwerk.core.TestConstants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformerTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformerTest.java deleted file mode 100644 index 07b0d933..00000000 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformerTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package fr.insee.kraftwerk.core.utils.xsl; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.nio.file.Path; - -import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; -import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import org.junit.jupiter.api.Test; -import org.xmlunit.assertj3.XmlAssert; - -import fr.insee.kraftwerk.core.TestConstants; -import fr.insee.kraftwerk.core.utils.TextFileReader; - -class SaxonTransformerTest { - - FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); - - @Test - void applyXsltScript() throws IOException { - String xsltTestScript = TestConstants.UNIT_TESTS_DIRECTORY + "/utils/xsl/do-nothing.xsl"; - String inXmlFile = TestConstants.UNIT_TESTS_DIRECTORY + "/utils/xsl/note.xml"; - String outXmlFile = TestConstants.UNIT_TESTS_DUMP + "/xsl-output.xml"; - // - SaxonTransformer saxonTransformer = new SaxonTransformer(fileUtilsInterface); - saxonTransformer.xslTransform(Path.of(inXmlFile), xsltTestScript,Path.of(outXmlFile)); - // - String inContent = TextFileReader.readFromPath(Path.of(inXmlFile), fileUtilsInterface); - String outContent = TextFileReader.readFromPath(Path.of(outXmlFile), fileUtilsInterface); - // - XmlAssert.assertThat(inContent).and(outContent).areSimilar(); - } -} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java index 93cf2fb8..2a82a049 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java @@ -2,7 +2,7 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriterTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriterTest.java index 12fee50a..3784405e 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriterTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriterTest.java @@ -1,7 +1,11 @@ package fr.insee.kraftwerk.core.vtl; +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.metadata.*; +import fr.insee.kraftwerk.core.outputs.scripts.ImportScriptTest; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; @@ -33,7 +37,7 @@ public void initVtlBindings() { @Test void testConvertToVtlDataset_rootOnly() { // - MetadataModel variablesMap = MetadataModelTest.createVariablesMap_rootOnly(); + MetadataModel variablesMap = ImportScriptTest.createVariablesMap_rootOnly(); // SurveyRawData testData = SurveyRawDataTest.createFakeData_rootOnly(); // @@ -70,7 +74,7 @@ void testConvertToVtlDataset_rootOnly() { @Test void testConvertToVtlDataset_oneLevel() { // - MetadataModel variablesMap = MetadataModelTest.createVariablesMap_oneLevel(); + MetadataModel variablesMap = ImportScriptTest.createVariablesMap_oneLevel(); // SurveyRawData testData = SurveyRawDataTest.createFakeData_oneLevel(); // @@ -111,7 +115,7 @@ void convertToVtlDataset_withSplitQuestionnaires() { // SurveyRawData paperLikeData = new SurveyRawData(); // - MetadataModel variables = MetadataModelTest.createVariablesMap_oneLevel(); + MetadataModel variables = ImportScriptTest.createVariablesMap_oneLevel(); paperLikeData.setMetadataModel(variables); // First household but split in several questionnaires diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java index 5ca525fc..421f5c1a 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java @@ -1,15 +1,16 @@ package cucumber.functional_tests; import cucumber.TestConstants; +import fr.insee.bpm.exceptions.MetadataParserException; import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.CalculatedProcessing; import fr.insee.kraftwerk.core.dataprocessing.DataProcessing; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.metadata.CalculatedVariables; -import fr.insee.kraftwerk.core.metadata.DDIReader; -import fr.insee.kraftwerk.core.metadata.LunaticReader; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.metadata.model.CalculatedVariables; +import fr.insee.bpm.metadata.reader.ddi.DDIReader; +import fr.insee.bpm.metadata.reader.lunatic.LunaticReader; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.parsers.DataParser; import fr.insee.kraftwerk.core.parsers.LunaticXmlDataParser; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; @@ -60,12 +61,12 @@ public void setUpCampaignPacks() { //TODO Put other sample test campaigns once they are ready } @Given("I read data from campaign {string}, mode {string}") - public void getCampaignFiles(String campaignName, String dataMode) throws MalformedURLException, KraftwerkException, URISyntaxException { + public void getCampaignFiles(String campaignName, String dataMode) throws MalformedURLException, KraftwerkException, URISyntaxException, MetadataParserException { this.campaignName = campaignName; this.dataMode = dataMode; // metadataModel = DDIReader.getMetadataFromDDI( - Constants.convertToUrl(campaignPacks.get(campaignName).get(dataMode).get("ddi")).toString(), new FileSystemImpl()); + Constants.convertToUrl(campaignPacks.get(campaignName).get(dataMode).get("ddi")).toString(), new FileSystemImpl().readFile(campaignPacks.get(campaignName).get(dataMode).get("ddi"))); // SurveyRawData data = new SurveyRawData(); data.setMetadataModel(metadataModel); @@ -80,7 +81,7 @@ public void getCampaignFiles(String campaignName, String dataMode) throws Malfor public void readCampaignData() { // CalculatedVariables calculatedVariables = LunaticReader.getCalculatedFromLunatic( - Path.of(campaignPacks.get(campaignName).get(dataMode).get("lunatic")), new FileSystemImpl()); + new FileSystemImpl().readFile(Path.of(campaignPacks.get(campaignName).get(dataMode).get("lunatic")).toString())); DataProcessing calculatedProcessing = new CalculatedProcessing(vtlBindings,calculatedVariables, new FileSystemImpl()); calculatedProcessing.applyVtlTransformations("TEST", null,errors); // diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/DDIGetterDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/DDIGetterDefinitions.java index 8646bc65..45b057f1 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/DDIGetterDefinitions.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/DDIGetterDefinitions.java @@ -3,7 +3,7 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.utils.TextFileReader; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; -import fr.insee.kraftwerk.core.utils.xsl.SaxonTransformer; +import fr.insee.bpm.utils.xsl.SaxonTransformer; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -43,8 +43,8 @@ public void compare_ddi(String nameDDI) throws IOException, URISyntaxException { tempFile.deleteOnExit(); Path tempPath = Paths.get(tempFile.getAbsolutePath()); URL url = new URI(linkDDI).toURL(); - SaxonTransformer transformer = new SaxonTransformer(new FileSystemImpl()); - transformer.xslTransform(url.toString(), Constants.XSLT_STRUCTURED_VARIABLES, tempPath); + SaxonTransformer transformer = new SaxonTransformer(); + transformer.xslTransform(url.toString(), new FileSystemImpl().readFile(url.toString()), Constants.XSLT_STRUCTURED_VARIABLES, tempPath); actualString = TextFileReader.readFromPath(tempPath, new FileSystemImpl()); diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java index 8ec72787..a94f1200 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java @@ -6,15 +6,15 @@ import com.opencsv.CSVReaderBuilder; import com.opencsv.exceptions.CsvValidationException; +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.api.process.MainProcessing; import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.exceptions.NullException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.MetadataUtils; -import fr.insee.kraftwerk.core.metadata.VariableType; import fr.insee.kraftwerk.core.outputs.OutputFiles; import fr.insee.kraftwerk.core.outputs.csv.CsvOutputFiles; import fr.insee.kraftwerk.core.sequence.*; diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/ParadataDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/ParadataDefinitions.java index 664ec562..e46c6d8f 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/ParadataDefinitions.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/ParadataDefinitions.java @@ -1,5 +1,6 @@ package cucumber.functional_tests; +import fr.insee.bpm.exceptions.MetadataParserException; import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.exceptions.NullException; @@ -7,7 +8,7 @@ import fr.insee.kraftwerk.core.extradata.paradata.ParadataParser; import fr.insee.kraftwerk.core.inputs.ModeInputs; import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import fr.insee.kraftwerk.core.metadata.DDIReader; +import fr.insee.bpm.metadata.reader.ddi.DDIReader; import fr.insee.kraftwerk.core.parsers.DataParser; import fr.insee.kraftwerk.core.parsers.DataParserManager; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; @@ -38,7 +39,7 @@ public class ParadataDefinitions { private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); @Given("We read data from input named {string}") - public void launch_all_steps(String campaignName) throws KraftwerkException { + public void launch_all_steps(String campaignName) throws KraftwerkException, MetadataParserException { Path campaignDirectory = Paths.get(FUNCTIONAL_TESTS_INPUT_DIRECTORY).resolve(campaignName); controlInputSequence = new ControlInputSequence(campaignDirectory.toString(), new FileSystemImpl()); @@ -49,7 +50,7 @@ public void launch_all_steps(String campaignName) throws KraftwerkException { // parse data data = new SurveyRawData(); data.setDataMode(modeInputs.getDataMode()); - data.setMetadataModel(DDIReader.getMetadataFromDDI(modeInputs.getDdiUrl(), fileUtilsInterface)); + data.setMetadataModel(DDIReader.getMetadataFromDDI(modeInputs.getDdiUrl(), fileUtilsInterface.readFile(modeInputs.getDdiUrl()))); DataParser parser = DataParserManager.getParser(modeInputs.getDataFormat(), data, fileUtilsInterface); parser.parseSurveyData(modeInputs.getDataFile(),null); // get paradata folder diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/VariablesGetterDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/VariablesGetterDefinitions.java index 9680470e..e933597b 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/VariablesGetterDefinitions.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/VariablesGetterDefinitions.java @@ -1,8 +1,8 @@ package cucumber.functional_tests; -import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.metadata.DDIReader; -import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.bpm.exceptions.MetadataParserException; +import fr.insee.bpm.metadata.reader.ddi.DDIReader; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; @@ -31,8 +31,8 @@ public void set_linkDDI(String linkDDI) throws MalformedURLException, URISyntaxE } @When("I try to collect the variables's infos") - public void collect_variables() throws KraftwerkException { - metadataModel = DDIReader.getMetadataFromDDI(linkDDI.toString(), new FileSystemImpl()); + public void collect_variables() throws MetadataParserException { + metadataModel = DDIReader.getMetadataFromDDI(linkDDI.toString(), new FileSystemImpl().readFile(linkDDI.toString())); } @Then("The variables I try to count should answer {int} and have {string} in it")