diff --git a/kraftwerk-api/.gitignore b/kraftwerk-api/.gitignore index b5d79ef0..dd88f8b4 100644 --- a/kraftwerk-api/.gitignore +++ b/kraftwerk-api/.gitignore @@ -1,2 +1,2 @@ # Config -#src/main/resources/kraftwerk.properties TODO \ No newline at end of file +src/main/resources/kraftwerk.properties \ No newline at end of file diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java index ca8116b8..75de94e4 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java @@ -1,6 +1,5 @@ package fr.insee.kraftwerk.api.process; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.ModeInputs; import fr.insee.kraftwerk.core.inputs.UserInputsFile; @@ -14,7 +13,7 @@ import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.bpm.metadata.model.MetadataModel; @@ -50,8 +49,7 @@ public class MainProcessing { List userInputsFileList; // for file by file process @Getter private VtlBindings vtlBindings = new VtlBindings(); - private KraftwerkExecutionLog kraftwerkExecutionLog; - private final List errors = new ArrayList<>(); + private KraftwerkExecutionContext kraftwerkExecutionContext; private LocalDateTime executionDateTime; private final FileUtilsInterface fileUtilsInterface; @@ -105,7 +103,7 @@ public void runMain() throws KraftwerkException { outputFileWriter(writeDatabase); } writeErrors(); - kraftwerkExecutionLog.setEndTimeStamp(System.currentTimeMillis()); + kraftwerkExecutionContext.setEndTimeStamp(System.currentTimeMillis()); writeLog(); } catch (SQLException e) { log.error(e.toString()); @@ -115,7 +113,7 @@ public void runMain() throws KraftwerkException { /* Step 1 : Init */ public void init() throws KraftwerkException { - kraftwerkExecutionLog = new KraftwerkExecutionLog(); //Init logger + kraftwerkExecutionContext = new KraftwerkExecutionContext(); //Init logger this.executionDateTime = LocalDateTime.now(); inDirectory = controlInputSequence.getInDirectory(inDirectoryParam); @@ -149,16 +147,18 @@ private void unimodalProcess() throws KraftwerkException { BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(withAllReportingData, fileUtilsInterface); for (String dataMode : userInputsFile.getModeInputsMap().keySet()) { MetadataModel metadataForMode = metadataModels.get(dataMode); - buildBindingsSequence.buildVtlBindings(userInputsFile, dataMode, vtlBindings, metadataForMode, withDDI, kraftwerkExecutionLog); + buildBindingsSequence.buildVtlBindings(userInputsFile, dataMode, vtlBindings, metadataForMode, withDDI, kraftwerkExecutionContext); UnimodalSequence unimodal = new UnimodalSequence(); - unimodal.applyUnimodalSequence(userInputsFile, dataMode, vtlBindings, errors, metadataModels, fileUtilsInterface); + unimodal.applyUnimodalSequence(userInputsFile, dataMode, vtlBindings, kraftwerkExecutionContext, + metadataModels, + fileUtilsInterface); } } /* Step 3 : multimodal VTL data processing */ private void multimodalProcess(){ MultimodalSequence multimodalSequence = new MultimodalSequence(); - multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModels, fileUtilsInterface); + multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, kraftwerkExecutionContext, metadataModels, fileUtilsInterface); } /* Step 4 : Insert into SQL database */ @@ -170,17 +170,17 @@ private void insertDatabase(Statement database) { /* Step 5 : Write output files */ private void outputFileWriter(Statement database) throws KraftwerkException { WriterSequence writerSequence = new WriterSequence(); - writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModels, errors, kraftwerkExecutionLog, database, fileUtilsInterface); + writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModels, kraftwerkExecutionContext, database, fileUtilsInterface); } /* Step 5 : Write errors */ private void writeErrors() { - TextFileWriter.writeErrorsFile(inDirectory, executionDateTime, errors, fileUtilsInterface); + TextFileWriter.writeErrorsFile(inDirectory, executionDateTime, kraftwerkExecutionContext, fileUtilsInterface); } /* Step 6 : Write log */ - private void writeLog() {TextFileWriter.writeLogFile(inDirectory, executionDateTime, kraftwerkExecutionLog, fileUtilsInterface);} + private void writeLog() {TextFileWriter.writeLogFile(inDirectory, executionDateTime, kraftwerkExecutionContext, fileUtilsInterface);} private static List getUserInputsFile(UserInputsFile source, boolean fileByFile) throws KraftwerkException { List userInputsFileList = new ArrayList<>(); diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java index cfdea9c2..4e81a5ad 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java @@ -20,6 +20,7 @@ import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.utils.TextFileWriter; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.Getter; import lombok.Setter; @@ -33,7 +34,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.time.LocalDateTime; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -44,13 +44,14 @@ public class MainProcessingGenesis { private ControlInputSequenceGenesis controlInputSequenceGenesis; @Getter private VtlBindings vtlBindings = new VtlBindings(); - private final List errors = new ArrayList<>(); @Getter private UserInputsGenesis userInputs; private LocalDateTime executionDateTime; private final FileUtilsInterface fileUtilsInterface; private Statement database; + private KraftwerkExecutionContext kraftwerkExecutionContext; + /* SPECIFIC VARIABLES */ @Getter private Path inDirectory; @@ -122,14 +123,15 @@ private void unimodalProcess(List suLatest) throws Kraft for (String dataMode : userInputs.getModeInputsMap().keySet()) { buildBindingsSequenceGenesis.buildVtlBindings(dataMode, vtlBindings, metadataModels, suLatest, inDirectory); UnimodalSequence unimodal = new UnimodalSequence(); - unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, errors, metadataModels, fileUtilsInterface); + unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, kraftwerkExecutionContext, metadataModels, fileUtilsInterface); } } /* Step 3 : multimodal VTL data processing */ private void multimodalProcess() { MultimodalSequence multimodalSequence = new MultimodalSequence(); - multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModels, fileUtilsInterface); + multimodalSequence.multimodalProcessing(userInputs, vtlBindings, kraftwerkExecutionContext, metadataModels, + fileUtilsInterface); } /* Step 4 : Insert into SQL database */ @@ -141,12 +143,12 @@ private void insertDatabase(){ /* Step 5 : Write output files */ private void outputFileWriter() throws KraftwerkException { WriterSequence writerSequence = new WriterSequence(); - writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModels, errors, null, database, fileUtilsInterface); + writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModels, null, database, fileUtilsInterface); } /* Step 6 : Write errors */ private void writeErrors() { - TextFileWriter.writeErrorsFile(inDirectory, executionDateTime, errors, fileUtilsInterface); + TextFileWriter.writeErrorsFile(inDirectory, executionDateTime, kraftwerkExecutionContext, fileUtilsInterface); } } diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java index 7723ac28..ae264a63 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java @@ -3,7 +3,6 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.api.configuration.MinioConfig; import fr.insee.kraftwerk.api.process.MainProcessing; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.StepEnum; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; @@ -14,6 +13,7 @@ import fr.insee.kraftwerk.core.utils.files.MinioImpl; import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.utils.TextFileWriter; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import io.minio.MinioClient; import io.swagger.v3.oas.annotations.Operation; @@ -26,11 +26,9 @@ import java.io.File; import java.nio.file.Path; -import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.time.LocalDateTime; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -82,10 +80,11 @@ public ResponseEntity buildVtlBindings( //Process BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(withAllReportingData, fileUtilsInterface); VtlReaderWriterSequence vtlWriterSequence = new VtlReaderWriterSequence(fileUtilsInterface); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); for (String dataMode : mp.getUserInputsFile().getModeInputsMap().keySet()) { try{ - buildBindingsSequence.buildVtlBindings(mp.getUserInputsFile(), dataMode, mp.getVtlBindings(),mp.getMetadataModels().get(dataMode), withDDI, null); + buildBindingsSequence.buildVtlBindings(mp.getUserInputsFile(), dataMode, mp.getVtlBindings(),mp.getMetadataModels().get(dataMode), withDDI, kraftwerkExecutionContext); } catch (KraftwerkException e){ return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } @@ -116,6 +115,7 @@ public ResponseEntity buildVtlBindingsByDataMode( fileUtilsInterface = new FileSystemImpl(); } + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize, fileUtilsInterface); try { mp.init(); @@ -126,7 +126,7 @@ public ResponseEntity buildVtlBindingsByDataMode( //Process BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(withAllReportingData, fileUtilsInterface); try{ - buildBindingsSequence.buildVtlBindings(mp.getUserInputsFile(), dataMode, mp.getVtlBindings(), mp.getMetadataModels().get(dataMode), withDDI, null); + buildBindingsSequence.buildVtlBindings(mp.getUserInputsFile(), dataMode, mp.getVtlBindings(), mp.getMetadataModels().get(dataMode), withDDI, kraftwerkExecutionContext); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } @@ -152,6 +152,7 @@ public ResponseEntity unimodalProcessing( }else{ fileUtilsInterface = new FileSystemImpl(); } + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); //Read data in JSON file Path inDirectory; @@ -167,7 +168,6 @@ public ResponseEntity unimodalProcessing( return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } VtlBindings vtlBindings = new VtlBindings(); - List errors = new ArrayList<>(); VtlReaderWriterSequence vtlReaderSequence = new VtlReaderWriterSequence(fileUtilsInterface); vtlReaderSequence.readDataset(FileUtilsInterface.transformToTemp(inDirectory).toString(),dataMode, StepEnum.BUILD_BINDINGS, vtlBindings); @@ -176,12 +176,12 @@ public ResponseEntity unimodalProcessing( //Process UnimodalSequence unimodal = new UnimodalSequence(); - unimodal.applyUnimodalSequence(userInputsFile, dataMode, vtlBindings, errors, metadataModelMap, fileUtilsInterface); + unimodal.applyUnimodalSequence(userInputsFile, dataMode, vtlBindings, kraftwerkExecutionContext, metadataModelMap, fileUtilsInterface); //Write technical outputs VtlReaderWriterSequence vtlWriterSequence = new VtlReaderWriterSequence(fileUtilsInterface); vtlWriterSequence.writeTempBindings(inDirectory, dataMode, vtlBindings, StepEnum.UNIMODAL_PROCESSING); - TextFileWriter.writeErrorsFile(inDirectory, LocalDateTime.now(), errors, fileUtilsInterface); + TextFileWriter.writeErrorsFile(inDirectory, LocalDateTime.now(), kraftwerkExecutionContext, fileUtilsInterface); return ResponseEntity.ok(inDirectoryParam+ " - "+dataMode); @@ -214,7 +214,7 @@ public ResponseEntity multimodalProcessing( } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); VtlReaderWriterSequence vtlReaderWriterSequence = new VtlReaderWriterSequence(fileUtilsInterface); @@ -229,13 +229,13 @@ public ResponseEntity multimodalProcessing( //Process MultimodalSequence multimodalSequence = new MultimodalSequence(); - multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModelMap, fileUtilsInterface); + multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, kraftwerkExecutionContext, metadataModelMap, fileUtilsInterface); //Write technical fils for (String datasetName : vtlBindings.getDatasetNames()) { vtlReaderWriterSequence.writeTempBindings(inDirectory, datasetName, vtlBindings, StepEnum.MULTIMODAL_PROCESSING); } - TextFileWriter.writeErrorsFile(inDirectory, LocalDateTime.now(), errors, fileUtilsInterface); + TextFileWriter.writeErrorsFile(inDirectory, LocalDateTime.now(), kraftwerkExecutionContext, fileUtilsInterface); return ResponseEntity.ok(inDirectoryParam); @@ -262,7 +262,8 @@ public ResponseEntity writeOutputFiles( } LocalDateTime executionDateTime = LocalDateTime.now(); VtlBindings vtlBindings = new VtlBindings(); - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); + // Read all bindings necessary to produce output String path = FileUtilsInterface.transformToTemp(inDirectory).toString(); List fileNames = fileUtilsInterface.listFileNames(path); @@ -282,7 +283,7 @@ public ResponseEntity writeOutputFiles( } Map metadataModelMap = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap(), fileUtilsInterface); try (Statement database = SqlUtils.openConnection().createStatement()) { - writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModelMap, errors, null, database, fileUtilsInterface); + writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModelMap, kraftwerkExecutionContext, database, fileUtilsInterface); } return ResponseEntity.ok(inDirectoryParam); diff --git a/kraftwerk-api/src/main/resources/application.properties b/kraftwerk-api/src/main/resources/application.properties index 658a1a3d..8cec9656 100644 --- a/kraftwerk-api/src/main/resources/application.properties +++ b/kraftwerk-api/src/main/resources/application.properties @@ -7,7 +7,7 @@ spring.application.name=@project.name@ @project.version@ fr.insee.kraftwerk.lang=fr # Import Ops properties -spring.config.import=classpath:i18n/messages_${fr.insee.kraftwerk.lang}.properties,kraftwerk.properties,optional:file:${catalina.base}/webapps/kraftwerk.properties +spring.config.import=classpath:i18n/messages_${fr.insee.kraftwerk.lang}.properties,optional:kraftwerk.properties,optional:file:${catalina.base}/webapps/kraftwerk.properties # Config Swagger (only for display) fr.insee.kraftwerk.version=@project.version@ diff --git a/kraftwerk-api/src/main/resources/kraftwerk.properties b/kraftwerk-api/src/main/resources/kraftwerk_example.properties similarity index 72% rename from kraftwerk-api/src/main/resources/kraftwerk.properties rename to kraftwerk-api/src/main/resources/kraftwerk_example.properties index b32b9bab..2b47b9d5 100644 --- a/kraftwerk-api/src/main/resources/kraftwerk.properties +++ b/kraftwerk-api/src/main/resources/kraftwerk_example.properties @@ -1,6 +1,7 @@ -########################################## -## Properties that are fixed by OPS ## -########################################## +################################################## +## Properties that are fixed by OPS ## +## For local usage, create kraftwerk.properties ## +################################################## # Folders for in and out fr.insee.postcollecte.files = *** diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java index 95c1b52d..89033eda 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java @@ -1,9 +1,8 @@ package fr.insee.kraftwerk.core.dataprocessing; import fr.insee.bpm.metadata.model.CalculatedVariables; - -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; import lombok.extern.log4j.Log4j2; @@ -18,7 +17,7 @@ public class CalculatedProcessing extends DataProcessing { /** Maximal number of iterations to resolve the order of execution of VTL expressions. */ public static final int MAXIMAL_RESOLVING_ITERATIONS = 100; - private CalculatedVariables calculatedVariables; + private final CalculatedVariables calculatedVariables; public CalculatedProcessing(VtlBindings vtlBindings, CalculatedVariables calculatedVariables, FileUtilsInterface fileUtilsInterface) { super(vtlBindings, fileUtilsInterface); @@ -31,13 +30,13 @@ public String getStepName() { } - public String applyCalculatedVtlTransformations(String bindingName, Path userVtlInstructionsPath, List errors){ + public String applyCalculatedVtlTransformations(String bindingName, Path userVtlInstructionsPath, KraftwerkExecutionContext kraftwerkExecutionContext){ // First step - String automatedVtlInstructions = applyAutomatedVtlInstructions(bindingName, errors); + String automatedVtlInstructions = applyAutomatedVtlInstructions(bindingName, kraftwerkExecutionContext); // Second step if(userVtlInstructionsPath != null) { - applyUserVtlInstructions(userVtlInstructionsPath, errors); - applyAutomatedVtlInstructions(bindingName, errors); + applyUserVtlInstructions(userVtlInstructionsPath, kraftwerkExecutionContext); + applyAutomatedVtlInstructions(bindingName, kraftwerkExecutionContext); } else { log.info(String.format("No user VTL instructions given for dataset named %s (step %s).", bindingName, getStepName())); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java index 9e5fb717..5b0c3844 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java @@ -3,8 +3,8 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.PaperUcq; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; import lombok.extern.log4j.Log4j2; @@ -45,11 +45,11 @@ public String getStepName() { * @param userVtlInstructionsPath User vtl script (none for this step). */ @Override - public String applyVtlTransformations(String bindingName, Path userVtlInstructionsPath, List errors) { + public String applyVtlTransformations(String bindingName, Path userVtlInstructionsPath, KraftwerkExecutionContext kraftwerkExecutionContext) { // Remove paper UCQ variables in vtl multimode dataset VtlScript cleanUpScript = generateVtlInstructions(bindingName); log.debug("Automated clean up instructions after step {} : {}", getStepName(), cleanUpScript); - vtlExecute.evalVtlScript(cleanUpScript, vtlBindings, errors); + vtlExecute.evalVtlScript(cleanUpScript, vtlBindings, kraftwerkExecutionContext); // Remove corresponding variables in VariablesMap removePaperUcqVariables(); // Remove unimodal datasets diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessing.java index 27f58748..bd134a49 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessing.java @@ -1,15 +1,14 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.utils.TextFileReader; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import fr.insee.kraftwerk.core.vtl.VtlScript; import lombok.extern.log4j.Log4j2; import java.nio.file.Path; -import java.util.List; /** @@ -36,16 +35,16 @@ protected DataProcessing(VtlBindings vtlBindings, FileUtilsInterface fileUtilsIn public abstract String getStepName(); - public String applyVtlTransformations(String bindingName, Path userVtlInstructionsPath, List errors){ + public String applyVtlTransformations(String bindingName, Path userVtlInstructionsPath, KraftwerkExecutionContext kraftwerkExecutionContext){ // First step - String automatedVtlInstructions = applyAutomatedVtlInstructions(bindingName, errors); + String automatedVtlInstructions = applyAutomatedVtlInstructions(bindingName, kraftwerkExecutionContext); // Second step if(userVtlInstructionsPath == null || !userVtlInstructionsPath.toFile().exists()){ log.info(String.format("No user VTL instructions given for dataset named %s (step %s).", bindingName, getStepName())); return automatedVtlInstructions; } - applyUserVtlInstructions(userVtlInstructionsPath, errors); + applyUserVtlInstructions(userVtlInstructionsPath, kraftwerkExecutionContext); return automatedVtlInstructions; } @@ -59,21 +58,21 @@ public String applyVtlTransformations(String bindingName, Path userVtlInstructio */ protected abstract VtlScript generateVtlInstructions(String bindingName); - protected String applyAutomatedVtlInstructions(String bindingName, List errors){ + protected String applyAutomatedVtlInstructions(String bindingName, KraftwerkExecutionContext kraftwerkExecutionContext){ VtlScript automatedInstructions = generateVtlInstructions(bindingName); log.debug(String.format("Automated VTL instructions generated for step %s: see temp file", getStepName())); if (!(automatedInstructions.isEmpty() || automatedInstructions.toString().contentEquals(""))) { - vtlExecute.evalVtlScript(automatedInstructions, vtlBindings, errors); + vtlExecute.evalVtlScript(automatedInstructions, vtlBindings, kraftwerkExecutionContext); } return automatedInstructions.toString(); } - protected void applyUserVtlInstructions(Path userVtlInstructionsPath, List errors){ + protected void applyUserVtlInstructions(Path userVtlInstructionsPath, KraftwerkExecutionContext kraftwerkExecutionContext){ String vtlScript = TextFileReader.readFromPath(userVtlInstructionsPath, fileUtilsInterface); log.info(String.format("User VTL instructions read for step %s:%n%s", getStepName(), vtlScript)); if (! (vtlScript == null || vtlScript.isEmpty() || vtlScript.contentEquals("")) ) { - vtlExecute.evalVtlScript(vtlScript, vtlBindings,errors); + vtlExecute.evalVtlScript(vtlScript, vtlBindings, kraftwerkExecutionContext); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java index a815b899..aaae1bd2 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java @@ -1,16 +1,15 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.TextFileReader; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.EnumUtils; import java.nio.file.Path; -import java.util.List; @Slf4j public class TCMSequencesProcessing extends DataProcessing { @@ -34,11 +33,11 @@ protected VtlScript generateVtlInstructions(String bindingName) { } @Override - public String applyAutomatedVtlInstructions(String bindingName, List errors){ + public String applyAutomatedVtlInstructions(String bindingName, KraftwerkExecutionContext kraftwerkExecutionContext){ VtlScript automatedInstructions = generateVtlInstructions(bindingName, metadataModel); log.debug(String.format("Automated VTL instructions generated for step %s: see temp file", getStepName())); if (!(automatedInstructions.isEmpty() || automatedInstructions.toString().contentEquals(""))) { - vtlExecute.evalVtlScript(automatedInstructions, vtlBindings, errors); + vtlExecute.evalVtlScript(automatedInstructions, vtlBindings, kraftwerkExecutionContext); } return automatedInstructions.toString(); } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java index 9019250b..57f57c09 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java @@ -5,6 +5,7 @@ import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -65,7 +66,7 @@ private void setOutputDatasetNames(List modes) { } } - public void writeImportScripts(Map metadataModels, List errors) { + public void writeImportScripts(Map metadataModels, KraftwerkExecutionContext kraftwerkExecutionContext) { //Should be override } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java index 208e762f..d3547129 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java @@ -1,7 +1,6 @@ package fr.insee.kraftwerk.core.outputs.csv; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.VariableType; @@ -10,7 +9,7 @@ import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -33,7 +32,7 @@ */ @Slf4j public class CsvOutputFiles extends OutputFiles { - private final KraftwerkExecutionLog kraftwerkExecutionLog; + private final KraftwerkExecutionContext kraftwerkExecutionContext; /** * When an instance is created, the output folder is created. @@ -43,11 +42,11 @@ public class CsvOutputFiles extends OutputFiles { */ public CsvOutputFiles(Path outDirectory, VtlBindings vtlBindings, List modes, Statement database, FileUtilsInterface fileUtilsInterface) { super(outDirectory, vtlBindings, modes, database, fileUtilsInterface); - this.kraftwerkExecutionLog = null; + this.kraftwerkExecutionContext = null; } - public CsvOutputFiles(Path outDirectory, VtlBindings vtlBindings, KraftwerkExecutionLog kraftwerkExecutionLog, List modes, Statement database, FileUtilsInterface fileUtilsInterface) { + public CsvOutputFiles(Path outDirectory, VtlBindings vtlBindings, KraftwerkExecutionContext kraftwerkExecutionContext, List modes, Statement database, FileUtilsInterface fileUtilsInterface) { super(outDirectory, vtlBindings, modes, database, fileUtilsInterface); - this.kraftwerkExecutionLog = kraftwerkExecutionLog; + this.kraftwerkExecutionContext = kraftwerkExecutionContext; } @@ -104,11 +103,11 @@ public void writeOutputTables() throws KraftwerkException { getFileUtilsInterface().moveFile(tmpOutputFile, outputFile); log.info(String.format("File: %s successfully written", outputFile)); //Count rows for functional log - if (kraftwerkExecutionLog != null) { + if (kraftwerkExecutionContext != null) { try(ResultSet countResult = this.getDatabase().executeQuery("SELECT COUNT(*) FROM '%s'".formatted(datasetName))){ countResult.next(); - kraftwerkExecutionLog.getLineCountByTableMap().put(datasetName, countResult.getInt(1)); + kraftwerkExecutionContext.getLineCountByTableMap().put(datasetName, countResult.getInt(1)); } } } catch (SQLException | IOException e) { @@ -175,7 +174,7 @@ private String applyNullTransformation(String csvLine) { } @Override - public void writeImportScripts(Map metadataModels, List errors) { + public void writeImportScripts(Map metadataModels, KraftwerkExecutionContext kraftwerkExecutionContext) { // Assemble required info to write scripts List tableScriptInfoList = new ArrayList<>(); for (String datasetName : getDatasetToCreate()) { @@ -187,7 +186,10 @@ public void writeImportScripts(Map metadataModels, List() : + kraftwerkExecutionContext.getErrors() + ).generateScript(), getFileUtilsInterface()); } /** diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/parquet/ParquetOutputFiles.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/parquet/ParquetOutputFiles.java index 5bd77934..1a819f19 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/parquet/ParquetOutputFiles.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/parquet/ParquetOutputFiles.java @@ -7,6 +7,7 @@ import fr.insee.kraftwerk.core.outputs.TableScriptInfo; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.extern.slf4j.Slf4j; @@ -74,7 +75,7 @@ public void writeOutputTables() throws KraftwerkException { @Override - public void writeImportScripts(Map metadataModels, List errors) { + public void writeImportScripts(Map metadataModels, KraftwerkExecutionContext kraftwerkExecutionContext) { // Assemble required info to write scripts List tableScriptInfoList = new ArrayList<>(); for (String datasetName : getDatasetToCreate()) { diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/DataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/DataParser.java index 9eb1ee11..9233f50a 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/DataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/DataParser.java @@ -3,7 +3,7 @@ import fr.insee.kraftwerk.core.exceptions.NullException; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import lombok.extern.log4j.Log4j2; import java.nio.file.Path; @@ -36,7 +36,7 @@ protected DataParser(SurveyRawData data, FileUtilsInterface fileUtilsInterface) * @param dataPath A data file, or a folder only containing data files. * @throws NullException -- throws null exception if datapath or a file is missing */ - public final void parseSurveyData(Path dataPath, KraftwerkExecutionLog kraftwerkExecutionLog) throws NullException { + public final void parseSurveyData(Path dataPath, KraftwerkExecutionContext kraftwerkExecutionContext) throws NullException { if (dataPath == null){ log.error(DATAPATH_IS_NULL); throw new NullException(DATAPATH_IS_NULL); @@ -47,16 +47,16 @@ public final void parseSurveyData(Path dataPath, KraftwerkExecutionLog kraftwerk } if(Boolean.FALSE.equals(fileUtilsInterface.isDirectory(dataPath.toString()))){ parseDataFile(dataPath); - if(kraftwerkExecutionLog != null) { - kraftwerkExecutionLog.getOkFileNames().add(dataPath.getFileName().toString()); + if(kraftwerkExecutionContext != null) { + kraftwerkExecutionContext.getOkFileNames().add(dataPath.getFileName().toString()); } } if(Boolean.TRUE.equals(fileUtilsInterface.isDirectory(dataPath.toString()))){ for(String path : fileUtilsInterface.listFilePaths(String.valueOf(dataPath))){ try { parseDataFile(Path.of(path)); - if(kraftwerkExecutionLog != null) { - kraftwerkExecutionLog.getOkFileNames().add(Path.of(path).getFileName().toString()); + if(kraftwerkExecutionContext != null) { + kraftwerkExecutionContext.getOkFileNames().add(Path.of(path).getFileName().toString()); } } catch (NullException e) { log.error("IOException occurred when trying to list data file: {} in folder {}", path, dataPath); @@ -71,7 +71,7 @@ public final void parseSurveyData(Path dataPath, KraftwerkExecutionLog kraftwerk * @param dataPath A data file, or a folder only containing data files. * @throws NullException -- throws null exception if datapath or a file is missing */ - public final void parseSurveyDataWithoutDDI(Path dataPath, Path lunaticFile, KraftwerkExecutionLog kraftwerkExecutionLog) throws NullException { + public final void parseSurveyDataWithoutDDI(Path dataPath, Path lunaticFile, KraftwerkExecutionContext kraftwerkExecutionContext) throws NullException { if (dataPath == null){ log.error(DATAPATH_IS_NULL); throw new NullException(DATAPATH_IS_NULL); @@ -82,16 +82,16 @@ public final void parseSurveyDataWithoutDDI(Path dataPath, Path lunaticFile, Kra } if(Boolean.FALSE.equals(fileUtilsInterface.isDirectory(dataPath.toString()))) { parseDataFileWithoutDDI(dataPath,lunaticFile); - if(kraftwerkExecutionLog != null) { - kraftwerkExecutionLog.getOkFileNames().add(dataPath.getFileName().toString()); + if(kraftwerkExecutionContext != null) { + kraftwerkExecutionContext.getOkFileNames().add(dataPath.getFileName().toString()); } } if(Boolean.TRUE.equals(fileUtilsInterface.isDirectory(dataPath.toString()))) { for(String path : fileUtilsInterface.listFilePaths(dataPath.toString())){ try { parseDataFileWithoutDDI(Path.of(path),lunaticFile); - if(kraftwerkExecutionLog != null) { - kraftwerkExecutionLog.getOkFileNames().add(Path.of(path).getFileName().toString()); + if(kraftwerkExecutionContext != null) { + kraftwerkExecutionContext.getOkFileNames().add(Path.of(path).getFileName().toString()); } } catch (NullException e) { log.error("IOException occurred when trying to list data file: {} in folder {}", path, dataPath); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java index 850a9cd7..5de31c4b 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java @@ -14,7 +14,7 @@ import fr.insee.kraftwerk.core.parsers.DataParserManager; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import lombok.extern.log4j.Log4j2; @@ -34,7 +34,7 @@ public BuildBindingsSequence(boolean withAllReportingData, FileUtilsInterface fi this.fileUtilsInterface = fileUtilsInterface; } - public void buildVtlBindings(UserInputsFile userInputsFile, String dataMode, VtlBindings vtlBindings, MetadataModel metadataModel, boolean withDDI, KraftwerkExecutionLog kraftwerkExecutionLog) throws KraftwerkException { + public void buildVtlBindings(UserInputsFile userInputsFile, String dataMode, VtlBindings vtlBindings, MetadataModel metadataModel, boolean withDDI, KraftwerkExecutionContext kraftwerkExecutionContext) throws KraftwerkException { ModeInputs modeInputs = userInputsFile.getModeInputs(dataMode); SurveyRawData data = new SurveyRawData(); @@ -46,9 +46,9 @@ public void buildVtlBindings(UserInputsFile userInputsFile, String dataMode, Vtl DataParser parser = DataParserManager.getParser(modeInputs.getDataFormat(), data, fileUtilsInterface); log.info("Parsing survey data file or folder : {}" , modeInputs.getDataFile().getFileName()); if (withDDI) { - parser.parseSurveyData(modeInputs.getDataFile(),kraftwerkExecutionLog); + parser.parseSurveyData(modeInputs.getDataFile(), kraftwerkExecutionContext); } else { - parser.parseSurveyDataWithoutDDI(modeInputs.getDataFile(), modeInputs.getLunaticFile(), kraftwerkExecutionLog); + parser.parseSurveyDataWithoutDDI(modeInputs.getDataFile(), modeInputs.getLunaticFile(), kraftwerkExecutionContext); } /* Step 2.2 : Get paradata for the survey */ diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java index 62f41eff..3be02fdb 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java @@ -4,7 +4,6 @@ import java.util.Map; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.CleanUpProcessing; import fr.insee.kraftwerk.core.dataprocessing.DataProcessing; import fr.insee.kraftwerk.core.dataprocessing.InformationLevelsProcessing; @@ -14,36 +13,37 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.TextFileWriter; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.NoArgsConstructor; @NoArgsConstructor public class MultimodalSequence { - public void multimodalProcessing(UserInputs userInputs, VtlBindings vtlBindings, List errors, Map metadataModels, FileUtilsInterface fileUtilsInterface) { + public void multimodalProcessing(UserInputs userInputs, VtlBindings vtlBindings, KraftwerkExecutionContext kraftwerkExecutionContext, Map metadataModels, FileUtilsInterface fileUtilsInterface) { String multimodeDatasetName = Constants.MULTIMODE_DATASET_NAME; /* Step 3.1 : aggregate unimodal datasets into a multimodal unique dataset */ DataProcessing reconciliationProcessing = new ReconciliationProcessing(vtlBindings, fileUtilsInterface); String vtlGenerate = reconciliationProcessing.applyVtlTransformations(multimodeDatasetName, - userInputs.getVtlReconciliationFile(), errors); + userInputs.getVtlReconciliationFile(), kraftwerkExecutionContext); TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "ReconciliationProcessing",multimodeDatasetName), vtlGenerate, fileUtilsInterface); /* Step 3.1.b : clean up processing */ CleanUpProcessing cleanUpProcessing = new CleanUpProcessing(vtlBindings, metadataModels, fileUtilsInterface); - vtlGenerate = cleanUpProcessing.applyVtlTransformations(multimodeDatasetName, null, errors); + vtlGenerate = cleanUpProcessing.applyVtlTransformations(multimodeDatasetName, null, kraftwerkExecutionContext); TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "CleanUpProcessing",multimodeDatasetName), vtlGenerate, fileUtilsInterface); /* Step 3.2 : treatments on the multimodal dataset */ DataProcessing multimodeTransformations = new MultimodeTransformations(vtlBindings, fileUtilsInterface); vtlGenerate = multimodeTransformations.applyVtlTransformations(multimodeDatasetName, - userInputs.getVtlTransformationsFile(), errors); + userInputs.getVtlTransformationsFile(), kraftwerkExecutionContext); TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "MultimodeTransformations",multimodeDatasetName), vtlGenerate, fileUtilsInterface); /* Step 3.3 : create datasets on each information level (i.e. each group) */ DataProcessing informationLevelsProcessing = new InformationLevelsProcessing(vtlBindings, fileUtilsInterface); vtlGenerate = informationLevelsProcessing.applyVtlTransformations(multimodeDatasetName, - userInputs.getVtlInformationLevelsFile(), errors); + userInputs.getVtlInformationLevelsFile(), kraftwerkExecutionContext); TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "InformationLevelsProcessing",multimodeDatasetName), vtlGenerate, fileUtilsInterface); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java index 5636f1c5..7da3984a 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java @@ -8,19 +8,18 @@ import fr.insee.bpm.metadata.reader.lunatic.LunaticReader; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.*; import fr.insee.kraftwerk.core.inputs.ModeInputs; import fr.insee.kraftwerk.core.inputs.UserInputs; import fr.insee.kraftwerk.core.metadata.*; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.TextFileWriter; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.NoArgsConstructor; import lombok.extern.log4j.Log4j2; import java.nio.file.Path; -import java.util.List; import java.util.Map; @NoArgsConstructor @@ -28,7 +27,7 @@ public class UnimodalSequence { public void applyUnimodalSequence(UserInputs userInputs, String dataMode, VtlBindings vtlBindings, - List errors, Map metadataModels, FileUtilsInterface fileUtilsInterface) { + KraftwerkExecutionContext kraftwerkExecutionContext, Map metadataModels, FileUtilsInterface fileUtilsInterface) { ModeInputs modeInputs = userInputs.getModeInputs(dataMode); String vtlGenerate; @@ -38,10 +37,7 @@ public void applyUnimodalSequence(UserInputs userInputs, String dataMode, VtlBin Variable variable = variablesMap.getVariable(variableName); if (variable.getSasFormat() != null && variable.getExpectedLength() modeInputsMap, Map metadataModels, - List errors, - KraftwerkExecutionLog kraftwerkExecutionLog, + KraftwerkExecutionContext kraftwerkExecutionContext, Statement database, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { Path outDirectory = FileUtilsInterface.transformToOut(inDirectory,executionDateTime); - writeCsvFiles(inDirectory, executionDateTime,vtlBindings, modeInputsMap, metadataModels, errors, kraftwerkExecutionLog, database, fileUtilsInterface); - writeParquetFiles(outDirectory, vtlBindings, modeInputsMap, metadataModels, errors, database, fileUtilsInterface); + writeCsvFiles(inDirectory, executionDateTime,vtlBindings, modeInputsMap, metadataModels, kraftwerkExecutionContext, database, fileUtilsInterface); + writeParquetFiles(outDirectory, vtlBindings, modeInputsMap, metadataModels, kraftwerkExecutionContext, database, fileUtilsInterface); } public void writeCsvFiles(Path inDirectory, @@ -42,18 +39,17 @@ public void writeCsvFiles(Path inDirectory, VtlBindings vtlBindings, Map modeInputsMap, Map metadataModels, - List errors, - KraftwerkExecutionLog kraftwerkExecutionLog, + KraftwerkExecutionContext kraftwerkExecutionContext, Statement database, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { //Write CSV Path outDirectory = FileUtilsInterface.transformToOut(inDirectory,executionDateTime); /* Step 5.1 : write csv output tables */ - OutputFiles csvOutputFiles = new CsvOutputFiles(outDirectory, vtlBindings, kraftwerkExecutionLog, new ArrayList<>(modeInputsMap.keySet()), + OutputFiles csvOutputFiles = new CsvOutputFiles(outDirectory, vtlBindings, kraftwerkExecutionContext, new ArrayList<>(modeInputsMap.keySet()), database, fileUtilsInterface); csvOutputFiles.writeOutputTables(); /* Step 5.2 : write scripts to import csv tables in several languages */ - csvOutputFiles.writeImportScripts(metadataModels, errors); + csvOutputFiles.writeImportScripts(metadataModels, kraftwerkExecutionContext); } @@ -62,12 +58,12 @@ private void writeParquetFiles(Path outDirectory, VtlBindings vtlBindings, Map modeInputsMap, Map metadataModels, - List errors, + KraftwerkExecutionContext kraftwerkExecutionContext, Statement database, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { /* Step 5.3 : write parquet output tables */ OutputFiles parquetOutputFiles = new ParquetOutputFiles(outDirectory, vtlBindings, new ArrayList<>(modeInputsMap.keySet()), database, fileUtilsInterface); parquetOutputFiles.writeOutputTables(); - parquetOutputFiles.writeImportScripts(metadataModels, errors); + parquetOutputFiles.writeImportScripts(metadataModels, kraftwerkExecutionContext); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileWriter.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileWriter.java index 11f397f5..e7a567f3 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileWriter.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileWriter.java @@ -1,16 +1,12 @@ package fr.insee.kraftwerk.core.utils; -import java.io.IOException; import java.nio.file.Path; -import java.nio.file.Paths; import java.time.LocalDateTime; -import java.util.List; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import lombok.extern.log4j.Log4j2; @@ -39,14 +35,14 @@ public static void writeFile(Path filePath, String fileContent, FileUtilsInterfa } } - public static void writeErrorsFile(Path inDirectory, LocalDateTime localDateTime, List errors, FileUtilsInterface fileUtilsInterface) { + public static void writeErrorsFile(Path inDirectory, LocalDateTime localDateTime, KraftwerkExecutionContext kraftwerkExecutionContext, FileUtilsInterface fileUtilsInterface) { Path tempOutputPath = FileUtilsInterface.transformToOut(inDirectory,localDateTime) .resolve(Constants.ERRORS_FILE_NAME); fileUtilsInterface.createDirectoryIfNotExist(tempOutputPath.getParent()); //Write errors file - if (!errors.isEmpty()) { - for (KraftwerkError error : errors) { + if (!kraftwerkExecutionContext.getErrors().isEmpty()) { + for (KraftwerkError error : kraftwerkExecutionContext.getErrors()) { fileUtilsInterface.writeFile(tempOutputPath.toString(), error.toString(), false); } log.info(String.format("Text file: %s successfully written", tempOutputPath)); @@ -55,11 +51,11 @@ public static void writeErrorsFile(Path inDirectory, LocalDateTime localDateTime } } - public static void writeLogFile(Path inDirectory, LocalDateTime localDateTime, KraftwerkExecutionLog kraftwerkExecutionLog, FileUtilsInterface fileUtilsInterface){ + public static void writeLogFile(Path inDirectory, LocalDateTime localDateTime, KraftwerkExecutionContext kraftwerkExecutionContext, FileUtilsInterface fileUtilsInterface){ Path tempOutputPath = FileUtilsInterface.transformToOut(inDirectory,localDateTime); - tempOutputPath = tempOutputPath.resolve(inDirectory.getFileName() + "_LOG_" + kraftwerkExecutionLog.getStartTimeStamp() +".txt"); + tempOutputPath = tempOutputPath.resolve(inDirectory.getFileName() + "_LOG_" + kraftwerkExecutionContext.getStartTimeStamp() +".txt"); - fileUtilsInterface.writeFile(tempOutputPath.toString(), kraftwerkExecutionLog.getFormattedString(), false); + fileUtilsInterface.writeFile(tempOutputPath.toString(), kraftwerkExecutionContext.getFormattedString(), false); } public static boolean pathContainsFolder(Path pathString, String folderToFind) { diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionLog.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionContext.java similarity index 81% rename from kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionLog.java rename to kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionContext.java index 6fa9747f..a5c552bd 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionLog.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionContext.java @@ -1,6 +1,7 @@ package fr.insee.kraftwerk.core.utils.log; import fr.insee.kraftwerk.core.Constants; +import fr.insee.kraftwerk.core.KraftwerkError; import lombok.Getter; import lombok.Setter; @@ -13,16 +14,19 @@ @Getter @Setter -public class KraftwerkExecutionLog { +public class KraftwerkExecutionContext { private long startTimeStamp; private long endTimeStamp; private Map lineCountByTableMap; private List okFileNames; - public KraftwerkExecutionLog() { + private List errors; + + public KraftwerkExecutionContext() { this.startTimeStamp = System.currentTimeMillis(); this.lineCountByTableMap = new HashMap<>(); this.okFileNames = new ArrayList<>(); + this.errors = new ArrayList<>(); } public String getFormattedString() { @@ -47,6 +51,11 @@ public String getFormattedString() { } return toWrite.toString(); + } + public void addUniqueError(KraftwerkError kraftwerkError){ + if (!errors.contains(kraftwerkError)){ + errors.add(kraftwerkError); + } } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlExecute.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlExecute.java index 2872dc1f..d3acd1b2 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlExecute.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlExecute.java @@ -3,11 +3,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; -import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.vtl.jackson.TrevasModule; import fr.insee.vtl.model.Dataset; import lombok.extern.log4j.Log4j2; @@ -24,7 +23,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; /** * Class that provide method to use the Trevas library. @@ -155,7 +153,7 @@ public Dataset getDataset(String bindingName, VtlBindings bindings){ * @param vtlScript * A string containing vtl instructions. */ - public void evalVtlScript(String vtlScript, VtlBindings bindings, List errors){ + public void evalVtlScript(String vtlScript, VtlBindings bindings, KraftwerkExecutionContext kraftwerkExecutionContext){ if(vtlScript != null && !vtlScript.isEmpty()) { try { // set script context @@ -168,35 +166,28 @@ public void evalVtlScript(String vtlScript, VtlBindings bindings, List errors, Throwable e) { - ErrorVtlTransformation error = new ErrorVtlTransformation(vtlScript, e.getMessage()); - if (!errors.contains(error)){ - errors.add(error); - } - } - /** * Evaluate the given VTL instructions and update the bindings. * The name of the input datasets in the script must refer to the names given in the bindings. @@ -204,10 +195,10 @@ private static void addError(String vtlScript, List errors, Thro * @param vtlScript * A string containing vtl instructions. */ - public void evalVtlScript(VtlScript vtlScript, VtlBindings bindings, List errors){ + public void evalVtlScript(VtlScript vtlScript, VtlBindings bindings, KraftwerkExecutionContext kraftwerkExecutionContext){ if(vtlScript != null && !vtlScript.isEmpty()) { for(String vtlInstruction : vtlScript) { - evalVtlScript(vtlInstruction, bindings, errors); + evalVtlScript(vtlInstruction, bindings, kraftwerkExecutionContext); } } else { log.info("null or empty VTL instructions list given. VTL bindings has not been changed."); diff --git a/kraftwerk-core/src/test/java/cucumber/unit_tests/AggregateDefinitions.java b/kraftwerk-core/src/test/java/cucumber/unit_tests/AggregateDefinitions.java index 00ed2efb..1c3c7788 100644 --- a/kraftwerk-core/src/test/java/cucumber/unit_tests/AggregateDefinitions.java +++ b/kraftwerk-core/src/test/java/cucumber/unit_tests/AggregateDefinitions.java @@ -8,6 +8,7 @@ import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import io.cucumber.java.en.Given; @@ -27,6 +28,7 @@ public class AggregateDefinitions { public List errors = new ArrayList<>(); VtlExecute vtlExecute = new VtlExecute(new FileSystemImpl()); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); @Given("We have some VTLBindings named {string} and {string}") public void initialize(String firstDataset, String secondDataset){ @@ -37,9 +39,9 @@ public void initialize(String firstDataset, String secondDataset){ vtlExecute.convertToVtlDataset(fakePapiData, secondDataset, vtlBindings); // add group prefixes GroupProcessing groupProcessing = new GroupProcessing(vtlBindings, fakeCawiData.getMetadataModel(), new FileSystemImpl()); - groupProcessing.applyVtlTransformations(firstDataset, null,errors); + groupProcessing.applyVtlTransformations(firstDataset, null, kraftwerkExecutionContext); GroupProcessing groupProcessing2 = new GroupProcessing(vtlBindings, fakePapiData.getMetadataModel(), new FileSystemImpl()); - groupProcessing2.applyVtlTransformations(secondDataset, null,errors); + groupProcessing2.applyVtlTransformations(secondDataset, null,kraftwerkExecutionContext); // assertTrue(vtlBindings.containsKey(firstDataset)); @@ -50,7 +52,7 @@ public void initialize(String firstDataset, String secondDataset){ public void collect_variables() { DataProcessing reconciliationProcessing = new ReconciliationProcessing(vtlBindings, new FileSystemImpl()); reconciliationProcessing.applyVtlTransformations( - "MULTIMODE", null,errors); + "MULTIMODE", null,kraftwerkExecutionContext); } @Then("The datasets I try to aggregate should return an aggregated dataset") diff --git a/kraftwerk-core/src/test/java/cucumber/unit_tests/EvalScriptDefinitions.java b/kraftwerk-core/src/test/java/cucumber/unit_tests/EvalScriptDefinitions.java index 02108eef..a781d20c 100644 --- a/kraftwerk-core/src/test/java/cucumber/unit_tests/EvalScriptDefinitions.java +++ b/kraftwerk-core/src/test/java/cucumber/unit_tests/EvalScriptDefinitions.java @@ -2,13 +2,10 @@ import static org.junit.Assert.assertEquals; -import java.util.ArrayList; -import java.util.List; - -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import io.cucumber.java.en.Given; @@ -20,6 +17,7 @@ public class EvalScriptDefinitions { public VtlBindings vtlBindings = new VtlBindings(); VtlExecute vtlExecute = new VtlExecute(new FileSystemImpl()); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); @Given("We have some simple VTLBindings") public void initialize() throws Exception { @@ -33,8 +31,7 @@ public void initialize() throws Exception { @When("I try to apply some VTL instruction : {string}") public void exportDataset(String vtlScript) throws Exception { - List errors = new ArrayList<>(); - vtlExecute.evalVtlScript(vtlScript, vtlBindings, errors); + vtlExecute.evalVtlScript(vtlScript, vtlBindings, kraftwerkExecutionContext); } @Then("The binding {string} should have {int} variables") diff --git a/kraftwerk-core/src/test/java/cucumber/unit_tests/ExportDatasetDefinitions.java b/kraftwerk-core/src/test/java/cucumber/unit_tests/ExportDatasetDefinitions.java index 7e4c9adc..f6dc4a73 100644 --- a/kraftwerk-core/src/test/java/cucumber/unit_tests/ExportDatasetDefinitions.java +++ b/kraftwerk-core/src/test/java/cucumber/unit_tests/ExportDatasetDefinitions.java @@ -5,6 +5,7 @@ import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import fr.insee.kraftwerk.core.vtl.VtlJsonDatasetWriter; @@ -12,8 +13,6 @@ import io.cucumber.java.en.Then; import io.cucumber.java.en.When; -import java.util.ArrayList; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -46,7 +45,7 @@ public void importDataset(String nameDataset) throws Exception { vtlExecute.putVtlDataset(tempDatasetPath, "OUTPUT_TEST_EXPORT", vtlBindings); // add group prefixes GroupProcessing groupProcessing = new GroupProcessing(vtlBindings, survey.getMetadataModel(), new FileSystemImpl()); - groupProcessing.applyVtlTransformations("OUTPUT_TEST_EXPORT", null, new ArrayList<>()); + groupProcessing.applyVtlTransformations("OUTPUT_TEST_EXPORT", null, new KraftwerkExecutionContext()); } @Then("I should get some dataset values from {string}") diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java index 50b46cac..ec8cefe0 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java @@ -1,12 +1,12 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.bpm.metadata.model.CalculatedVariables; import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; import fr.insee.vtl.model.Dataset; @@ -17,7 +17,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -27,7 +26,7 @@ class CalculatedProcessingTest { private static CalculatedVariables fooCalculated; private static MetadataModel fooMetadataModel; private static VtlBindings vtlBindings; - private static List errors; + private static KraftwerkExecutionContext kraftwerkExecutionContext; private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); @BeforeAll @@ -48,7 +47,7 @@ static void setFooCalculated() { fooMetadataModel.getVariables().putVariable(new Variable("FOO3", fooMetadataModel.getRootGroup(), VariableType.STRING)); // vtlBindings = new VtlBindings(); - errors = new ArrayList<>(); + kraftwerkExecutionContext = new KraftwerkExecutionContext(); } @Test @@ -71,7 +70,7 @@ void testIfCalculatedAreProcessed() { VtlBindings vtlBindings = getVtlBindings(); // CalculatedProcessing processing = new CalculatedProcessing(vtlBindings, fooCalculated, fileUtilsInterface); - processing.applyAutomatedVtlInstructions("TEST", errors); + processing.applyAutomatedVtlInstructions("TEST", kraftwerkExecutionContext); // Dataset outDataset = vtlBindings.getDataset("TEST"); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java index c534848e..00de554c 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java @@ -6,9 +6,9 @@ import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; @@ -75,7 +75,7 @@ void applyCleanUp() { // Metadata variables object Map metadatas = new LinkedHashMap<>(); // Errors list - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); // MetadataModel cawiMetadata = new MetadataModel(); cawiMetadata.getVariables().putVariable(new Variable("FOO", cawiMetadata.getRootGroup(), VariableType.STRING)); @@ -104,7 +104,7 @@ void applyCleanUp() { // Apply clean up CleanUpProcessing cleanUpProcessing = new CleanUpProcessing(vtlBindings, metadatas, fileUtilsInterface); - cleanUpProcessing.applyVtlTransformations("MULTIMODE", null,errors); + cleanUpProcessing.applyVtlTransformations("MULTIMODE", null,kraftwerkExecutionContext); // Are paper indicator variables removed in VTL multimode dataset ? assertFalse(vtlBindings.getDataset("MULTIMODE").getDataStructure().containsKey("GENDER_1")); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java index 1fab0e55..36ca790c 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java @@ -4,16 +4,15 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariableType; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -37,7 +36,7 @@ void addPrefixes() { new Structured.Component("FOO2", Long.class, Dataset.Role.MEASURE) ) ); - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); VtlBindings vtlBindings = new VtlBindings(); vtlBindings.put("TEST", initialDataset); // @@ -48,7 +47,7 @@ void addPrefixes() { metadata.getVariables().putVariable(new Variable("FOO1", metadata.getGroup("DEPTH1"), VariableType.NUMBER)); metadata.getVariables().putVariable(new Variable("FOO2", metadata.getGroup("DEPTH2"), VariableType.NUMBER)); // - new GroupProcessing(vtlBindings, metadata, fileUtilsInterface).applyAutomatedVtlInstructions("TEST",errors); + new GroupProcessing(vtlBindings, metadata, fileUtilsInterface).applyAutomatedVtlInstructions("TEST", kraftwerkExecutionContext); Dataset outDataset = vtlBindings.getDataset("TEST"); // diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsTest.java index c347bbf8..b06b754c 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsTest.java @@ -3,17 +3,16 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import org.junit.jupiter.api.Test; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; @@ -49,13 +48,13 @@ class InformationLevelsTest { @Test void applyInformationLevelsProcessing() { - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); // VtlBindings vtlBindings = new VtlBindings(); vtlBindings.put("MULTIMODE", testDataset); // InformationLevelsProcessing processing = new InformationLevelsProcessing(vtlBindings, fileUtilsInterface); - processing.applyAutomatedVtlInstructions("MULTIMODE",errors); + processing.applyAutomatedVtlInstructions("MULTIMODE", kraftwerkExecutionContext); // Dataset rootDataset = vtlBindings.getDataset(Constants.ROOT_GROUP_NAME); Dataset loopDataset = vtlBindings.getDataset("LOOP"); @@ -89,13 +88,13 @@ void applyInformationLevelsProcessing() { @Test void applyInformationLevelsProcessingWithPartialIdNUll() { - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); // VtlBindings vtlBindings = new VtlBindings(); vtlBindings.put("MULTIMODE", testDatasetWithPartialIdNull); // InformationLevelsProcessing processing = new InformationLevelsProcessing(vtlBindings, fileUtilsInterface); - processing.applyAutomatedVtlInstructions("MULTIMODE",errors); + processing.applyAutomatedVtlInstructions("MULTIMODE", kraftwerkExecutionContext); // Dataset rootDataset = vtlBindings.getDataset(Constants.ROOT_GROUP_NAME); Dataset loopDataset = vtlBindings.getDataset("LOOP"); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java index bd0962ac..2bb100c8 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java @@ -5,10 +5,9 @@ import fr.insee.bpm.metadata.model.Variable; import fr.insee.bpm.metadata.model.VariableType; import fr.insee.bpm.metadata.model.VariablesMap; - -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; @@ -16,7 +15,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -65,7 +63,7 @@ private void addPaperUcq(VariablesMap variablesMap, Group group, String variable @Test void testPaperDataProcessing() { // - List errors =new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); MetadataModel metadataModel = new MetadataModel(); Group rootGroup = metadataModel.getRootGroup(); metadataModel.getVariables().putVariable(new Variable("FOO", rootGroup, VariableType.NUMBER)); @@ -76,7 +74,7 @@ void testPaperDataProcessing() { vtlBindings.put("TEST", paperDataset); // PaperDataProcessing paperDataProcessing = new PaperDataProcessing(vtlBindings, metadataModel, fileUtilsInterface); - paperDataProcessing.applyAutomatedVtlInstructions("TEST", errors); + paperDataProcessing.applyAutomatedVtlInstructions("TEST", kraftwerkExecutionContext); // Dataset paperDsModified = vtlBindings.getDataset("TEST"); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationTest.java index 58bf71c0..8110c2ac 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationTest.java @@ -1,9 +1,9 @@ package fr.insee.kraftwerk.core.dataprocessing; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; @@ -13,7 +13,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -24,8 +23,8 @@ class ReconciliationTest { private VtlBindings vtlBindings; - private List errors; private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + private KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); InMemoryDataset capiDataset = new InMemoryDataset( List.of( @@ -74,7 +73,7 @@ class ReconciliationTest { @BeforeEach void initVtlBindings() { vtlBindings = new VtlBindings(); - errors = new ArrayList<>(); + kraftwerkExecutionContext = new KraftwerkExecutionContext(); } @ParameterizedTest @@ -84,7 +83,7 @@ void applyReconciliation_singleMode(String dsName) { vtlBindings.put("SINGLE_MODE", testDatasets.get(dsName)); // ReconciliationProcessing reconciliation = new ReconciliationProcessing(vtlBindings, fileUtilsInterface); - reconciliation.applyVtlTransformations("MULTIMODE", null,errors); + reconciliation.applyVtlTransformations("MULTIMODE", null, kraftwerkExecutionContext); // Dataset multimodeDataset = vtlBindings.getDataset("MULTIMODE"); assertNotNull(multimodeDataset); @@ -96,7 +95,7 @@ private Dataset applyReconciliation_twoModes(String mode1, String mode2) { vtlBindings.put(mode2, testDatasets.get(mode2)); // ReconciliationProcessing reconciliation = new ReconciliationProcessing(vtlBindings, fileUtilsInterface); - reconciliation.applyVtlTransformations("MULTIMODE", null,errors); + reconciliation.applyVtlTransformations("MULTIMODE", null, kraftwerkExecutionContext); // return vtlBindings.getDataset("MULTIMODE"); } @@ -140,7 +139,7 @@ void applyReconciliation_threeModes() { vtlBindings.put("PAPI", papiDataset); // ReconciliationProcessing reconciliation = new ReconciliationProcessing(vtlBindings, fileUtilsInterface); - reconciliation.applyVtlTransformations("MULTIMODE", null,errors); + reconciliation.applyVtlTransformations("MULTIMODE", null, kraftwerkExecutionContext); // Dataset multimodeDataset = vtlBindings.getDataset("MULTIMODE"); // diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java index 2b27902c..68c304df 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java @@ -1,11 +1,11 @@ package fr.insee.kraftwerk.core.dataprocessing; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.TestConstants; import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.Sequence; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; @@ -72,7 +72,8 @@ void check_standard_vtl_execution(){ metadataModel.getSequences().add(new Sequence(TCMSequenceEnum.TCM_THLHAB.name())); // Errors list - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); + StringBuilder expectedScriptBuilder = new StringBuilder(); for (TCMModuleEnum module : modules){ expectedScriptBuilder.append(String.format(FORMAT_INSTRUCTION,module)); @@ -82,7 +83,7 @@ void check_standard_vtl_execution(){ //WHEN TCMSequencesProcessing processing = new TCMSequencesProcessing(vtlBindings, metadataModel, Path.of(TestConstants.UNIT_TESTS_DIRECTORY).resolve("vtl").toString(), fileUtilsInterface); - String scriptString = processing.applyAutomatedVtlInstructions("TEST", errors); + String scriptString = processing.applyAutomatedVtlInstructions("TEST", kraftwerkExecutionContext); //THEN Assertions.assertThat(scriptString).isEqualToIgnoringNewLines(expectedScript); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java index fdac5a45..e92efa68 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java @@ -5,7 +5,6 @@ import fr.insee.bpm.metadata.model.PaperUcq; import fr.insee.bpm.metadata.model.UcqVariable; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.GroupProcessing; import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.Variable; @@ -16,6 +15,7 @@ import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import fr.insee.vtl.model.Dataset; @@ -25,7 +25,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -63,11 +62,11 @@ private void instantiateMap() { vtlExecute.convertToVtlDataset(srdPaper, "PAPI", vtlBindings); // add group prefixes - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); GroupProcessing groupProcessing = new GroupProcessing(vtlBindings, srdWeb.getMetadataModel(), fileUtilsInterface); - groupProcessing.applyVtlTransformations("CAWI", null, errors); + groupProcessing.applyVtlTransformations("CAWI", null, kraftwerkExecutionContext); GroupProcessing groupProcessing2 = new GroupProcessing(vtlBindings, srdPaper.getMetadataModel(), fileUtilsInterface); - groupProcessing2.applyVtlTransformations("PAPI", null, errors); + groupProcessing2.applyVtlTransformations("PAPI", null, kraftwerkExecutionContext); dataStructure = vtlBindings.getDataset("CAWI").getDataStructure(); tableScriptInfo = new TableScriptInfo("MULTIMODE", "TEST", dataStructure, metadata); @@ -104,13 +103,13 @@ void testGetAllLengthWithNumberType() { @Test void numberTypeInDatasets() { - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); Dataset ds = new InMemoryDataset( List.of(List.of(1L)), List.of(new Structured.Component("ID", Long.class, Dataset.Role.IDENTIFIER)) ); vtlBindings.put("test", ds); - vtlExecute.evalVtlScript("test := test [calc foo := 4.1];", vtlBindings,errors); + vtlExecute.evalVtlScript("test := test [calc foo := 4.1];", vtlBindings, kraftwerkExecutionContext); Dataset outDs = vtlBindings.getDataset("test"); assertEquals(Double.class, outDs.getDataPoints().get(0).get("foo").getClass()); // => "NUMBER" type in Trevas datasets is java "Double" type diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionContextTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionContextTest.java new file mode 100644 index 00000000..200afe44 --- /dev/null +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionContextTest.java @@ -0,0 +1,83 @@ +package fr.insee.kraftwerk.core.utils.log; + +import fr.insee.bpm.metadata.model.Group; +import fr.insee.bpm.metadata.model.Variable; +import fr.insee.bpm.metadata.model.VariableType; +import fr.insee.kraftwerk.core.KraftwerkError; +import fr.insee.kraftwerk.core.metadata.ErrorVariableLength; +import fr.insee.kraftwerk.core.vtl.ErrorVtlTransformation; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.HashMap; + +class KraftwerkExecutionContextTest { + @Test + void getFormattedString_test(){ + //GIVEN + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss.SSS"); + long start = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC); + long stop = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC); + + kraftwerkExecutionContext.setStartTimeStamp(start); + kraftwerkExecutionContext.setEndTimeStamp(stop); + kraftwerkExecutionContext.setOkFileNames(new ArrayList<>()); + kraftwerkExecutionContext.setLineCountByTableMap(new HashMap<>()); + + kraftwerkExecutionContext.getOkFileNames().add("TEST.xml"); + kraftwerkExecutionContext.getLineCountByTableMap().put("RACINE",1); + + //WHEN + String formattedString = kraftwerkExecutionContext.getFormattedString(); + + //THEN + Assertions.assertThat(formattedString).contains("TEST.xml", "RACINE", simpleDateFormat.format(start), simpleDateFormat.format(stop)); + } + + @Test + void addUniqueError_test(){ + //Given + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); + KraftwerkError kraftwerkError = new ErrorVtlTransformation("test","test"); + + //When + kraftwerkExecutionContext.addUniqueError(kraftwerkError); + + //Then + Assertions.assertThat(kraftwerkExecutionContext.getErrors()).isNotEmpty().containsExactly(kraftwerkError); + } + + @Test + void addUniqueError_2xSame_test(){ + //Given + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); + KraftwerkError kraftwerkError = new ErrorVtlTransformation("test","test"); + + //When + kraftwerkExecutionContext.addUniqueError(kraftwerkError); + kraftwerkExecutionContext.addUniqueError(kraftwerkError); + + //Then + Assertions.assertThat(kraftwerkExecutionContext.getErrors()).isNotEmpty().hasSize(1).containsExactly(kraftwerkError); + } + + @Test + void addUniqueError_2xDifferent_test(){ + //Given + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); + KraftwerkError kraftwerkError = new ErrorVtlTransformation("test","test"); + KraftwerkError kraftwerkError2 = new ErrorVariableLength(new Variable("TESTVAR",new Group("TESTGROUP"), VariableType.STRING),"WEB"); + + //When + kraftwerkExecutionContext.addUniqueError(kraftwerkError); + kraftwerkExecutionContext.addUniqueError(kraftwerkError2); + + //Then + Assertions.assertThat(kraftwerkExecutionContext.getErrors()).isNotEmpty().hasSize(2).containsExactly(kraftwerkError, kraftwerkError2); + } +} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionLogTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionLogTest.java deleted file mode 100644 index ac181309..00000000 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionLogTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package fr.insee.kraftwerk.core.utils.log; - -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.text.SimpleDateFormat; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.HashMap; - -class KraftwerkExecutionLogTest { - @Test - void getFormattedString_test(){ - //GIVEN - KraftwerkExecutionLog kraftwerkExecutionLog = new KraftwerkExecutionLog(); - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss.SSS"); - long start = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC); - long stop = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC); - - kraftwerkExecutionLog.setStartTimeStamp(start); - kraftwerkExecutionLog.setEndTimeStamp(stop); - kraftwerkExecutionLog.setOkFileNames(new ArrayList<>()); - kraftwerkExecutionLog.setLineCountByTableMap(new HashMap<>()); - - kraftwerkExecutionLog.getOkFileNames().add("TEST.xml"); - kraftwerkExecutionLog.getLineCountByTableMap().put("RACINE",1); - - //WHEN - String formattedString = kraftwerkExecutionLog.getFormattedString(); - - //THEN - Assertions.assertThat(formattedString).contains("TEST.xml", "RACINE", simpleDateFormat.format(start), simpleDateFormat.format(stop)); - } -} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java index 2a82a049..1ee9a8bb 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java @@ -7,6 +7,7 @@ import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; import fr.insee.vtl.model.InMemoryDataset; @@ -26,7 +27,7 @@ class VtlBindingsTest { private VtlBindings vtlBindings; - private List errors; + private KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); VtlExecute vtlExecute = new VtlExecute(fileUtilsInterface); @@ -49,7 +50,7 @@ class VtlBindingsTest { @BeforeEach public void initVtlBindings() { vtlBindings = new VtlBindings(); - errors = new ArrayList<>(); + kraftwerkExecutionContext = new KraftwerkExecutionContext(); } @Test @@ -97,7 +98,7 @@ void evalVtlScriptTest_uniqueString() { vtlScript.append(" \"\" ))];"); log.info("Test VTL script:"); log.info(vtlScript.toString()); - vtlExecute.evalVtlScript(vtlScript.toString(), vtlBindings,errors); + vtlExecute.evalVtlScript(vtlScript.toString(), vtlBindings, kraftwerkExecutionContext); // Dataset ds = vtlBindings.getDataset("TEST"); @@ -113,9 +114,9 @@ void evalVtlScriptTest_uniqueString() { void evalEmptyVtlString() { VtlBindings vtlBindingsInitial = vtlBindings; List errors = new ArrayList<>(); - vtlExecute.evalVtlScript((String) null, vtlBindings, errors); - vtlExecute.evalVtlScript((VtlScript) null, vtlBindings,errors); - vtlExecute.evalVtlScript("", vtlBindings, errors); + vtlExecute.evalVtlScript((String) null, vtlBindings, kraftwerkExecutionContext); + vtlExecute.evalVtlScript((VtlScript) null, vtlBindings, kraftwerkExecutionContext); + vtlExecute.evalVtlScript("", vtlBindings, kraftwerkExecutionContext); assertEquals(vtlBindingsInitial, vtlBindings); assertEquals(0, errors.size()); } @@ -123,8 +124,8 @@ void evalEmptyVtlString() { @Test void evalEmptyVtlScriptObject() { - vtlExecute.evalVtlScript(new VtlScript(), vtlBindings,errors); - assertEquals(0, errors.size()); + vtlExecute.evalVtlScript(new VtlScript(), vtlBindings, kraftwerkExecutionContext); + assertEquals(0, kraftwerkExecutionContext.getErrors().size()); } @@ -137,7 +138,7 @@ void evalVtlScriptTest_scriptObject() { vtlScript.add("TEST := TEST [calc new1 := \"new\"];"); vtlScript.add("nOt VtL cOdE "); // should write a warning in the log but not throw an exception vtlScript.add("TEST := TEST [calc new2 := 2];"); - vtlExecute.evalVtlScript(vtlScript, vtlBindings,errors); + vtlExecute.evalVtlScript(vtlScript, vtlBindings, kraftwerkExecutionContext); // Dataset ds = vtlBindings.getDataset("TEST"); diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java index 421f5c1a..4d60ebd5 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java @@ -3,7 +3,6 @@ import cucumber.TestConstants; import fr.insee.bpm.exceptions.MetadataParserException; import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.CalculatedProcessing; import fr.insee.kraftwerk.core.dataprocessing.DataProcessing; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; @@ -15,6 +14,7 @@ import fr.insee.kraftwerk.core.parsers.LunaticXmlDataParser; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import fr.insee.vtl.model.Dataset; @@ -46,7 +46,7 @@ public class CalculatedProcessingDefinition { private List variableNamesList; VtlExecute vtlExecute = new VtlExecute(new FileSystemImpl()); - List errors = new ArrayList<>(); + KraftwerkExecutionContext kraftwerkExecutionContext = new KraftwerkExecutionContext(); @ParameterType("(?:[^,]*)(?:,\\s?[^,]*)*") public List listOfStrings(String arg){ @@ -83,7 +83,7 @@ public void readCampaignData() { CalculatedVariables calculatedVariables = LunaticReader.getCalculatedFromLunatic( new FileSystemImpl().readFile(Path.of(campaignPacks.get(campaignName).get(dataMode).get("lunatic")).toString())); DataProcessing calculatedProcessing = new CalculatedProcessing(vtlBindings,calculatedVariables, new FileSystemImpl()); - calculatedProcessing.applyVtlTransformations("TEST", null,errors); + calculatedProcessing.applyVtlTransformations("TEST", null, kraftwerkExecutionContext); // outDataset = vtlBindings.getDataset("TEST"); } diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java index a94f1200..1a188014 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java @@ -21,6 +21,7 @@ import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.SqlUtils; +import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionContext; import fr.insee.kraftwerk.core.vtl.VtlBindings; import io.cucumber.java.AfterAll; import io.cucumber.java.BeforeAll; @@ -175,7 +176,7 @@ public void unimodal_treatments() throws KraftwerkException, SQLException, NullE boolean withDDI = true; buildBindingsSequence.buildVtlBindings(userInputs, dataMode, vtlBindings, metadataModelMap.get(dataMode), withDDI, null); UnimodalSequence unimodal = new UnimodalSequence(); - unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, errors, metadataModelMap, new FileSystemImpl()); + unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, new KraftwerkExecutionContext(), metadataModelMap, new FileSystemImpl()); } } } @@ -184,7 +185,7 @@ public void unimodal_treatments() throws KraftwerkException, SQLException, NullE public void aggregate_datasets() throws SQLException { MultimodalSequence multimodalSequence = new MultimodalSequence(); try (Statement statement = database.createStatement()) { - multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModelMap, new FileSystemImpl()); + multimodalSequence.multimodalProcessing(userInputs, vtlBindings, new KraftwerkExecutionContext(), metadataModelMap, new FileSystemImpl()); } } @@ -193,7 +194,7 @@ public void export_results() throws KraftwerkException, SQLException { try (Statement statement = database.createStatement()) { WriterSequence writerSequence = new WriterSequence(); LocalDateTime localDateTime = LocalDateTime.now(); - writerSequence.writeOutputFiles(inDirectory, localDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModelMap, errors, null, statement, new FileSystemImpl()); + writerSequence.writeOutputFiles(inDirectory, localDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModelMap, null, statement, new FileSystemImpl()); writeErrorsFile(inDirectory, localDateTime, errors); outputFiles = new CsvOutputFiles(outDirectory, vtlBindings, userInputs.getModes(), statement, new FileSystemImpl()); }