diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d592072..fbe44696 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog -## 2.2.0 [TODO] - DuckDB implementation for output +## 3.0.0 [TODO] - Kubernetes support ### Added +- Launch treatment with command line +- S3/MinIO support : All file system calls goes through an interface with both OS file system and MinIO implementations + +- DuckDB implementation for output - Transfer Vtl datasets into DuckDB before output step - SQL util class for SQL operations @@ -10,7 +14,7 @@ ### Removed - Avro -- + ## 2.1.0 [2024-06-11] - Change Lunatic reader ### Changed - Refactor of Lunatic reader diff --git a/Dockerfile b/Dockerfile index cc12cfc5..673500f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ # Java 21 FROM gitlab-registry.insee.fr:443/kubernetes/images/run/jre:21.0.1_12-jre-jammy-rootless -COPY --chown=$JAVA_USER:$JAVA_USER Kraftwerk/kraftwerk-api/target/kraftwerk-api-*app-to-import.jar kraftwerk.jar +ARG VERSION_APPLICATION +COPY --chown=$JAVA_USER:$JAVA_USER Kraftwerk/kraftwerk-api/target/kraftwerk-api-$VERSION_APPLICATION.jar kraftwerk.jar EXPOSE 8080 +#Docker run without additionnal params to use REST API mode, add batch parameters to use batch mode ENTRYPOINT ["java","-jar","/kraftwerk.jar"] \ No newline at end of file diff --git a/README.md b/README.md index 4d2622b7..50b437fb 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,18 @@ User documentation and functional tests are still in an [inhouse project](https: Kraftwerk uses [Lombok](https://projectlombok.org/). +## Launch + +If no argument is specified in the `java -jar` command, Kraftwerk will launch +as a REST API. +Otherwise, it will launch on batch mode and apply treatments on one campaign +with the specified arguments. +The required arguments for batch mode are as follows (in order) : +1. Service to use (`MAIN`,`FILEBYFILE`,`GENESIS`,`LUNATIC_ONLY`) +2. Archive at end of execution (`false` or `true`) +3. Integrate all reporting datas (`false` or `true`) +4. Campaign name + :fr: Kraftwerk est une application Java Spring conçue pour valider et traiter des données provenant d'enquêtes multimodes, afin de générer des tableaux de données prêts à être utilisés à des fins statistiques. @@ -37,3 +49,14 @@ La documentation utilisateur et les tests fonctionnels sont encore dans un [proj * Maven 3.6 + Kraftwerk utilise [Lombok](https://projectlombok.org/). + +## Lancement + +Si aucun paramètre n'est spécifié dans la commande `java -jar`, Kraftwerk se lancera +en tant qu'API REST. +Sinon, il va se lancer en mode batch et appliquer les traitements sur une campagne +avec les paramètres spécifiés. Les paramètres requis pour le mode batch sont les suivants (dans l'ordre) : +1. Service à utiliser (`MAIN`,`FILEBYFILE`,`GENESIS`,`LUNATIC_ONLY`) +2. Archiver à la fin de l'exécution (`false` ou `true`) +3. Integrate all reporting datas (`false` ou `true`) +4. Nom de la campagne \ No newline at end of file diff --git a/kraftwerk-api/pom.xml b/kraftwerk-api/pom.xml index 120e3cf4..1dec62d3 100644 --- a/kraftwerk-api/pom.xml +++ b/kraftwerk-api/pom.xml @@ -71,6 +71,9 @@ org.springframework.boot spring-boot-maven-plugin + + JAR + diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/KraftwerkApi.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/KraftwerkApi.java index d52cce83..7df38182 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/KraftwerkApi.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/KraftwerkApi.java @@ -1,13 +1,16 @@ package fr.insee.kraftwerk.api; +import fr.insee.kraftwerk.api.configuration.MinioConfig; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.context.properties.ConfigurationPropertiesScan; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; @SpringBootApplication @ConfigurationPropertiesScan +@EnableConfigurationProperties(MinioConfig.class) public class KraftwerkApi extends SpringBootServletInitializer { diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/batch/KraftwerkBatch.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/batch/KraftwerkBatch.java new file mode 100644 index 00000000..cd199d59 --- /dev/null +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/batch/KraftwerkBatch.java @@ -0,0 +1,110 @@ +package fr.insee.kraftwerk.api.batch; + +import fr.insee.kraftwerk.api.configuration.ConfigProperties; +import fr.insee.kraftwerk.api.configuration.MinioConfig; +import fr.insee.kraftwerk.api.process.MainProcessing; +import fr.insee.kraftwerk.api.process.MainProcessingGenesis; +import fr.insee.kraftwerk.api.services.KraftwerkService; +import fr.insee.kraftwerk.core.utils.files.MinioImpl; +import io.minio.MinioClient; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class KraftwerkBatch implements CommandLineRunner { + + ConfigProperties configProperties; + MinioConfig minioConfig; + MinioClient minioClient; + + @Value("${fr.insee.postcollecte.files}") + protected String defaultDirectory; + + @Value("${fr.insee.postcollecte.size-limit}") + protected long limitSize; + + @Autowired + public KraftwerkBatch(ConfigProperties configProperties, MinioConfig minioConfig) { + this.configProperties = configProperties; + this.minioConfig = minioConfig; + if(minioConfig.isEnable()){ + minioClient = MinioClient.builder().endpoint(minioConfig.getEndpoint()).credentials(minioConfig.getAccessKey(), minioConfig.getSecretKey()).build(); + } + } + + @Override + public void run(String... args) { + try { + //If .jar launched with cli args + if (args.length > 0) { + log.info("Launching Kraftwerk in CLI mode..."); + + //Check arguments + checkArgs(args); + + //Parse arguments + //0. Service to use (MAIN,FILEBYFILE,GENESIS,LUNATIC_ONLY) + //1. Archive at end of execution (false or true) + //2. Integrate all reporting datas (false or true) + //3. Campaign name + KraftwerkServiceType kraftwerkServiceType = KraftwerkServiceType.valueOf(args[0]); + boolean archiveAtEnd = Boolean.parseBoolean(args[1]); + boolean withAllReportingData = Boolean.parseBoolean(args[2]); + String inDirectory = args[3]; + + //Kraftwerk service type related parameters + boolean fileByFile = kraftwerkServiceType == KraftwerkServiceType.FILE_BY_FILE; + boolean withDDI = kraftwerkServiceType != KraftwerkServiceType.LUNATIC_ONLY; + if (kraftwerkServiceType != KraftwerkServiceType.MAIN) { + withAllReportingData = false; + } + if (kraftwerkServiceType == KraftwerkServiceType.GENESIS) { + archiveAtEnd = false; + } + + + //Run kraftwerk + if (kraftwerkServiceType == KraftwerkServiceType.GENESIS) { + MainProcessingGenesis mainProcessingGenesis = new MainProcessingGenesis(configProperties, new MinioImpl(minioClient, minioConfig.getBucketName())); + mainProcessingGenesis.runMain(inDirectory); + } else { + MainProcessing mainProcessing = new MainProcessing(inDirectory, fileByFile, withAllReportingData, withDDI, defaultDirectory, limitSize, new MinioImpl(minioClient, minioConfig.getBucketName())); + mainProcessing.runMain(); + } + + //Archive + if (Boolean.TRUE.equals(archiveAtEnd)) { + KraftwerkService kraftwerkService = new KraftwerkService(minioConfig); + kraftwerkService.archive(inDirectory, new MinioImpl(minioClient, minioConfig.getBucketName())); + } + System.exit(0); + } + }catch(Exception e){ + log.error(e.toString()); + System.exit(1); + } + log.info("Launching Kraftwerk in API mode..."); + } + + /** + * Throws a IllegalArgumentException if the arguments are not valid (ex: unparseable boolean) + * KraftwerkServiceType is already checked by valueOf + * @param args list of CLI arguments + * @throws IllegalArgumentException if invalid argument + */ + private static void checkArgs(String[] args) throws IllegalArgumentException{ + if(args.length != 4) { + throw new IllegalArgumentException("Invalid number of arguments ! Got %s instead of 4 !".formatted(args.length)); + } + if(!args[1].equals("true") && !args[1].equals("false")){ + throw new IllegalArgumentException("Invalid archiveAtEnd boolean argument ! : %s".formatted(args[1])); + } + if(!args[2].equals("true") && !args[2].equals("false")){ + throw new IllegalArgumentException("Invalid withAllReportingData boolean argument ! %s".formatted(args[2])); + } + } +} diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/batch/KraftwerkServiceType.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/batch/KraftwerkServiceType.java new file mode 100644 index 00000000..031f6294 --- /dev/null +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/batch/KraftwerkServiceType.java @@ -0,0 +1,11 @@ +package fr.insee.kraftwerk.api.batch; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public enum KraftwerkServiceType { + MAIN, + LUNATIC_ONLY, + GENESIS, + FILE_BY_FILE; +} diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/configuration/MinioConfig.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/configuration/MinioConfig.java new file mode 100644 index 00000000..4f20355a --- /dev/null +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/configuration/MinioConfig.java @@ -0,0 +1,24 @@ +package fr.insee.kraftwerk.api.configuration; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties +@Getter +public class MinioConfig { + @Value("${fr.insee.postcollecte.minio.endpoint}") + private String endpoint; + + @Value("${fr.insee.postcollecte.minio.access_key}") + private String accessKey; + + @Value("${fr.insee.postcollecte.minio.secret_key}") + private String secretKey; + + @Value("${fr.insee.postcollecte.minio.enable}") + private boolean enable; + + @Value("${fr.insee.postcollecte.minio.bucket_name}") + private String bucketName; +} diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java index afbba818..b96438bb 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java @@ -7,16 +7,14 @@ import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.MetadataUtils; import fr.insee.kraftwerk.core.sequence.*; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.Getter; import lombok.extern.log4j.Log4j2; -import org.apache.commons.io.FileUtils; -import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.sql.Connection; import java.sql.SQLException; @@ -25,7 +23,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Stream; @Log4j2 public class MainProcessing { @@ -49,6 +46,7 @@ public class MainProcessing { private KraftwerkExecutionLog kraftwerkExecutionLog; private final List errors = new ArrayList<>(); private LocalDateTime executionDateTime; + private final FileUtilsInterface fileUtilsInterface; /** @@ -59,24 +57,26 @@ public class MainProcessing { private final long limitSize; - public MainProcessing(String inDirectoryParam, boolean fileByFile,boolean withAllReportingData,boolean withDDI, String defaultDirectory, long limitSize) { + public MainProcessing(String inDirectoryParam, boolean fileByFile,boolean withAllReportingData,boolean withDDI, String defaultDirectory, long limitSize, FileUtilsInterface fileUtilsInterface) { super(); this.inDirectoryParam = inDirectoryParam; this.fileByFile = fileByFile; this.withAllReportingData = withAllReportingData; this.withDDI=withDDI; this.limitSize = limitSize; - controlInputSequence = new ControlInputSequence(defaultDirectory); + controlInputSequence = new ControlInputSequence(defaultDirectory, fileUtilsInterface); + this.fileUtilsInterface = fileUtilsInterface; } - public MainProcessing(String inDirectoryParam, boolean fileByFile, String defaultDirectory, long limitSize) { + public MainProcessing(String inDirectoryParam, boolean fileByFile, String defaultDirectory, long limitSize, FileUtilsInterface fileUtilsInterface) { super(); this.inDirectoryParam = inDirectoryParam; this.fileByFile = fileByFile; this.withAllReportingData = !fileByFile; this.withDDI=true; this.limitSize = limitSize; - controlInputSequence = new ControlInputSequence(defaultDirectory); + controlInputSequence = new ControlInputSequence(defaultDirectory, fileUtilsInterface); + this.fileUtilsInterface = fileUtilsInterface; } @@ -116,9 +116,9 @@ public void init() throws KraftwerkException { String campaignName = inDirectory.getFileName().toString(); log.info("Kraftwerk main service started for campaign: " + campaignName); - userInputsFile = controlInputSequence.getUserInputs(inDirectory); + userInputsFile = controlInputSequence.getUserInputs(inDirectory, fileUtilsInterface); - metadataModels = withDDI ? MetadataUtils.getMetadata(userInputsFile.getModeInputsMap()) : MetadataUtils.getMetadataFromLunatic(userInputsFile.getModeInputsMap()); + metadataModels = withDDI ? MetadataUtils.getMetadata(userInputsFile.getModeInputsMap(), fileUtilsInterface) : MetadataUtils.getMetadataFromLunatic(userInputsFile.getModeInputsMap(), fileUtilsInterface); userInputsFileList = getUserInputsFile(userInputsFile, fileByFile); @@ -139,19 +139,19 @@ public void init() throws KraftwerkException { /* Step 2 : unimodal data */ private void unimodalProcess() throws KraftwerkException { - BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(withAllReportingData); + BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(withAllReportingData, fileUtilsInterface); for (String dataMode : userInputsFile.getModeInputsMap().keySet()) { MetadataModel metadataForMode = metadataModels.get(dataMode); buildBindingsSequence.buildVtlBindings(userInputsFile, dataMode, vtlBindings, metadataForMode, withDDI, kraftwerkExecutionLog); UnimodalSequence unimodal = new UnimodalSequence(); - unimodal.applyUnimodalSequence(userInputsFile, dataMode, vtlBindings, errors, metadataModels); + unimodal.applyUnimodalSequence(userInputsFile, dataMode, vtlBindings, errors, metadataModels, fileUtilsInterface); } } /* Step 3 : multimodal VTL data processing */ private void multimodalProcess(){ MultimodalSequence multimodalSequence = new MultimodalSequence(); - multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModels); + multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModels, fileUtilsInterface); } /* Step 4 : Insert into SQL database */ @@ -163,17 +163,17 @@ private void insertDatabase(Statement database) throws SQLException { /* Step 5 : Write output files */ private void outputFileWriter(Statement database) throws KraftwerkException { WriterSequence writerSequence = new WriterSequence(); - writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModels, errors, kraftwerkExecutionLog,database); + writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModels, errors, kraftwerkExecutionLog, database, fileUtilsInterface); } /* Step 5 : Write errors */ private void writeErrors() { - TextFileWriter.writeErrorsFile(inDirectory, executionDateTime, errors); + TextFileWriter.writeErrorsFile(inDirectory, executionDateTime, errors, fileUtilsInterface); } /* Step 6 : Write log */ - private void writeLog() {TextFileWriter.writeLogFile(inDirectory, executionDateTime, kraftwerkExecutionLog);} + private void writeLog() {TextFileWriter.writeLogFile(inDirectory, executionDateTime, kraftwerkExecutionLog, fileUtilsInterface);} private static List getUserInputsFile(UserInputsFile source, boolean fileByFile) throws KraftwerkException { List userInputsFileList = new ArrayList<>(); @@ -181,7 +181,7 @@ private static List getUserInputsFile(UserInputsFile source, boo for (String dataMode : source.getModeInputsMap().keySet()) { List dataFiles = getFilesToProcess(source, dataMode); for (Path dataFile : dataFiles) { - UserInputsFile currentFileInputs = new UserInputsFile(source.getUserInputFile(), source.getUserInputFile().getParent()); + UserInputsFile currentFileInputs = new UserInputsFile(source.getUserInputFile(), source.getUserInputFile().getParent(), source.getFileUtilsInterface()); currentFileInputs.setVtlReconciliationFile(source.getVtlReconciliationFile()); currentFileInputs.setVtlInformationLevelsFile(source.getVtlInformationLevelsFile()); currentFileInputs.setVtlTransformationsFile(source.getVtlTransformationsFile()); @@ -214,26 +214,25 @@ private static List getFilesToProcess(UserInputsFile userInputsFile, Strin log.error("Datapath is null"); return files; } - if (Files.isRegularFile(dataPath)) { - files.add(dataPath); - } else if (Files.isDirectory(dataPath)) { - try (Stream stream = Files.list(dataPath)) { - stream.forEach(files::add); - } catch (IOException e) { - log.error(String.format("IOException occurred when trying to list data files of folder: %s", - dataPath)); - } - } else { + if(userInputsFile.getFileUtilsInterface().isDirectory(dataPath.toString()) == null){ log.warn(String.format("Data path given could not be identified as a file or folder: %s", dataPath)); + return files; + } + if(Boolean.TRUE.equals(userInputsFile.getFileUtilsInterface().isDirectory(dataPath.toString()))){ + for(String path : userInputsFile.getFileUtilsInterface().listFilePaths(dataPath.toString())){ + files.add(Path.of(path)); + } + }else{ + files.add(dataPath); } return files; } private void isDataTooBig(UserInputsFile userInputsFile, String errorMessage, long limitSize) throws KraftwerkException { for (String dataMode : userInputsFile.getModeInputsMap().keySet()){ - long dataSize = FileUtils.sizeOf(userInputsFile.getModeInputs(dataMode).getDataFile().toFile()); + long dataSize = userInputsFile.getFileUtilsInterface().getSizeOf(userInputsFile.getModeInputs(dataMode).getDataFile().toString()); if (dataSize > limitSize) { - log.error("Size of data folder/file {} : {}",userInputsFile.getModeInputs(dataMode).getDataFile(), FileUtils.byteCountToDisplaySize(dataSize)); + log.error("Size of data folder/file {} : {}",userInputsFile.getModeInputs(dataMode).getDataFile(), dataSize); throw new KraftwerkException(413,errorMessage); } } diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java index 40561aa1..1410032c 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java @@ -15,6 +15,7 @@ import fr.insee.kraftwerk.core.sequence.MultimodalSequence; import fr.insee.kraftwerk.core.sequence.UnimodalSequence; import fr.insee.kraftwerk.core.sequence.WriterSequence; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.vtl.VtlBindings; @@ -23,7 +24,6 @@ import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.ListUtils; import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.stereotype.Component; import java.io.IOException; import java.nio.file.Path; @@ -36,7 +36,6 @@ import java.util.Map; @Log4j2 -@Component public class MainProcessingGenesis { @Setter @@ -47,6 +46,7 @@ public class MainProcessingGenesis { @Getter private UserInputsGenesis userInputs; private LocalDateTime executionDateTime; + private final FileUtilsInterface fileUtilsInterface; private Statement database; /* SPECIFIC VARIABLES */ @@ -60,19 +60,21 @@ public class MainProcessingGenesis { private final GenesisClient client; - public MainProcessingGenesis(ConfigProperties config) { + public MainProcessingGenesis(ConfigProperties config, FileUtilsInterface fileUtilsInterface) { this.client = new GenesisClient(new RestTemplateBuilder(), config); + this.fileUtilsInterface = fileUtilsInterface; } - public void init(String idCampaign) throws KraftwerkException, IOException { + public void init(String idCampaign) throws KraftwerkException { log.info("Kraftwerk main service started for campaign: " + idCampaign); + this.controlInputSequenceGenesis = new ControlInputSequenceGenesis("", fileUtilsInterface); this.executionDateTime = LocalDateTime.now(); inDirectory = controlInputSequenceGenesis.getInDirectory(idCampaign); //First we check the modes present in database for the given questionnaire //We build userInputs for the given questionnaire - userInputs = new UserInputsGenesis(controlInputSequenceGenesis.isHasConfigFile(), inDirectory, client.getModes(idCampaign)); + userInputs = new UserInputsGenesis(controlInputSequenceGenesis.isHasConfigFile(), inDirectory, client.getModes(idCampaign), fileUtilsInterface); if (!userInputs.getModes().isEmpty()) { - metadataModels = MetadataUtilsGenesis.getMetadata(userInputs.getModeInputsMap()); + metadataModels = MetadataUtilsGenesis.getMetadata(userInputs.getModeInputsMap(), fileUtilsInterface); } else { log.error("No source found for campaign " + idCampaign); } @@ -110,18 +112,18 @@ public void runMain(String idCampaign) throws KraftwerkException, IOException { } private void unimodalProcess(List suLatest) throws KraftwerkException { - BuildBindingsSequenceGenesis buildBindingsSequenceGenesis = new BuildBindingsSequenceGenesis(); + BuildBindingsSequenceGenesis buildBindingsSequenceGenesis = new BuildBindingsSequenceGenesis(fileUtilsInterface); for (String dataMode : userInputs.getModeInputsMap().keySet()) { buildBindingsSequenceGenesis.buildVtlBindings(dataMode, vtlBindings, metadataModels, suLatest, inDirectory); UnimodalSequence unimodal = new UnimodalSequence(); - unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, errors, metadataModels); + unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, errors, metadataModels, fileUtilsInterface); } } /* Step 3 : multimodal VTL data processing */ private void multimodalProcess() { MultimodalSequence multimodalSequence = new MultimodalSequence(); - multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModels); + multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModels, fileUtilsInterface); } /* Step 4 : Insert into SQL database */ @@ -133,12 +135,12 @@ private void insertDatabase(){ /* Step 5 : Write output files */ private void outputFileWriter() throws KraftwerkException { WriterSequence writerSequence = new WriterSequence(); - writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModels, errors, null, database); + writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModels, errors, null, database, fileUtilsInterface); } /* Step 6 : Write errors */ private void writeErrors() { - TextFileWriter.writeErrorsFile(inDirectory, executionDateTime, errors); + TextFileWriter.writeErrorsFile(inDirectory, executionDateTime, errors, fileUtilsInterface); } } diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/HealthcheckService.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/HealthcheckService.java index 3dc61a99..9ea54933 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/HealthcheckService.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/HealthcheckService.java @@ -32,7 +32,8 @@ public class HealthcheckService extends KraftwerkService { @Autowired public HealthcheckService(ConfigProperties configProperties) { - this.configProperties = configProperties; + super(null); + this.configProperties = configProperties; this.client = new GenesisClient(new RestTemplateBuilder(), configProperties); } diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/KraftwerkService.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/KraftwerkService.java index d0632af9..dd1a39b6 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/KraftwerkService.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/KraftwerkService.java @@ -1,14 +1,19 @@ package fr.insee.kraftwerk.api.services; +import fr.insee.kraftwerk.api.configuration.MinioConfig; import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.sequence.ControlInputSequence; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.files.MinioImpl; +import io.minio.MinioClient; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import jakarta.annotation.PostConstruct; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; @@ -37,17 +42,32 @@ public class KraftwerkService { @Value("${fr.insee.postcollecte.size-limit}") protected long limitSize; + MinioConfig minioConfig; + + protected ControlInputSequence controlInputSequence ; + + @Autowired + public KraftwerkService(MinioConfig minioConfig){ + this.minioConfig = minioConfig; + } @PostConstruct public void initializeWithProperties() { + FileUtilsInterface fileUtilsInterface; + if(minioConfig.isEnable()){ + MinioClient minioClient = MinioClient.builder().endpoint(minioConfig.getEndpoint()).credentials(minioConfig.getAccessKey(), minioConfig.getSecretKey()).build(); + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } if (StringUtils.isNotEmpty(csvOutputsQuoteChar)) { Constants.setCsvOutputQuoteChar(csvOutputsQuoteChar.trim().charAt(0)); } - controlInputSequence = new ControlInputSequence(defaultDirectory); + controlInputSequence = new ControlInputSequence(defaultDirectory, fileUtilsInterface); } - protected ResponseEntity archive(String inDirectoryParam) { + public ResponseEntity archive(String inDirectoryParam, FileUtilsInterface fileUtilsInterface) { Path inDirectory; try { inDirectory = controlInputSequence.getInDirectory(inDirectoryParam); @@ -56,19 +76,19 @@ protected ResponseEntity archive(String inDirectoryParam) { } /* Step 4.3 : move kraftwerk.json to a secondary folder */ - FileUtils.renameInputFile(inDirectory); + fileUtilsInterface.renameInputFile(inDirectory); /* Step 4.4 : move differential data to a secondary folder */ try { - FileUtils.archiveInputFiles(controlInputSequence.getUserInputs(inDirectory)); + fileUtilsInterface.archiveInputFiles(controlInputSequence.getUserInputs(inDirectory, fileUtilsInterface)); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } //delete temp directory - Path tempOutputPath = FileUtils.transformToTemp(inDirectory); + Path tempOutputPath = FileUtilsInterface.transformToTemp(inDirectory); try { - FileUtils.deleteDirectory(tempOutputPath); + fileUtilsInterface.deleteDirectory(tempOutputPath); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/MainService.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/MainService.java index 07569e29..c287520a 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/MainService.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/MainService.java @@ -2,13 +2,19 @@ import fr.insee.kraftwerk.api.configuration.ConfigProperties; +import fr.insee.kraftwerk.api.configuration.MinioConfig; import fr.insee.kraftwerk.api.process.MainProcessing; import fr.insee.kraftwerk.api.process.MainProcessingGenesis; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.sequence.ControlInputSequenceGenesis; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.files.MinioImpl; +import io.minio.MinioClient; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -21,14 +27,28 @@ @RestController +@Slf4j @Tag(name = "${tag.main}") public class MainService extends KraftwerkService { ConfigProperties configProperties; + MinioClient minioClient; + boolean useMinio; + @Autowired - public MainService(ConfigProperties configProperties) { - this.configProperties = configProperties; + public MainService(ConfigProperties configProperties, MinioConfig minioConfig) { + super(minioConfig); + this.configProperties = configProperties; + this.minioConfig = minioConfig; + useMinio = false; + if(minioConfig == null){ + log.warn("Minio config null !"); + } + if(minioConfig != null && minioConfig.isEnable()){ + minioClient = MinioClient.builder().endpoint(minioConfig.getEndpoint()).credentials(minioConfig.getAccessKey(), minioConfig.getSecretKey()).build(); + useMinio = true; + } } @PutMapping(value = "/main") @@ -40,7 +60,14 @@ public ResponseEntity mainService( ) { boolean fileByFile = false; boolean withDDI = true; - MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize); + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + + MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize, fileUtilsInterface); try { mp.runMain(); } catch (KraftwerkException e) { @@ -48,7 +75,7 @@ public ResponseEntity mainService( } /* Step 4.3- 4.4 : Archive */ - if (Boolean.TRUE.equals(archiveAtEnd)) archive(inDirectoryParam); + if (Boolean.TRUE.equals(archiveAtEnd)) archive(inDirectoryParam,fileUtilsInterface); return ResponseEntity.ok(inDirectoryParam); } @@ -62,14 +89,21 @@ public ResponseEntity mainFileByFile( boolean fileByFile = true; boolean withAllReportingData = false; boolean withDDI = true; - MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize); + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + + MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize, fileUtilsInterface); try { mp.runMain(); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } /* Step 4.3- 4.4 : Archive */ - if (Boolean.TRUE.equals(archiveAtEnd)) archive(inDirectoryParam); + if (Boolean.TRUE.equals(archiveAtEnd)) archive(inDirectoryParam, fileUtilsInterface); return ResponseEntity.ok(inDirectoryParam); } @@ -83,14 +117,21 @@ public ResponseEntity mainLunaticOnly( boolean withDDI = false; boolean fileByFile = false; boolean withAllReportingData = false; - MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize); + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + + MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize, fileUtilsInterface); try { mp.runMain(); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } /* Step 4.3- 4.4 : Archive */ - if (Boolean.TRUE.equals(archiveAtEnd)) archive(inDirectoryParam); + if (Boolean.TRUE.equals(archiveAtEnd)) archive(inDirectoryParam, fileUtilsInterface); return ResponseEntity.ok(inDirectoryParam); } @@ -99,9 +140,16 @@ public ResponseEntity mainLunaticOnly( @Operation(operationId = "mainGenesis", summary = "${summary.mainGenesis}", description = "${description.mainGenesis}") public ResponseEntity mainGenesis( @Parameter(description = "${param.idCampaign}", required = true, example = INDIRECTORY_EXAMPLE) @RequestBody String idCampaign) { - MainProcessingGenesis mpGenesis = new MainProcessingGenesis(configProperties); + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + + MainProcessingGenesis mpGenesis = new MainProcessingGenesis(configProperties, fileUtilsInterface); + try { - mpGenesis.setControlInputSequenceGenesis(new ControlInputSequenceGenesis(defaultDirectory)); mpGenesis.runMain(idCampaign); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/SplitterService.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/SplitterService.java index ea4ff382..c8fdbca8 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/SplitterService.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/SplitterService.java @@ -1,9 +1,16 @@ package fr.insee.kraftwerk.api.services; -import fr.insee.kraftwerk.core.utils.XMLSplitter; +import fr.insee.kraftwerk.api.configuration.MinioConfig; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileSystemType; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.files.MinioImpl; +import fr.insee.kraftwerk.core.utils.xml.XmlSplitter; +import io.minio.MinioClient; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PutMapping; @@ -15,15 +22,27 @@ @Log4j2 public class SplitterService extends KraftwerkService{ + @Autowired + public SplitterService(MinioConfig minioConfig) { + super(minioConfig); + } + @Operation(summary = "Split a XML file into smaller ones") @PutMapping(path = "/split/lunatic-xml") public ResponseEntity saveResponsesFromXmlFile(@RequestParam("inputFolder") String inputFolder, @RequestParam("outputFolder") String outputFolder, @RequestParam("filename") String filename, - @RequestParam("nbResponsesByFile") int nbSU) + @RequestParam("nbResponsesByFile") int nbSU, + @RequestParam("fileSystemType") FileSystemType fileSystemType) throws Exception { - log.info("Split XML file : " + filename + " into " + nbSU + " SU by file"); - XMLSplitter.split(String.format("%s/in/%s/",defaultDirectory,inputFolder), filename, String.format("%s/in/%s/",defaultDirectory,outputFolder), "SurveyUnit", nbSU); + log.info("Split XML file : " + filename + " into " + nbSU + " SU by file using " + + (fileSystemType.equals(FileSystemType.MINIO) ? "Minio" : "OS file system")); + + FileUtilsInterface fileUtilsInterface = fileSystemType.equals(FileSystemType.MINIO) ? + new MinioImpl(MinioClient.builder().credentials(minioConfig.getAccessKey(),minioConfig.getSecretKey()).endpoint(minioConfig.getEndpoint()).build(), minioConfig.getBucketName()) : + new FileSystemImpl(); + + XmlSplitter.split(String.format("%s/in/%s/",defaultDirectory,inputFolder), filename, String.format("%s/in/%s/",defaultDirectory,outputFolder), "SurveyUnit", nbSU, fileUtilsInterface); return new ResponseEntity<>("File split", HttpStatus.OK); } diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java index 1d25c730..be3bfce1 100644 --- a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java +++ b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java @@ -1,5 +1,6 @@ package fr.insee.kraftwerk.api.services; +import fr.insee.kraftwerk.api.configuration.MinioConfig; import fr.insee.kraftwerk.api.process.MainProcessing; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.dataprocessing.StepEnum; @@ -8,13 +9,18 @@ import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.MetadataUtils; import fr.insee.kraftwerk.core.sequence.*; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.files.MinioImpl; import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.vtl.VtlBindings; +import io.minio.MinioClient; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; @@ -22,6 +28,7 @@ import java.nio.file.Path; import java.sql.Connection; import java.sql.SQLException; +import java.sql.Statement; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -31,7 +38,23 @@ @RestController @Tag(name = "${tag.stepbystep}") +@Slf4j public class StepByStepService extends KraftwerkService { + MinioClient minioClient; + boolean useMinio; + + @Autowired + public StepByStepService(MinioConfig minioConfig) { + super(minioConfig); + useMinio = false; + if(minioConfig == null){ + log.warn("Minio config null !"); + } + if(minioConfig != null && minioConfig.isEnable()){ + minioClient = MinioClient.builder().endpoint(minioConfig.getEndpoint()).credentials(minioConfig.getAccessKey(), minioConfig.getSecretKey()).build(); + useMinio = true; + } + } @PutMapping(value = "/buildVtlBindings") @Operation(operationId = "buildVtlBindings", summary = "${summary.buildVtlBindings}", description = "${description.buildVtlBindings}") @@ -42,7 +65,14 @@ public ResponseEntity buildVtlBindings( //Read data files boolean fileByFile = false; boolean withDDI = true; - MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize); + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + + MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize, fileUtilsInterface); try { mp.init(); } catch (KraftwerkException e) { @@ -50,8 +80,8 @@ public ResponseEntity buildVtlBindings( } //Process - BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(withAllReportingData); - VtlReaderWriterSequence vtlWriterSequence = new VtlReaderWriterSequence(); + BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(withAllReportingData, fileUtilsInterface); + VtlReaderWriterSequence vtlWriterSequence = new VtlReaderWriterSequence(fileUtilsInterface); for (String dataMode : mp.getUserInputsFile().getModeInputsMap().keySet()) { try{ @@ -79,7 +109,14 @@ public ResponseEntity buildVtlBindingsByDataMode( //Read data files boolean fileByFile = false; boolean withDDI = true; - MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize); + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + + MainProcessing mp = new MainProcessing(inDirectoryParam, fileByFile,withAllReportingData,withDDI, defaultDirectory, limitSize, fileUtilsInterface); try { mp.init(); } catch (KraftwerkException e) { @@ -87,14 +124,14 @@ public ResponseEntity buildVtlBindingsByDataMode( } //Process - BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(withAllReportingData); + BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(withAllReportingData, fileUtilsInterface); try{ buildBindingsSequence.buildVtlBindings(mp.getUserInputsFile(), dataMode, mp.getVtlBindings(), mp.getMetadataModels().get(dataMode), withDDI, null); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } - VtlReaderWriterSequence vtlWriterSequence = new VtlReaderWriterSequence(); + VtlReaderWriterSequence vtlWriterSequence = new VtlReaderWriterSequence(fileUtilsInterface); vtlWriterSequence.writeTempBindings(mp.getInDirectory(), dataMode, mp.getVtlBindings(), StepEnum.BUILD_BINDINGS); return ResponseEntity.ok(inDirectoryParam+ " - "+dataMode); @@ -109,6 +146,13 @@ public ResponseEntity unimodalProcessing( @Parameter(description = "${param.inDirectory}", required = true, example = INDIRECTORY_EXAMPLE) @RequestBody String inDirectoryParam, @Parameter(description = "${param.dataMode}", required = true) @RequestParam String dataMode ) { + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + //Read data in JSON file Path inDirectory; try { @@ -118,26 +162,26 @@ public ResponseEntity unimodalProcessing( } UserInputsFile userInputsFile; try { - userInputsFile = controlInputSequence.getUserInputs(inDirectory); + userInputsFile = controlInputSequence.getUserInputs(inDirectory, fileUtilsInterface); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } VtlBindings vtlBindings = new VtlBindings(); List errors = new ArrayList<>(); - VtlReaderWriterSequence vtlReaderSequence = new VtlReaderWriterSequence(); - vtlReaderSequence.readDataset(FileUtils.transformToTemp(inDirectory).toString(),dataMode, StepEnum.BUILD_BINDINGS, vtlBindings); + VtlReaderWriterSequence vtlReaderSequence = new VtlReaderWriterSequence(fileUtilsInterface); + vtlReaderSequence.readDataset(FileUtilsInterface.transformToTemp(inDirectory).toString(),dataMode, StepEnum.BUILD_BINDINGS, vtlBindings); - Map metadataModelMap = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap()); + Map metadataModelMap = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap(), fileUtilsInterface); //Process UnimodalSequence unimodal = new UnimodalSequence(); - unimodal.applyUnimodalSequence(userInputsFile, dataMode, vtlBindings, errors, metadataModelMap); + unimodal.applyUnimodalSequence(userInputsFile, dataMode, vtlBindings, errors, metadataModelMap, fileUtilsInterface); //Write technical outputs - VtlReaderWriterSequence vtlWriterSequence = new VtlReaderWriterSequence(); + VtlReaderWriterSequence vtlWriterSequence = new VtlReaderWriterSequence(fileUtilsInterface); vtlWriterSequence.writeTempBindings(inDirectory, dataMode, vtlBindings, StepEnum.UNIMODAL_PROCESSING); - TextFileWriter.writeErrorsFile(inDirectory, LocalDateTime.now(), errors); + TextFileWriter.writeErrorsFile(inDirectory, LocalDateTime.now(), errors, fileUtilsInterface); return ResponseEntity.ok(inDirectoryParam+ " - "+dataMode); @@ -150,6 +194,13 @@ public ResponseEntity unimodalProcessing( public ResponseEntity multimodalProcessing( @Parameter(description = "${param.inDirectory}", required = true, example = INDIRECTORY_EXAMPLE) @RequestBody String inDirectoryParam ) { + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + //Read data in JSON file Path inDirectory; try { @@ -159,32 +210,32 @@ public ResponseEntity multimodalProcessing( } UserInputsFile userInputsFile; try { - userInputsFile = controlInputSequence.getUserInputs(inDirectory); + userInputsFile = controlInputSequence.getUserInputs(inDirectory, fileUtilsInterface); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } List errors = new ArrayList<>(); - VtlReaderWriterSequence vtlReaderWriterSequence = new VtlReaderWriterSequence(); + VtlReaderWriterSequence vtlReaderWriterSequence = new VtlReaderWriterSequence(fileUtilsInterface); //Test VtlBindings vtlBindings = new VtlBindings(); for (String dataMode : userInputsFile.getModeInputsMap().keySet()) { - vtlReaderWriterSequence.readDataset(FileUtils.transformToTemp(inDirectory).toString(),dataMode, StepEnum.UNIMODAL_PROCESSING, vtlBindings); + vtlReaderWriterSequence.readDataset(FileUtilsInterface.transformToTemp(inDirectory).toString(),dataMode, StepEnum.UNIMODAL_PROCESSING, vtlBindings); } - Map metadataModelMap = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap()); + Map metadataModelMap = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap(), fileUtilsInterface); //Process MultimodalSequence multimodalSequence = new MultimodalSequence(); - multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModelMap); + multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModelMap, fileUtilsInterface); //Write technical fils for (String datasetName : vtlBindings.getDatasetNames()) { vtlReaderWriterSequence.writeTempBindings(inDirectory, datasetName, vtlBindings, StepEnum.MULTIMODAL_PROCESSING); } - TextFileWriter.writeErrorsFile(inDirectory, LocalDateTime.now(), errors); + TextFileWriter.writeErrorsFile(inDirectory, LocalDateTime.now(), errors, fileUtilsInterface); return ResponseEntity.ok(inDirectoryParam); @@ -196,6 +247,13 @@ public ResponseEntity multimodalProcessing( public ResponseEntity writeOutputFiles( @Parameter(description = "${param.inDirectory}", required = true, example = INDIRECTORY_EXAMPLE) @RequestBody String inDirectoryParam ) throws KraftwerkException, SQLException { + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + Path inDirectory; try { inDirectory = controlInputSequence.getInDirectory(inDirectoryParam); @@ -206,43 +264,42 @@ public ResponseEntity writeOutputFiles( VtlBindings vtlBindings = new VtlBindings(); List errors = new ArrayList<>(); // Read all bindings necessary to produce output - String path = FileUtils.transformToTemp(inDirectory).toString(); - List fileNames = FileUtils.listFiles(path); + String path = FileUtilsInterface.transformToTemp(inDirectory).toString(); + List fileNames = fileUtilsInterface.listFileNames(path); fileNames = fileNames.stream().filter(name -> name.endsWith(StepEnum.MULTIMODAL_PROCESSING.getStepLabel()+JSON)).toList(); for (String name : fileNames){ String pathBindings = path + File.separator + name; String bindingName = name.substring(0, name.indexOf("_"+StepEnum.MULTIMODAL_PROCESSING.getStepLabel())); - VtlReaderWriterSequence vtlReaderSequence = new VtlReaderWriterSequence(); + VtlReaderWriterSequence vtlReaderSequence = new VtlReaderWriterSequence(fileUtilsInterface); vtlReaderSequence.readDataset(pathBindings, bindingName, vtlBindings); } WriterSequence writerSequence = new WriterSequence(); UserInputsFile userInputsFile; try { - userInputsFile = controlInputSequence.getUserInputs(inDirectory); + userInputsFile = controlInputSequence.getUserInputs(inDirectory, fileUtilsInterface); } catch (KraftwerkException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } - Map metadataModelMap = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap()); - try (Connection database = SqlUtils.openConnection()) { - writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModelMap, errors, null,database.createStatement()); + Map metadataModelMap = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap(), fileUtilsInterface); + try (Statement database = SqlUtils.openConnection().createStatement()) { + writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModelMap, errors, null, database, fileUtilsInterface); } return ResponseEntity.ok(inDirectoryParam); } - @PutMapping(value = "/archive") @Operation(operationId = "archive", summary = "${summary.archive}", description = "${description.archive}") public ResponseEntity archiveService( - @Parameter(description = "${param.inDirectory}", required = true, example = INDIRECTORY_EXAMPLE) @RequestBody String inDirectoryParam) - { - return archive(inDirectoryParam); + @Parameter(description = "${param.inDirectory}", required = true, example = INDIRECTORY_EXAMPLE) @RequestBody String inDirectoryParam) { + FileUtilsInterface fileUtilsInterface; + if(Boolean.TRUE.equals(useMinio)){ + fileUtilsInterface = new MinioImpl(minioClient, minioConfig.getBucketName()); + }else{ + fileUtilsInterface = new FileSystemImpl(); + } + return archive(inDirectoryParam, fileUtilsInterface); } - - - - - } \ No newline at end of file diff --git a/kraftwerk-api/src/main/resources/application.properties b/kraftwerk-api/src/main/resources/application.properties index 0185af9a..658a1a3d 100644 --- a/kraftwerk-api/src/main/resources/application.properties +++ b/kraftwerk-api/src/main/resources/application.properties @@ -20,3 +20,13 @@ fr.insee.postcollecte.csv.output.quote =" fr.insee.postcollecte.size-limit = 419430400 +#-------------------------------------------------------------------------- +# Actuator +# There are several ways to configure it : +# https://docs.spring.io/spring-boot/docs/3.0.0/reference/htmlsingle/#actuator.endpoints.exposing +#-------------------------------------------------------------------------- +management.health.probes.enabled=true +management.endpoint.health.show-details=always +#Global deactivation and activation endpoint by endpoint +#management.endpoints.enabled-by-default=false +#management.endpoint.info.enabled=true \ No newline at end of file diff --git a/kraftwerk-core/pom.xml b/kraftwerk-core/pom.xml index 6f1cd6f1..8264003f 100644 --- a/kraftwerk-core/pom.xml +++ b/kraftwerk-core/pom.xml @@ -98,6 +98,13 @@ 4.5.0-M2 + + + io.minio + minio + 8.5.10 + + @@ -106,5 +113,4 @@ ${duckdb.version} - \ No newline at end of file diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java index 10d9067d..ab3db1a3 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java @@ -1,18 +1,20 @@ package fr.insee.kraftwerk.core; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.Getter; import lombok.extern.log4j.Log4j2; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import java.io.File; -import java.io.FileReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.TimeUnit; @@ -61,6 +63,8 @@ private Constants() {} public static final String DUCKDB_URL = "jdbc:duckdb:"; public static final int DB_CONNECTION_TRY_COUNT = 10; + public static final String DDI_FILE_REGEX = "ddi[\\w,\\s-]+\\.xml"; + public static final String LUNATIC_FILE_REGEX = "lunatic[\\w,\\s-]+\\.json"; // ----- Explicit Variables Names @@ -206,10 +210,11 @@ public static File getFileFromPath(String path) { * @throws ParseException * @throws IOException */ - public static Object readJsonSimple(Path filename) throws IOException, ParseException { - FileReader reader = new FileReader(filename.toString()); - JSONParser jsonParser = new JSONParser(); - return jsonParser.parse(reader); + public static Object readJsonSimple(Path filename, FileUtilsInterface fileUtilsInterface) throws IOException, ParseException { + try(InputStream inputStream = fileUtilsInterface.readFile(filename.toString())){ + JSONParser jsonParser = new JSONParser(); + return jsonParser.parse(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + } } // ---------- Maths function diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/data/model/ExternalVariable.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/data/model/ExternalVariable.java index 23f2b5cd..85f0afd9 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/data/model/ExternalVariable.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/data/model/ExternalVariable.java @@ -1,13 +1,16 @@ package fr.insee.kraftwerk.core.data.model; import lombok.Getter; +import lombok.Setter; import java.util.List; public class ExternalVariable { @Getter + @Setter private String idVar; @Getter + @Setter private List values; } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java index 0eefff72..f6046afe 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessing.java @@ -2,6 +2,7 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.metadata.CalculatedVariables; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; import lombok.extern.log4j.Log4j2; @@ -18,8 +19,8 @@ public class CalculatedProcessing extends DataProcessing { public static final int MAXIMAL_RESOLVING_ITERATIONS = 100; private CalculatedVariables calculatedVariables; - public CalculatedProcessing(VtlBindings vtlBindings, CalculatedVariables calculatedVariables) { - super(vtlBindings); + public CalculatedProcessing(VtlBindings vtlBindings, CalculatedVariables calculatedVariables, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, fileUtilsInterface); this.calculatedVariables = calculatedVariables; } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java index bdb06e41..1317e6d7 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessing.java @@ -4,6 +4,7 @@ import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.PaperUcq; import fr.insee.kraftwerk.core.metadata.VariablesMap; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; import lombok.extern.log4j.Log4j2; @@ -24,8 +25,8 @@ public class CleanUpProcessing extends DataProcessing { private Map metadataModels; - public CleanUpProcessing(VtlBindings vtlBindings, Map metadataModels) { - super(vtlBindings); + public CleanUpProcessing(VtlBindings vtlBindings, Map metadataModels, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, fileUtilsInterface); this.metadataModels=metadataModels; } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessing.java index 35e5b28a..27f58748 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessing.java @@ -2,6 +2,7 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.utils.TextFileReader; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import fr.insee.kraftwerk.core.vtl.VtlScript; @@ -25,9 +26,12 @@ public abstract class DataProcessing { protected final VtlBindings vtlBindings; VtlExecute vtlExecute; - protected DataProcessing(VtlBindings vtlBindings){ + FileUtilsInterface fileUtilsInterface; + + protected DataProcessing(VtlBindings vtlBindings, FileUtilsInterface fileUtilsInterface){ this.vtlBindings = vtlBindings; - vtlExecute = new VtlExecute(); + vtlExecute = new VtlExecute(fileUtilsInterface); + this.fileUtilsInterface = fileUtilsInterface; } public abstract String getStepName(); @@ -65,7 +69,7 @@ protected String applyAutomatedVtlInstructions(String bindingName, List errors){ - String vtlScript = TextFileReader.readFromPath(userVtlInstructionsPath); + String vtlScript = TextFileReader.readFromPath(userVtlInstructionsPath, fileUtilsInterface); log.info(String.format("User VTL instructions read for step %s:%n%s", getStepName(), vtlScript)); if (! (vtlScript == null || vtlScript.isEmpty() || vtlScript.contentEquals("")) ) { diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessingManager.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessingManager.java index c26c2ea6..a8257fe3 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessingManager.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/DataProcessingManager.java @@ -2,6 +2,7 @@ import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.parsers.DataFormat; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.extern.log4j.Log4j2; @@ -22,17 +23,17 @@ private DataProcessingManager() { * * @return One of the concrete parsers. */ - public static UnimodalDataProcessing getProcessingClass(DataFormat dataFormat, VtlBindings vtlBindings, MetadataModel metadataModel) { + public static UnimodalDataProcessing getProcessingClass(DataFormat dataFormat, VtlBindings vtlBindings, MetadataModel metadataModel, FileUtilsInterface fileUtilsInterface) { UnimodalDataProcessing dataProcessing = null; switch (dataFormat) { case XFORMS: - dataProcessing = new XformsDataProcessing(vtlBindings, metadataModel); + dataProcessing = new XformsDataProcessing(vtlBindings, metadataModel, fileUtilsInterface); break; case PAPER: - dataProcessing = new PaperDataProcessing(vtlBindings, metadataModel); + dataProcessing = new PaperDataProcessing(vtlBindings, metadataModel, fileUtilsInterface); break; case LUNATIC_XML , LUNATIC_JSON: - dataProcessing = new LunaticDataProcessing(vtlBindings, metadataModel); + dataProcessing = new LunaticDataProcessing(vtlBindings, metadataModel, fileUtilsInterface); break; default: log.warn("Unknown data collection tool, null returned."); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessing.java index dcf054f3..cda4868c 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessing.java @@ -2,6 +2,7 @@ import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.VariablesMap; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; @@ -9,8 +10,8 @@ public class GroupProcessing extends DataProcessing{ private MetadataModel metadataModel; - public GroupProcessing(VtlBindings vtlBindings, MetadataModel metadataModel) { - super(vtlBindings); + public GroupProcessing(VtlBindings vtlBindings, MetadataModel metadataModel, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, fileUtilsInterface); this.metadataModel = metadataModel; } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsProcessing.java index dfae189b..3848d713 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsProcessing.java @@ -2,6 +2,7 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlMacros; import fr.insee.kraftwerk.core.vtl.VtlScript; @@ -15,8 +16,8 @@ */ public class InformationLevelsProcessing extends DataProcessing { - public InformationLevelsProcessing(VtlBindings vtlBindings) { - super(vtlBindings); + public InformationLevelsProcessing(VtlBindings vtlBindings, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, fileUtilsInterface); } @Override diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/LunaticDataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/LunaticDataProcessing.java index b0126759..40d17c09 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/LunaticDataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/LunaticDataProcessing.java @@ -1,6 +1,7 @@ package fr.insee.kraftwerk.core.dataprocessing; import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; @@ -9,8 +10,8 @@ */ public class LunaticDataProcessing extends UnimodalDataProcessing { - public LunaticDataProcessing(VtlBindings vtlBindings, MetadataModel metadataModel) { - super(vtlBindings, metadataModel); + public LunaticDataProcessing(VtlBindings vtlBindings, MetadataModel metadataModel, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, metadataModel, fileUtilsInterface); } @Override diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/MultimodeTransformations.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/MultimodeTransformations.java index aee13db4..2eacbb62 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/MultimodeTransformations.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/MultimodeTransformations.java @@ -1,12 +1,13 @@ package fr.insee.kraftwerk.core.dataprocessing; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; public class MultimodeTransformations extends DataProcessing { - public MultimodeTransformations(VtlBindings vtlBindings) { - super(vtlBindings); + public MultimodeTransformations(VtlBindings vtlBindings, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, fileUtilsInterface); } @Override diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessing.java index 4f7606bc..1df415ef 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessing.java @@ -3,6 +3,7 @@ import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.UcqModality; import fr.insee.kraftwerk.core.metadata.UcqVariable; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; @@ -13,8 +14,8 @@ */ public class PaperDataProcessing extends UnimodalDataProcessing { - public PaperDataProcessing(VtlBindings vtlBindings, MetadataModel metadataModel) { - super(vtlBindings, metadataModel); + public PaperDataProcessing(VtlBindings vtlBindings, MetadataModel metadataModel, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, metadataModel, fileUtilsInterface); } @Override diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationProcessing.java index 34f0e188..07a3fcfe 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationProcessing.java @@ -1,6 +1,7 @@ package fr.insee.kraftwerk.core.dataprocessing; import fr.insee.kraftwerk.core.Constants; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlMacros; import fr.insee.kraftwerk.core.vtl.VtlScript; @@ -17,14 +18,14 @@ public class ReconciliationProcessing extends DataProcessing { private final String modeVariableIdentifier; /** Return processing instance with default mode variable name. */ - public ReconciliationProcessing(VtlBindings vtlBindings) { - super(vtlBindings); + public ReconciliationProcessing(VtlBindings vtlBindings, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, fileUtilsInterface); this.modeVariableIdentifier = Constants.MODE_VARIABLE_NAME; } /** Return processing instance with mode variable name given. */ - public ReconciliationProcessing(VtlBindings vtlBindings, String modeVariableName) { - super(vtlBindings); + public ReconciliationProcessing(VtlBindings vtlBindings, String modeVariableName, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, fileUtilsInterface); this.modeVariableIdentifier = modeVariableName; } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java index a8b637f6..17839cd1 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequencesProcessing.java @@ -3,6 +3,7 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.utils.TextFileReader; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; import lombok.extern.slf4j.Slf4j; @@ -16,8 +17,8 @@ public class TCMSequencesProcessing extends DataProcessing { private final String vtlDirectory; private final MetadataModel metadataModel; - public TCMSequencesProcessing(VtlBindings vtlBindings, MetadataModel metadataModel,String vtlDirectory) { - super(vtlBindings); + public TCMSequencesProcessing(VtlBindings vtlBindings, MetadataModel metadataModel, String vtlDirectory, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, fileUtilsInterface); this.metadataModel = metadataModel; this.vtlDirectory = vtlDirectory; } @@ -60,7 +61,7 @@ protected VtlScript generateVtlInstructions(String bindingName, MetadataModel me for(TCMModuleEnum tcmModuleEnum : tcmSequenceEnum.getTcmModules()){ Path tcmModulePath = Path.of(this.vtlDirectory).resolve("tcm").resolve(tcmModuleEnum.toString() + ".vtl"); if(tcmModulePath.toFile().exists()){ - vtlScriptBuilder.append(TextFileReader.readFromPath(tcmModulePath)); + vtlScriptBuilder.append(TextFileReader.readFromPath(tcmModulePath, fileUtilsInterface)); vtlScriptBuilder.append(System.lineSeparator()); log.info("TCM VTL instructions read for module " + tcmModuleEnum); }else{ diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/UnimodalDataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/UnimodalDataProcessing.java index 1a891820..f20f0873 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/UnimodalDataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/UnimodalDataProcessing.java @@ -1,6 +1,7 @@ package fr.insee.kraftwerk.core.dataprocessing; import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; @@ -11,8 +12,8 @@ public abstract class UnimodalDataProcessing extends DataProcessing { protected MetadataModel metadataModel; - protected UnimodalDataProcessing(VtlBindings vtlBindings, MetadataModel metadataModel){ - super(vtlBindings); + protected UnimodalDataProcessing(VtlBindings vtlBindings, MetadataModel metadataModel, FileUtilsInterface fileUtilsInterface){ + super(vtlBindings, fileUtilsInterface); this.metadataModel = metadataModel; } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/XformsDataProcessing.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/XformsDataProcessing.java index b0c52bc9..81bb30ad 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/XformsDataProcessing.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/dataprocessing/XformsDataProcessing.java @@ -1,6 +1,7 @@ package fr.insee.kraftwerk.core.dataprocessing; import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; @@ -9,8 +10,8 @@ */ public class XformsDataProcessing extends UnimodalDataProcessing { - public XformsDataProcessing(VtlBindings vtlBindings, MetadataModel metadataModel) { - super(vtlBindings, metadataModel); + public XformsDataProcessing(VtlBindings vtlBindings, MetadataModel metadataModel, FileUtilsInterface fileUtilsInterface) { + super(vtlBindings, metadataModel, fileUtilsInterface); } @Override diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParadataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParadataParser.java index 18c1331e..3101e6f2 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParadataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParadataParser.java @@ -8,6 +8,7 @@ import fr.insee.kraftwerk.core.metadata.VariablesMap; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -26,6 +27,12 @@ public class ParadataParser { private final List inputFields = Arrays.asList("RADIO", "CHECKBOX", "INPUT", "DATEPICKER"); + private final FileUtilsInterface fileUtilsInterface; + + public ParadataParser(FileUtilsInterface fileUtilsInterface) { + this.fileUtilsInterface = fileUtilsInterface; + } + public void parseParadata(Paradata paradata, SurveyRawData surveyRawData) throws NullException { log.info("Paradata parser being implemented for Survey Unit : {} !", @@ -35,32 +42,25 @@ public void parseParadata(Paradata paradata, SurveyRawData surveyRawData) throws throw new NullException("JSONFile not defined"); if (!filePath.toString().contentEquals("")) { - + // Parse each ParaDataUE + List listParaDataUE = new ArrayList<>(); // Get all filepaths for each ParadataUE - try (Stream walk = Files.walk(filePath)) { - List listFilePaths = walk.filter(Files::isRegularFile) - .filter(file -> surveyRawData.getIdSurveyUnits().contains(getIdFromFilename(file))).toList(); - // Parse each ParaDataUE - List listParaDataUE = new ArrayList<>(); - - for (Path fileParaDataPath : listFilePaths) { - ParaDataUE paraDataUE = new ParaDataUE(); - paraDataUE.setFilepath(fileParaDataPath); - parseParadataUE(paraDataUE, surveyRawData); - paraDataUE.sortEvents(); - paraDataUE.setSurveyValidationDateTimeStamp(Constants.PARADATA_SURVEY_VALIDATION_EVENT_NAME); - if (paraDataUE.getEvents().size() > 2) { - paraDataUE.createOrchestratorsAndSessions(); - integrateParaDataVariablesIntoUE(paraDataUE, surveyRawData); - listParaDataUE.add(paraDataUE); - } + for(String fileParaDataPath : fileUtilsInterface.listFilePaths(filePath.toString()).stream().filter( + s -> surveyRawData.getIdSurveyUnits().contains(getIdFromFilename(Path.of(s))) + ).toList()){ + ParaDataUE paraDataUE = new ParaDataUE(); + paraDataUE.setFilepath(Path.of(fileParaDataPath)); + parseParadataUE(paraDataUE, surveyRawData); + paraDataUE.sortEvents(); + paraDataUE.setSurveyValidationDateTimeStamp(Constants.PARADATA_SURVEY_VALIDATION_EVENT_NAME); + if (paraDataUE.getEvents().size() > 2) { + paraDataUE.createOrchestratorsAndSessions(); + integrateParaDataVariablesIntoUE(paraDataUE, surveyRawData); + listParaDataUE.add(paraDataUE); } - paradata.setListParadataUE(listParaDataUE); - } catch (Exception e) { - log.error(e.getMessage()); } + paradata.setListParadataUE(listParaDataUE); } - } private String getIdFromFilename(Path file) { @@ -134,7 +134,7 @@ private JSONObject getParadataFromJson(ParaDataUE paradataUE) throws NullExcepti Path filePath = paradataUE.getFilepath(); JSONObject jsonObject; try { - jsonObject = (JSONObject) Constants.readJsonSimple(filePath); + jsonObject = (JSONObject) Constants.readJsonSimple(filePath, fileUtilsInterface); } catch (Exception e) { throw new NullException("Can't read JSON file - " + e.getClass() + " " + e.getMessage()); } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java index 57dadf8d..5da311e8 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java @@ -1,8 +1,8 @@ package fr.insee.kraftwerk.core.extradata.reportingdata; -import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.file.Path; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -18,6 +18,7 @@ import fr.insee.kraftwerk.core.exceptions.NullException; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; @Log4j2 @@ -25,15 +26,14 @@ public class CSVReportingDataParser extends ReportingDataParser { private CSVReader csvReader; + public CSVReportingDataParser(FileUtilsInterface fileUtilsInterface) { + super(fileUtilsInterface); + } + public void parseReportingData(ReportingData reportingData, SurveyRawData data, boolean withAllReportingData) throws NullException { Path filePath = reportingData.getFilepath(); - try{ - readFile(filePath); - } catch (NullPointerException e) { - throw new NullException(); - } - - try { + try(InputStream inputStream = fileUtilsInterface.readFile(filePath.toString())){ + readFile(inputStream); String[] header = this.csvReader.readNext(); if (controlHeader(header)) { String[] nextRecord; @@ -56,7 +56,7 @@ public void parseReportingData(ReportingData reportingData, SurveyRawData data, reportingDataUE.addState(state); reportingData.addReportingDataUE(reportingDataUE); } - integrateReportingDataIntoUE(data, reportingData, withAllReportingData); + integrateReportingDataIntoUE(data, reportingData, withAllReportingData, fileUtilsInterface); } else { log.error("Following CSV file is malformed : {}", filePath); } @@ -95,14 +95,10 @@ public boolean controlHeader(String[] header) { && header[6].contentEquals("adresse") && header[7].contentEquals("numeroDeLot")); } - private void readFile(Path filePath) { - try { - FileReader filereader = new FileReader(filePath.toString()); - this.csvReader = new CSVReader(filereader); - CSVParser parser = (new CSVParserBuilder()).withSeparator(',').build(); - this.csvReader = (new CSVReaderBuilder(filereader)).withCSVParser(parser).build(); - } catch (FileNotFoundException e) { - log.error("Unable to find the file {}, FileNotFoundException {}", filePath, e); - } + private void readFile(InputStream inputStream) { + InputStreamReader inputStreamReader = new InputStreamReader(inputStream); + this.csvReader = new CSVReader(inputStreamReader); + CSVParser parser = (new CSVParserBuilder()).withSeparator(',').build(); + this.csvReader = (new CSVReaderBuilder(inputStreamReader)).withCSVParser(parser).build(); } } \ No newline at end of file diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java index 11047bb8..cd49960f 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java @@ -8,6 +8,7 @@ import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.DateUtils; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import java.sql.Date; @@ -19,9 +20,15 @@ public abstract class ReportingDataParser { private int maxStates = 0; private int maxAttempts = 0; private int maxComments = 0; + protected FileUtilsInterface fileUtilsInterface; + + public ReportingDataParser(FileUtilsInterface fileUtilsInterface) { + this.fileUtilsInterface = fileUtilsInterface; + } protected void integrateReportingDataIntoUE(SurveyRawData surveyRawData, ReportingData reportingData, - boolean withAllReportingData) { + boolean withAllReportingData, FileUtilsInterface fileUtilsInterface) { + this.fileUtilsInterface = fileUtilsInterface; this.maxStates = countMaxStates(reportingData); this.maxAttempts = countMaxAttempts(reportingData); this.maxComments = countMaxComments(reportingData); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/XMLReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/XMLReportingDataParser.java index 0001b4b5..2d037dd4 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/XMLReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/XMLReportingDataParser.java @@ -3,7 +3,8 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.NullException; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; -import fr.insee.kraftwerk.core.utils.XmlFileReader; +import fr.insee.kraftwerk.core.utils.xml.XmlFileReader; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import nu.xom.Document; import nu.xom.Element; @@ -17,8 +18,12 @@ public class XMLReportingDataParser extends ReportingDataParser { private Document document; - - public void parseReportingData(ReportingData reportingData, SurveyRawData data, boolean withAllReportingData) throws NullException { + + public XMLReportingDataParser(FileUtilsInterface fileUtilsInterface) { + super(fileUtilsInterface); + } + + public void parseReportingData(ReportingData reportingData, SurveyRawData data, boolean withAllReportingData) throws NullException { Path filePath = reportingData.getFilepath(); readFile(filePath); Element root; @@ -98,13 +103,13 @@ public void parseReportingData(ReportingData reportingData, SurveyRawData data, reportingData.addReportingDataUE(reportingDataUE); } - integrateReportingDataIntoUE(data, reportingData, withAllReportingData); + integrateReportingDataIntoUE(data, reportingData, withAllReportingData, fileUtilsInterface); this.document = null; } private void readFile(Path filePath) { - XmlFileReader xmlFileReader = new XmlFileReader(); + XmlFileReader xmlFileReader = new XmlFileReader(fileUtilsInterface); this.document = xmlFileReader.readXmlFile(filePath); if (this.document != null) { log.info("Successfully parsed Coleman/Moog answers file: " + filePath); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/ModeInputs.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/ModeInputs.java index 0f79cfbd..eb88e596 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/ModeInputs.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/ModeInputs.java @@ -5,7 +5,6 @@ import lombok.Getter; import lombok.Setter; -import java.net.URL; import java.nio.file.Path; /** @@ -16,7 +15,7 @@ public class ModeInputs { protected Path dataFile; - protected URL ddiUrl; + protected String ddiUrl; protected Path lunaticFile; protected DataFormat dataFormat; protected String dataMode; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputs.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputs.java index 741f90ff..fc9c297d 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputs.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputs.java @@ -1,5 +1,6 @@ package fr.insee.kraftwerk.core.inputs; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.Getter; import lombok.Setter; @@ -17,9 +18,11 @@ public class UserInputs { protected Path vtlTransformationsFile; protected Path vtlInformationLevelsFile; protected String multimodeDatasetName; + protected FileUtilsInterface fileUtilsInterface; - public UserInputs(Path inputDirectory) { + public UserInputs(Path inputDirectory, FileUtilsInterface fileUtilsInterface) { this.inputDirectory = inputDirectory; + this.fileUtilsInterface = fileUtilsInterface; } public ModeInputs getModeInputs(String modeName) { return modeInputsMap.get(modeName); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputsFile.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputsFile.java index 34c3a2fe..1384dd87 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputsFile.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputsFile.java @@ -4,13 +4,12 @@ import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.exceptions.MissingMandatoryFieldException; import fr.insee.kraftwerk.core.exceptions.UnknownDataFormatException; -import fr.insee.kraftwerk.core.utils.FileUtils; -import fr.insee.kraftwerk.core.utils.JsonFileReader; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.JsonReader; import lombok.Getter; import lombok.Setter; import lombok.extern.log4j.Log4j2; -import java.io.File; import java.io.IOException; import java.net.URL; import java.nio.file.Path; @@ -28,8 +27,8 @@ public class UserInputsFile extends UserInputs { private final Set mandatoryFields = Set.of("survey_data", "data_mode", "data_file", "data_format", "multimode_dataset_name"); - public UserInputsFile(Path userConfigFile, Path inputDirectory) throws KraftwerkException { - super(inputDirectory); + public UserInputsFile(Path userConfigFile, Path inputDirectory, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { + super(inputDirectory,fileUtilsInterface); this.userInputFile = userConfigFile; readUserInputs(); } @@ -37,7 +36,7 @@ public UserInputsFile(Path userConfigFile, Path inputDirectory) throws Kraftwerk private void readUserInputs() throws UnknownDataFormatException, MissingMandatoryFieldException, KraftwerkException { try { - JsonNode userInputs = JsonFileReader.read(userInputFile); + JsonNode userInputs = JsonReader.read(userInputFile,fileUtilsInterface); // JsonNode filesNode = userInputs.get("survey_data"); for (JsonNode fileNode : filesNode) { @@ -46,13 +45,13 @@ private void readUserInputs() throws UnknownDataFormatException, MissingMandator String dataFolder = readField(fileNode, "data_file"); String paradataFolder = readField(fileNode, "paradata_folder"); String reportingFolder = readField(fileNode, "reporting_data_file"); - Path dataPath = (new File(dataFolder).exists()) ? convertToUserPath(dataFolder) : FileUtils.convertToPath(dataFolder,inputDirectory); - URL ddiFile = FileUtils.convertToUrl(readField(fileNode, "DDI_file"),inputDirectory); - Path lunaticFile = FileUtils.convertToPath(readField(fileNode, "lunatic_file"),inputDirectory); + Path dataPath = (fileUtilsInterface.isFileExists(dataFolder)) ? convertToUserPath(dataFolder) : fileUtilsInterface.convertToPath(dataFolder,inputDirectory); + String ddiFile = fileUtilsInterface.convertToUrl(readField(fileNode, "DDI_file"),inputDirectory); + Path lunaticFile = fileUtilsInterface.convertToPath(readField(fileNode, "lunatic_file"),inputDirectory); String dataFormat = readField(fileNode, "data_format"); - Path paradataPath = (paradataFolder != null && new File(paradataFolder).exists()) ? convertToUserPath(paradataFolder) : FileUtils.convertToPath(paradataFolder,inputDirectory); - Path reportingDataFile = (reportingFolder != null && new File(reportingFolder).exists()) ? convertToUserPath(reportingFolder) : FileUtils.convertToPath(reportingFolder,inputDirectory); - Path vtlFile = FileUtils.convertToPath(readField(fileNode, "mode_specifications"),inputDirectory); + Path paradataPath = (paradataFolder != null && fileUtilsInterface.isFileExists(paradataFolder)) ? convertToUserPath(paradataFolder) : fileUtilsInterface.convertToPath(paradataFolder,inputDirectory); + Path reportingDataFile = (reportingFolder != null && fileUtilsInterface.isFileExists(reportingFolder)) ? convertToUserPath(reportingFolder) : fileUtilsInterface.convertToPath(reportingFolder,inputDirectory); + Path vtlFile = fileUtilsInterface.convertToPath(readField(fileNode, "mode_specifications"),inputDirectory); ModeInputs modeInputs = new ModeInputs(); modeInputs.setDataFile(dataPath); modeInputs.setDdiUrl(ddiFile); @@ -65,9 +64,9 @@ private void readUserInputs() throws UnknownDataFormatException, MissingMandator } // multimodeDatasetName = readField(userInputs, "multimode_dataset_name"); - vtlReconciliationFile = FileUtils.convertToPath(readField(userInputs, "reconciliation_specifications"),inputDirectory); - vtlTransformationsFile = FileUtils.convertToPath(readField(userInputs, "transformation_specifications"),inputDirectory); - vtlInformationLevelsFile = FileUtils.convertToPath(readField(userInputs, "information_levels_specifications"),inputDirectory); + vtlReconciliationFile = fileUtilsInterface.convertToPath(readField(userInputs, "reconciliation_specifications"),inputDirectory); + vtlTransformationsFile = fileUtilsInterface.convertToPath(readField(userInputs, "transformation_specifications"),inputDirectory); + vtlInformationLevelsFile = fileUtilsInterface.convertToPath(readField(userInputs, "information_levels_specifications"),inputDirectory); } catch (IOException e) { log.error("Unable to read user input file: {} , {}", userInputFile, e); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputsGenesis.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputsGenesis.java index f62a5125..1e232805 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputsGenesis.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/inputs/UserInputsGenesis.java @@ -6,16 +6,14 @@ import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.exceptions.MissingMandatoryFieldException; import fr.insee.kraftwerk.core.exceptions.UnknownDataFormatException; -import fr.insee.kraftwerk.core.utils.FileUtils; -import fr.insee.kraftwerk.core.utils.JsonFileReader; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.JsonReader; import lombok.extern.log4j.Log4j2; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.stream.Stream; @Log4j2 public class UserInputsGenesis extends UserInputs{ @@ -24,17 +22,17 @@ public class UserInputsGenesis extends UserInputs{ private final List modes; - public UserInputsGenesis(boolean hasConfigFile, Path inputDirectory, List modes) throws KraftwerkException, IOException { - super(inputDirectory); + public UserInputsGenesis(boolean hasConfigFile, Path inputDirectory, List modes, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { + super(inputDirectory, fileUtilsInterface); this.hasConfigFile = hasConfigFile; this.modes=modes; computeInputs(); } - private void computeInputs() throws KraftwerkException, IOException { + private void computeInputs() throws KraftwerkException { UserInputsFile userInputsFile; if(hasConfigFile){ - userInputsFile = new UserInputsFile(inputDirectory.resolve(Constants.USER_INPUT_FILE), inputDirectory); + userInputsFile = new UserInputsFile(inputDirectory.resolve(Constants.USER_INPUT_FILE), inputDirectory, fileUtilsInterface); modeInputsMap = userInputsFile.getModeInputsMap(); }else{ for (Mode mode : modes) { @@ -49,9 +47,9 @@ private void computeInputs() throws KraftwerkException, IOException { * @param mode mode to get inputs from * @return a ModeInputs object */ - private ModeInputs getModeInputs(Mode mode) throws KraftwerkException, IOException { + private ModeInputs getModeInputs(Mode mode) throws KraftwerkException { ModeInputs modeInputs = new ModeInputs(); - modeInputs.setDdiUrl(findDDIFile(inputDirectory.resolve(mode.name())).toFile().toURI().toURL()); + modeInputs.setDdiUrl(findDDIFile(inputDirectory.resolve(mode.name())).toString()); modeInputs.setLunaticFile(findLunaticFile(inputDirectory.resolve(mode.name()))); modeInputs.setDataMode(mode.name()); if (mode == Mode.WEB || mode == Mode.TEL || mode == Mode.F2F) { @@ -69,13 +67,13 @@ private ModeInputs getModeInputs(Mode mode) throws KraftwerkException, IOExcepti private Path getModeVtlFile(Mode mode) throws UnknownDataFormatException, MissingMandatoryFieldException, KraftwerkException { Path userInputFile = inputDirectory.resolve(Constants.USER_INPUT_FILE); try { - JsonNode userInputs = JsonFileReader.read(userInputFile); + JsonNode userInputs = JsonReader.read(userInputFile, fileUtilsInterface); JsonNode filesNode = userInputs.get("survey_data"); for (JsonNode fileNode : filesNode) { String dataMode = readField(fileNode, "data_mode"); if (dataMode == null) {break;} if (dataMode.equals(mode.name())) { - return FileUtils.convertToPath(readField(fileNode, "mode_specifications"),inputDirectory); + return fileUtilsInterface.convertToPath(readField(fileNode, "mode_specifications"),inputDirectory); } } } catch (IOException e) { @@ -106,11 +104,8 @@ private String readField(JsonNode node, String field) throws MissingMandatoryFie * @return Path of the DDI file * @throws IOException – if an I/O error is thrown when accessing the starting file */ - public Path findDDIFile(Path specDirectory) throws KraftwerkException, IOException { - try (Stream files = Files.find(specDirectory, 1, (path, basicFileAttributes) -> path.toFile().getName().toLowerCase().matches("ddi[\\w,\\s-]+\\.xml"))) { - return files.findFirst() - .orElseThrow(() -> new KraftwerkException(404, "No DDI file (ddi*.xml) found in " + specDirectory.toString())); - } + public Path findDDIFile(Path specDirectory) throws KraftwerkException { + return Path.of(fileUtilsInterface.findFile(String.valueOf(specDirectory),Constants.DDI_FILE_REGEX)); } /** @@ -119,10 +114,7 @@ public Path findDDIFile(Path specDirectory) throws KraftwerkException, IOExcepti * @return Path of the Lunatic file * @throws IOException – if an I/O error is thrown when accessing the starting file */ - public Path findLunaticFile(Path specDirectory) throws KraftwerkException, IOException { - try (Stream files = Files.find(specDirectory, 1, (path, basicFileAttributes) -> path.toFile().getName().toLowerCase().matches("lunatic[\\w,\\s-]+\\.json"))) { - return files.findFirst() - .orElseThrow(() -> new KraftwerkException(404, "No Lunatic file (lunatic*.xml) found in " + specDirectory.toString())); - } + public Path findLunaticFile(Path specDirectory) throws KraftwerkException { + return Path.of(fileUtilsInterface.findFile(String.valueOf(specDirectory),Constants.LUNATIC_FILE_REGEX)); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/DDIReader.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/DDIReader.java index 83c84fdb..bfa5beb2 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/DDIReader.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/DDIReader.java @@ -2,6 +2,7 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.xsl.SaxonTransformer; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang3.StringUtils; @@ -17,8 +18,8 @@ import javax.xml.parsers.ParserConfigurationException; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.MalformedURLException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -38,12 +39,12 @@ private DDIReader() { * written in the system temporary folder with the name 'variables.xml'. This file is * deleted after being used or when the virtual machine terminates. * - * @param ddiUrl : Path to the DDI file. + * @param ddiUrlString : Path to the DDI file. * * @return The variables found in the DDI. * @throws KraftwerkException */ - public static MetadataModel getMetadataFromDDI(URL ddiUrl) throws KraftwerkException { + public static MetadataModel getMetadataFromDDI(String ddiUrlString, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { try { // Path of the output 'variables.xml' temp file @@ -51,7 +52,7 @@ public static MetadataModel getMetadataFromDDI(URL ddiUrl) throws KraftwerkExcep variablesFile.deleteOnExit(); Path variablesTempFilePath = variablesFile.toPath(); - transformDDI(ddiUrl, variablesTempFilePath); + transformDDI(ddiUrlString, variablesTempFilePath, fileUtilsInterface); MetadataModel metadataModel = readVariables(variablesTempFilePath); Files.delete(variablesFile.toPath()); @@ -59,7 +60,7 @@ public static MetadataModel getMetadataFromDDI(URL ddiUrl) throws KraftwerkExcep } catch (MalformedURLException e) { - log.error(String.format("Error when converting file path '%s' to an URL.", ddiUrl), e); + log.error(String.format("Error when converting file path '%s' to an URL.", ddiUrlString), e); return null; } catch (IOException e) { log.error("Unable to write temp file.", e); @@ -73,12 +74,12 @@ public static MetadataModel getMetadataFromDDI(URL ddiUrl) throws KraftwerkExcep /** * Apply the XSLT_STRUCTURED_VARIABLES transformation. * - * @param ddiUrl : URL of the DDI file. + * @param ddiUrlString : URL of the DDI file. * @param variablesFilePath : Path of the 'variables.xml' file to be generated. */ - private static void transformDDI(URL ddiUrl, Path variablesFilePath) { - SaxonTransformer saxonTransformer = new SaxonTransformer(); - saxonTransformer.xslTransform(ddiUrl, Constants.XSLT_STRUCTURED_VARIABLES, variablesFilePath); + private static void transformDDI(String ddiUrlString, Path variablesFilePath, FileUtilsInterface fileUtilsInterface) throws IOException { + SaxonTransformer saxonTransformer = new SaxonTransformer(fileUtilsInterface); + saxonTransformer.xslTransform(ddiUrlString, Constants.XSLT_STRUCTURED_VARIABLES, variablesFilePath); } /** @@ -230,10 +231,8 @@ private static void addValues(UcqVariable variable, NodeList valueElements) { private static boolean nodeIsElementWithName(Node groupNode, String name) { return name.equals(groupNode.getNodeName()) && groupNode.getNodeType() == Node.ELEMENT_NODE; } - private static Element readXmlFile(Path variablesFilePath) throws ParserConfigurationException, SAXException, IOException, KraftwerkException { - File file = variablesFilePath.toFile(); System.setProperty("javax.xml.parsers.DocumentBuilderFactory", "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); @@ -241,11 +240,12 @@ private static Element readXmlFile(Path variablesFilePath) factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse(file); - if (document == null) - throw new KraftwerkException(500, "Can't read DDI - DDI is null"); - - return document.getDocumentElement(); + try(InputStream inputStream = Files.newInputStream(variablesFilePath)){ + Document document = builder.parse(inputStream); + if (document == null) + throw new KraftwerkException(500, "Can't read DDI - DDI is null"); + return document.getDocumentElement(); + } } private static String getFirstChildValue(Element variableElement, String childTagName) { diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/LunaticReader.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/LunaticReader.java index 142de0f2..52051f96 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/LunaticReader.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/LunaticReader.java @@ -3,7 +3,8 @@ import com.fasterxml.jackson.databind.JsonNode; import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.metadata.CalculatedVariables.CalculatedVariable; -import fr.insee.kraftwerk.core.utils.JsonFileReader; +import fr.insee.kraftwerk.core.utils.JsonReader; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import java.io.IOException; @@ -38,9 +39,9 @@ private LunaticReader() { * @param lunaticFile Path to a lunatic questionnaire file. * @return A CalculatedVariables map. */ - public static CalculatedVariables getCalculatedFromLunatic(Path lunaticFile) { + public static CalculatedVariables getCalculatedFromLunatic(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { try { - JsonNode rootNode = JsonFileReader.read(lunaticFile); + JsonNode rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); String lunaticModelVersion = rootNode.get(LUNATIC_MODEL_VERSION).asText(); boolean isLunaticV2 = compareVersions(lunaticModelVersion, "2.3.0") > 0; @@ -76,9 +77,9 @@ public static CalculatedVariables getCalculatedFromLunatic(Path lunaticFile) { * @param lunaticFile Path to a lunatic questionnaire file. * @return A List of String. */ - public static List getMissingVariablesFromLunatic(Path lunaticFile) { + public static List getMissingVariablesFromLunatic(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { try { - JsonNode rootNode = JsonFileReader.read(lunaticFile); + JsonNode rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); List variables = new ArrayList<>(); List varsEno = Arrays.asList(Constants.getEnoVariables()); @@ -100,9 +101,9 @@ public static List getMissingVariablesFromLunatic(Path lunaticFile) { * @param lunaticFile Path to a lunatic questionnaire file. * @return A List of String. */ - public static List getFilterResultFromLunatic(Path lunaticFile) { + public static List getFilterResultFromLunatic(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { try { - JsonNode rootNode = JsonFileReader.read(lunaticFile); + JsonNode rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); List variables = new ArrayList<>(); JsonNode variablesNode = rootNode.get(VARIABLES); @@ -115,9 +116,9 @@ public static List getFilterResultFromLunatic(Path lunaticFile) { } } - public static String getLunaticModelVersion(Path lunaticFile){ + public static String getLunaticModelVersion(Path lunaticFile, FileUtilsInterface fileUtilsInterface){ try { - JsonNode rootNode = JsonFileReader.read(lunaticFile); + JsonNode rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); return rootNode.get(LUNATIC_MODEL_VERSION).toString(); } catch (IOException e) { @@ -133,10 +134,10 @@ public static String getLunaticModelVersion(Path lunaticFile){ * @param lunaticFile : Path to a Lunatic specification file. * @return The variables found in the Lunatic specification. */ - public static MetadataModel getMetadataFromLunatic(Path lunaticFile) { + public static MetadataModel getMetadataFromLunatic(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { JsonNode rootNode; try { - rootNode = JsonFileReader.read(lunaticFile); + rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); List variables = new ArrayList<>(); JsonNode variablesNode = rootNode.get(VARIABLES); variablesNode.forEach(newVar -> variables.add(newVar.get("name").asText())); @@ -405,10 +406,10 @@ private static Group getNewGroup(MetadataModel metadataModel, String newName, Gr * @param lunaticFile : Path to a Lunatic specification file. * @return the questionnaire model id */ - public static String getQuestionnaireModelId(Path lunaticFile) { + public static String getQuestionnaireModelId(Path lunaticFile, FileUtilsInterface fileUtilsInterface) { JsonNode rootNode; try { - rootNode = JsonFileReader.read(lunaticFile); + rootNode = JsonReader.read(lunaticFile, fileUtilsInterface); return rootNode.get("id").asText(); } catch (IOException e) { log.error(EXCEPTION_MESSAGE + lunaticFile); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtils.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtils.java index 9def414a..34cd633a 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtils.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtils.java @@ -3,6 +3,7 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.ModeInputs; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import java.util.LinkedHashMap; @@ -16,31 +17,31 @@ private MetadataUtils(){ throw new IllegalStateException("Utility class"); } - public static Map getMetadata(Map modeInputsMap){ + public static Map getMetadata(Map modeInputsMap, FileUtilsInterface fileUtilsInterface){ Map metadataModels = new LinkedHashMap<>(); - modeInputsMap.forEach((k, v) -> putToMetadataModels(k,v,metadataModels)); + modeInputsMap.forEach((k, v) -> putToMetadataModels(k,v,metadataModels, fileUtilsInterface)); return metadataModels; } - private static void putToMetadataModels(String dataMode, ModeInputs modeInputs, Map metadataModels ) { + private static void putToMetadataModels(String dataMode, ModeInputs modeInputs, Map metadataModels, FileUtilsInterface fileUtilsInterface) { // Step 1 : we add the variables read in the DDI MetadataModel metadataModel = new MetadataModel(); try { - metadataModel = DDIReader.getMetadataFromDDI(modeInputs.getDdiUrl()); + metadataModel = DDIReader.getMetadataFromDDI(modeInputs.getDdiUrl(), fileUtilsInterface); } catch (KraftwerkException e) { log.error(e.getMessage()); } // Step 2 : we add the variables that are only present in the Lunatic file if (modeInputs.getLunaticFile() != null) { // We read and store lunaticModelVersion - metadataModel.putSpecVersions(SpecType.LUNATIC,LunaticReader.getLunaticModelVersion(modeInputs.getLunaticFile())); + metadataModel.putSpecVersions(SpecType.LUNATIC,LunaticReader.getLunaticModelVersion(modeInputs.getLunaticFile(), fileUtilsInterface)); // First we add the collected _MISSING variables - List missingVars = LunaticReader.getMissingVariablesFromLunatic(modeInputs.getLunaticFile()); + List missingVars = LunaticReader.getMissingVariablesFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); for (String missingVar : missingVars) { addLunaticVariable(metadataModel, missingVar, Constants.MISSING_SUFFIX, VariableType.STRING); } // Then we add calculated FILTER_RESULT_ variables - List filterResults = LunaticReader.getFilterResultFromLunatic(modeInputs.getLunaticFile()); + List filterResults = LunaticReader.getFilterResultFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); for (String filterResult : filterResults) { addLunaticVariable(metadataModel, filterResult, Constants.FILTER_RESULT_PREFIX, VariableType.BOOLEAN); } @@ -77,14 +78,14 @@ public static void addLunaticVariable(MetadataModel metadata, String missingVar, metadata.getVariables().putVariable(new Variable(missingVar, group, varType)); } - public static Map getMetadataFromLunatic(Map modeInputsMap) { + public static Map getMetadataFromLunatic(Map modeInputsMap, FileUtilsInterface fileUtilsInterface) { Map metadataModels = new LinkedHashMap<>(); - modeInputsMap.forEach((k, v) -> putToMetadataVariableFromLunatic(k,v,metadataModels)); + modeInputsMap.forEach((k, v) -> putToMetadataVariableFromLunatic(k,v,metadataModels, fileUtilsInterface)); return metadataModels; } - private static void putToMetadataVariableFromLunatic(String dataMode, ModeInputs modeInputs, Map metadataModels ) { - MetadataModel metadataModel = LunaticReader.getMetadataFromLunatic(modeInputs.getLunaticFile()); + private static void putToMetadataVariableFromLunatic(String dataMode, ModeInputs modeInputs, Map metadataModels, FileUtilsInterface fileUtilsInterface) { + MetadataModel metadataModel = LunaticReader.getMetadataFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); // We add the variables for pairwise links if (metadataModel.getVariables().getVariable(Constants.LIENS) != null) { // We identify the group containing the individuals diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsGenesis.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsGenesis.java index 62f04f0e..5279a71d 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsGenesis.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/metadata/MetadataUtilsGenesis.java @@ -3,6 +3,7 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.ModeInputs; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import java.util.LinkedHashMap; @@ -16,29 +17,29 @@ private MetadataUtilsGenesis() { throw new IllegalStateException("Utility class"); } - public static Map getMetadata(Map modeInputsMap) throws KraftwerkException { + public static Map getMetadata(Map modeInputsMap, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { Map metadataModels = new LinkedHashMap<>(); for (Map.Entry entry : modeInputsMap.entrySet()) { String k = entry.getKey(); ModeInputs v = entry.getValue(); - putToMetadataVariable(k, v, metadataModels); + putToMetadataVariable(k, v, metadataModels, fileUtilsInterface); } return metadataModels; } - private static void putToMetadataVariable(String dataMode, ModeInputs modeInputsGenesis, Map metadataModels ) throws KraftwerkException { + private static void putToMetadataVariable(String dataMode, ModeInputs modeInputsGenesis, Map metadataModels, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { // Step 1 : we add the variables read in the DDI - MetadataModel metadataModel = DDIReader.getMetadataFromDDI(modeInputsGenesis.getDdiUrl()); + MetadataModel metadataModel = DDIReader.getMetadataFromDDI(modeInputsGenesis.getDdiUrl(), fileUtilsInterface); // Step 2 : we add the variables that are only present in the Lunatic file if (modeInputsGenesis.getLunaticFile() != null) { // First we add the collected _MISSING variables - List missingVars = LunaticReader.getMissingVariablesFromLunatic(modeInputsGenesis.getLunaticFile()); + List missingVars = LunaticReader.getMissingVariablesFromLunatic(modeInputsGenesis.getLunaticFile(), fileUtilsInterface); for (String missingVar : missingVars) { addLunaticVariable(metadataModel, missingVar, Constants.MISSING_SUFFIX, VariableType.STRING); } // Then we add calculated FILTER_RESULT_ variables - List filterResults = LunaticReader.getFilterResultFromLunatic(modeInputsGenesis.getLunaticFile()); + List filterResults = LunaticReader.getFilterResultFromLunatic(modeInputsGenesis.getLunaticFile(), fileUtilsInterface); for (String filterResult : filterResults) { addLunaticVariable(metadataModel, filterResult, Constants.FILTER_RESULT_PREFIX, VariableType.BOOLEAN); } @@ -65,14 +66,14 @@ public static void addLunaticVariable(MetadataModel metadataModel, String missin metadataModel.getVariables().putVariable(new Variable(missingVar, group, varType)); } - public static Map getMetadataFromLunatic(Map modeInputsMap) { + public static Map getMetadataFromLunatic(Map modeInputsMap, FileUtilsInterface fileUtilsInterface) { Map metadataModels = new LinkedHashMap<>(); - modeInputsMap.forEach((k, v) -> putToMetadataVariableFromLunatic(k,v,metadataModels)); + modeInputsMap.forEach((k, v) -> putToMetadataVariableFromLunatic(k,v,metadataModels, fileUtilsInterface)); return metadataModels; } - private static void putToMetadataVariableFromLunatic(String dataMode, ModeInputs modeInputs, Map metadataModels ) { - MetadataModel metadataModel = LunaticReader.getMetadataFromLunatic(modeInputs.getLunaticFile()); + private static void putToMetadataVariableFromLunatic(String dataMode, ModeInputs modeInputs, Map metadataModels, FileUtilsInterface fileUtilsInterface) { + MetadataModel metadataModel = LunaticReader.getMetadataFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); metadataModels.put(dataMode, metadataModel); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java index e5a0dde4..53b3b7a5 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/OutputFiles.java @@ -4,7 +4,7 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -28,6 +28,7 @@ public abstract class OutputFiles { private final VtlBindings vtlBindings; private final Set datasetToCreate = new HashSet<>(); private final Statement database; + protected final FileUtilsInterface fileUtilsInterface; /** * When an instance is created, the output folder is created. @@ -35,17 +36,18 @@ public abstract class OutputFiles { * @param outDirectory Out directory defined in application properties. * @param vtlBindings Vtl bindings where datasets are stored. */ - protected OutputFiles(Path outDirectory, VtlBindings vtlBindings, List modes, Statement database) { + protected OutputFiles(Path outDirectory, VtlBindings vtlBindings, List modes, Statement database, FileUtilsInterface fileUtilsInterface) { this.vtlBindings = vtlBindings; setOutputDatasetNames(modes); outputFolder = outDirectory; - createOutputFolder(); this.database = database; + this.fileUtilsInterface = fileUtilsInterface; + createOutputFolder(); } /** Create output folder if doesn't exist. */ private void createOutputFolder() { - FileUtils.createDirectoryIfNotExist(outputFolder); + fileUtilsInterface.createDirectoryIfNotExist(outputFolder); } /** See getOutputDatasetNames doc. */ diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java index 0a259064..718fe2f0 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java @@ -9,6 +9,7 @@ import fr.insee.kraftwerk.core.outputs.TableScriptInfo; import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.utils.TextFileWriter; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.extern.slf4j.Slf4j; @@ -40,12 +41,12 @@ public class CsvOutputFiles extends OutputFiles { * @param outDirectory Out directory defined in application properties. * @param vtlBindings Vtl bindings where datasets are stored. */ - public CsvOutputFiles(Path outDirectory, VtlBindings vtlBindings, List modes, Statement database) { - super(outDirectory, vtlBindings, modes, database); + public CsvOutputFiles(Path outDirectory, VtlBindings vtlBindings, List modes, Statement database, FileUtilsInterface fileUtilsInterface) { + super(outDirectory, vtlBindings, modes, database, fileUtilsInterface); this.kraftwerkExecutionLog = null; } - public CsvOutputFiles(Path outDirectory, VtlBindings vtlBindings, KraftwerkExecutionLog kraftwerkExecutionLog, List modes, Statement database) { - super(outDirectory, vtlBindings, modes, database); + public CsvOutputFiles(Path outDirectory, VtlBindings vtlBindings, KraftwerkExecutionLog kraftwerkExecutionLog, List modes, Statement database, FileUtilsInterface fileUtilsInterface) { + super(outDirectory, vtlBindings, modes, database, fileUtilsInterface); this.kraftwerkExecutionLog = kraftwerkExecutionLog; } @@ -56,8 +57,10 @@ public CsvOutputFiles(Path outDirectory, VtlBindings vtlBindings, KraftwerkExecu @Override public void writeOutputTables() throws KraftwerkException { for (String datasetName : getDatasetToCreate()) { - File outputFile = getOutputFolder().resolve(outputFileName(datasetName)).toFile(); try { + //Temporary file + File tmpOutputFile = File.createTempFile(outputFileName(datasetName), null); + //Get column names List columnNames = SqlUtils.getColumnNames(getDatabase(), datasetName); @@ -72,28 +75,31 @@ public void writeOutputTables() throws KraftwerkException { List boolColumnIndexes = new ArrayList<>(); //Create file with double quotes header - Files.write(outputFile.toPath(), buildHeader(columnNames, boolColumnNames, boolColumnIndexes).getBytes()); + Files.write(tmpOutputFile.toPath(), buildHeader(columnNames, boolColumnNames, boolColumnIndexes).getBytes()); //Data export into temp file - StringBuilder exportCsvQuery = getExportCsvQuery(datasetName, outputFile, columnNames); + StringBuilder exportCsvQuery = getExportCsvQuery(datasetName, tmpOutputFile, columnNames); this.getDatabase().execute(exportCsvQuery.toString()); //Apply csv format transformations //Merge data file with header file //Read line by line to avoid memory waste - try(BufferedReader bufferedReader = Files.newBufferedReader(Path.of(outputFile.toPath().toAbsolutePath() + "data"))){ + try(BufferedReader bufferedReader = Files.newBufferedReader(Path.of(tmpOutputFile.toPath().toAbsolutePath() + "data"))){ String line = bufferedReader.readLine(); while(line != null){ //Apply transformations to elements line = applyNullTransformation(line); line = applyBooleanTransformations(line, boolColumnIndexes); - Files.write(outputFile.toPath(),(line + "\n").getBytes(),StandardOpenOption.APPEND); + Files.write(tmpOutputFile.toPath(),(line + "\n").getBytes(),StandardOpenOption.APPEND); line = bufferedReader.readLine(); } } - Files.deleteIfExists(Path.of(outputFile.toPath().toAbsolutePath() + "data")); + Files.deleteIfExists(Path.of(tmpOutputFile.toPath().toAbsolutePath() + "data")); + + //Move to output folder + getFileUtilsInterface().moveFile(tmpOutputFile.toPath().toAbsolutePath(), getOutputFolder().resolve(outputFileName(datasetName)).toString()); //Count rows for functional log if (kraftwerkExecutionLog != null) { @@ -177,9 +183,9 @@ public void writeImportScripts(Map metadataModels, List nbParquetFilesbyDataset = new HashMap<>(); + /** * When an instance is created, the output folder is created. - * + * * @param outDirectory Out directory defined in application properties. * @param vtlBindings Vtl bindings where datasets are stored. + * @param modes list of modes names + * @param database connection to duckDb database + * @param fileUtilsInterface file interface to use (file system or minio) */ - public ParquetOutputFiles(Path outDirectory, VtlBindings vtlBindings, List modes, Statement databaseConnection) { - super(outDirectory, vtlBindings, modes, databaseConnection); + public ParquetOutputFiles(Path outDirectory, VtlBindings vtlBindings, List modes, Statement database, + FileUtilsInterface fileUtilsInterface) { + super(outDirectory, vtlBindings, modes, database, fileUtilsInterface); } @@ -45,11 +54,16 @@ public ParquetOutputFiles(Path outDirectory, VtlBindings vtlBindings, List metadataModels, List stream = Files.list(dataPath)){ - stream.forEach(t -> { - try { - parseDataFile(t); - if(kraftwerkExecutionLog != null) { - kraftwerkExecutionLog.getOkFileNames().add(t.getFileName().toString()); - } - } catch (NullException e) { - log.error("IOException occurred when trying to list data file: {} in folder {}", t, dataPath); + if(Boolean.TRUE.equals(fileUtilsInterface.isDirectory(dataPath.toString()))){ + for(String path : fileUtilsInterface.listFilePaths(String.valueOf(dataPath))){ + try { + parseDataFile(Path.of(path)); + if(kraftwerkExecutionLog != null) { + kraftwerkExecutionLog.getOkFileNames().add(Path.of(path).getFileName().toString()); } - }); - } catch (IOException e) { - log.error(String.format("IOException occurred when trying to list data files of folder: %s", dataPath)); + } catch (NullException e) { + log.error("IOException occurred when trying to list data file: {} in folder {}", path, dataPath); + } } } - - else { - log.warn(String.format("Data path given could not be identified as a file or folder: %s", dataPath)); - log.warn("No data was parsed."); - } - } /** @@ -83,35 +76,28 @@ public final void parseSurveyDataWithoutDDI(Path dataPath, Path lunaticFile, Kra log.error(DATAPATH_IS_NULL); throw new NullException(DATAPATH_IS_NULL); } - if (Files.isRegularFile(dataPath)) { + if(fileUtilsInterface.isDirectory(dataPath.toString()) == null){ + log.warn(String.format("Data path given could not be identified as a file or folder: %s", dataPath)); + log.warn("No data was parsed."); + } + if(Boolean.FALSE.equals(fileUtilsInterface.isDirectory(dataPath.toString()))) { parseDataFileWithoutDDI(dataPath,lunaticFile); if(kraftwerkExecutionLog != null) { kraftwerkExecutionLog.getOkFileNames().add(dataPath.getFileName().toString()); } } - - else if (Files.isDirectory(dataPath)) { - try (Stream stream = Files.list(dataPath)){ - stream.forEach(t -> { - try { - parseDataFileWithoutDDI(t,lunaticFile); - if(kraftwerkExecutionLog != null) { - kraftwerkExecutionLog.getOkFileNames().add(t.getFileName().toString()); - } - } catch (NullException e) { - log.error("IOException occurred when trying to list data file: {} in folder {}", t, dataPath); + if(Boolean.TRUE.equals(fileUtilsInterface.isDirectory(dataPath.toString()))) { + for(String path : fileUtilsInterface.listFilePaths(dataPath.toString())){ + try { + parseDataFileWithoutDDI(Path.of(path),lunaticFile); + if(kraftwerkExecutionLog != null) { + kraftwerkExecutionLog.getOkFileNames().add(Path.of(path).getFileName().toString()); } - }); - } catch (IOException e) { - log.error(String.format("IOException occurred when trying to list data files of folder: %s", dataPath)); + } catch (NullException e) { + log.error("IOException occurred when trying to list data file: {} in folder {}", path, dataPath); + } } } - - else { - log.warn(String.format("Data path given could not be identified as a file or folder: %s", dataPath)); - log.warn("No data was parsed."); - } - } /** diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/DataParserManager.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/DataParserManager.java index 59036010..e0a43222 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/DataParserManager.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/DataParserManager.java @@ -1,6 +1,7 @@ package fr.insee.kraftwerk.core.parsers; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; /** @@ -22,12 +23,12 @@ private DataParserManager(){ * @return * One of the concrete parsers. */ - public static DataParser getParser(DataFormat dataFormat, SurveyRawData data) { + public static DataParser getParser(DataFormat dataFormat, SurveyRawData data, FileUtilsInterface fileUtilsInterface) { return switch (dataFormat) { - case XFORMS -> new XformsDataParser(data); - case PAPER -> new PaperDataParser(data); - case LUNATIC_XML -> new LunaticXmlDataParser(data); - case LUNATIC_JSON -> new LunaticJsonDataParser(data); + case XFORMS -> new XformsDataParser(data, fileUtilsInterface); + case PAPER -> new PaperDataParser(data, fileUtilsInterface); + case LUNATIC_XML -> new LunaticXmlDataParser(data, fileUtilsInterface); + case LUNATIC_JSON -> new LunaticJsonDataParser(data,fileUtilsInterface); default -> { log.debug(String.format("Unknown data format: %s", dataFormat)); throw new IllegalArgumentException(String.format("Unknown data format: %s", dataFormat)); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticJsonDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticJsonDataParser.java index 2b1162e6..0403623c 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticJsonDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticJsonDataParser.java @@ -5,6 +5,7 @@ import fr.insee.kraftwerk.core.rawdata.GroupInstance; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import org.json.simple.JSONObject; @@ -18,8 +19,8 @@ public class LunaticJsonDataParser extends DataParser { * @param data The SurveyRawData to be filled by the parseSurveyData method. * The variables must have been previously set. */ - public LunaticJsonDataParser(SurveyRawData data) { - super(data); + public LunaticJsonDataParser(SurveyRawData data, FileUtilsInterface fileUtilsInterface) { + super(data,fileUtilsInterface); } @Override @@ -27,9 +28,10 @@ void parseDataFile(Path filePath) throws NullException { log.warn("Lunatic data parser being implemented!"); // - JSONObject jsonObject = null; + JSONObject jsonObject; + try { - jsonObject = (JSONObject) Constants.readJsonSimple(filePath); + jsonObject = (JSONObject) Constants.readJsonSimple(filePath, fileUtilsInterface); } catch (Exception e) { throw new NullException("Can't read JSON file - "+e.getClass()+" "+ e.getMessage()); } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParser.java index 7c6564fa..a7cea016 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParser.java @@ -6,7 +6,8 @@ import fr.insee.kraftwerk.core.rawdata.GroupInstance; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; -import fr.insee.kraftwerk.core.utils.XmlFileReader; +import fr.insee.kraftwerk.core.utils.xml.XmlFileReader; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import nu.xom.Document; import nu.xom.Element; @@ -33,8 +34,8 @@ public class LunaticXmlDataParser extends DataParser { * @param data The SurveyRawData to be filled by the parseSurveyData method. The * variables must have been previously set. */ - public LunaticXmlDataParser(SurveyRawData data) { - super(data); + public LunaticXmlDataParser(SurveyRawData data, FileUtilsInterface fileUtilsInterface) { + super(data, fileUtilsInterface); } /** @@ -44,7 +45,7 @@ public LunaticXmlDataParser(SurveyRawData data) { * @return The parsed document. */ private Document readXmlFile(Path filePath) { - XmlFileReader xmlFileReader = new XmlFileReader(); + XmlFileReader xmlFileReader = new XmlFileReader(fileUtilsInterface); Document document = xmlFileReader.readXmlFile(filePath); if (document == null) { log.warn("Failed to parse Lunatic answers file: " + filePath); @@ -61,7 +62,7 @@ private void parseDataFile(Path filePath, Path lunaticFile) { .getChildElements("SurveyUnit"); String questionnaireModelId = null; if (lunaticFile != null) { - questionnaireModelId = LunaticReader.getQuestionnaireModelId(lunaticFile); + questionnaireModelId = LunaticReader.getQuestionnaireModelId(lunaticFile, fileUtilsInterface); } for (int i = 0; i < questionnaireNodeList.size(); i++) { diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/PaperDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/PaperDataParser.java index 21047f43..c421b017 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/PaperDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/PaperDataParser.java @@ -13,11 +13,12 @@ import fr.insee.kraftwerk.core.rawdata.GroupInstance; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; -import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.file.Path; import java.util.Arrays; import java.util.HashMap; @@ -32,8 +33,8 @@ @Log4j2 public class PaperDataParser extends DataParser { - /** File reader */ - private FileReader filereader; + /** Input stream reader */ + private InputStreamReader inputStreamReader; /** Csv reader */ private CSVReader csvReader; @@ -43,45 +44,38 @@ public class PaperDataParser extends DataParser { * @param data The SurveyRawData to be filled by the parseSurveyData method. The * variables must have been previously set. */ - public PaperDataParser(SurveyRawData data) { - super(data); + public PaperDataParser(SurveyRawData data, FileUtilsInterface fileUtilsInterface) { + super(data, fileUtilsInterface); } /** * Instantiate a CSVReader. * - * @param filePath Path to the CSV file. + * @param inputStream stream to the CSV file. */ - private void readCsvFile(Path filePath) { - try { - // Create an object of file reader - // class with CSV file as a parameter. - filereader = new FileReader(filePath.toString()); - // create csvReader object passing - // file reader as a parameter - csvReader = new CSVReader(filereader); - - // create csvParser object with - // custom separator semicolon - CSVParser parser = new CSVParserBuilder().withSeparator(Constants.CSV_PAPER_DATA_SEPARATOR).build(); - - // create csvReader object with parameter - // file reader and parser - csvReader = new CSVReaderBuilder(filereader) - // .withSkipLines(1) // (uncomment to ignore header) - .withCSVParser(parser).build(); - - } catch (FileNotFoundException e) { - log.error(String.format("Unable to find the file %s", filePath), e); - } + private void readCsvFileStream(InputStream inputStream) { + // Create an object of file reader + // class with CSV file as a parameter. + inputStreamReader = new InputStreamReader(inputStream); + // create csvReader object passing + // file reader as a parameter + csvReader = new CSVReader(inputStreamReader); + + // create csvParser object with + // custom separator semicolon + CSVParser parser = new CSVParserBuilder().withSeparator(Constants.CSV_PAPER_DATA_SEPARATOR).build(); + + // create csvReader object with parameter + // file reader and parser + csvReader = new CSVReaderBuilder(inputStreamReader) + // .withSkipLines(1) // (uncomment to ignore header) + .withCSVParser(parser).build(); } @Override void parseDataFile(Path filePath) { - - readCsvFile(filePath); - - try { + try(InputStream inputStream = fileUtilsInterface.readFile(filePath.toString())){ + readCsvFileStream(inputStream); /* * We first map the variables in the header (first line) of the CSV file to the @@ -161,7 +155,6 @@ void parseDataFile(Path filePath) { } data.addQuestionnaire(questionnaireData); } - filereader.close(); csvReader.close(); } catch (CsvValidationException e) { log.error(String.format("Following CSV file is malformed: %s", filePath), e); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/XformsDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/XformsDataParser.java index af2d6b9f..f1f74ad2 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/XformsDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/parsers/XformsDataParser.java @@ -4,7 +4,8 @@ import fr.insee.kraftwerk.core.rawdata.GroupInstance; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; -import fr.insee.kraftwerk.core.utils.XmlFileReader; +import fr.insee.kraftwerk.core.utils.xml.XmlFileReader; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import nu.xom.Document; import nu.xom.Element; @@ -23,8 +24,8 @@ public class XformsDataParser extends DataParser { * @param data The SurveyRawData to be filled by the parseSurveyData method. * The variables must have been previously set. */ - public XformsDataParser(SurveyRawData data) { - super(data); + public XformsDataParser(SurveyRawData data, FileUtilsInterface fileUtilsInterface) { + super(data, fileUtilsInterface); } /** @@ -33,7 +34,7 @@ public XformsDataParser(SurveyRawData data) { * @return The parsed document. */ private Document readXmlFile(Path filePath) { - XmlFileReader xmlFileReader = new XmlFileReader(); + XmlFileReader xmlFileReader = new XmlFileReader(fileUtilsInterface); Document document = xmlFileReader.readXmlFile(filePath); if (document != null) { log.info("Successfully parsed Xforms answers file: " + filePath); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java index aa6a6012..29a57d24 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequence.java @@ -13,6 +13,7 @@ import fr.insee.kraftwerk.core.parsers.DataParser; import fr.insee.kraftwerk.core.parsers.DataParserManager; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; @@ -25,10 +26,12 @@ public class BuildBindingsSequence { VtlExecute vtlExecute; private final boolean withAllReportingData; + private final FileUtilsInterface fileUtilsInterface; - public BuildBindingsSequence(boolean withAllReportingData) { - vtlExecute = new VtlExecute(); + public BuildBindingsSequence(boolean withAllReportingData, FileUtilsInterface fileUtilsInterface) { + vtlExecute = new VtlExecute(fileUtilsInterface); this.withAllReportingData = withAllReportingData; + this.fileUtilsInterface = fileUtilsInterface; } public void buildVtlBindings(UserInputsFile userInputsFile, String dataMode, VtlBindings vtlBindings, MetadataModel metadataModel, boolean withDDI, KraftwerkExecutionLog kraftwerkExecutionLog) throws KraftwerkException { @@ -40,7 +43,7 @@ public void buildVtlBindings(UserInputsFile userInputsFile, String dataMode, Vtl /* Step 2.1 : Fill the data object with the survey answers file */ data.setDataFilePath(modeInputs.getDataFile()); - DataParser parser = DataParserManager.getParser(modeInputs.getDataFormat(), data); + DataParser parser = DataParserManager.getParser(modeInputs.getDataFormat(), data, fileUtilsInterface); log.info("Parsing survey data file " + modeInputs.getDataFile().getFileName()); if (withDDI) { parser.parseSurveyData(modeInputs.getDataFile(),kraftwerkExecutionLog); @@ -62,7 +65,7 @@ public void buildVtlBindings(UserInputsFile userInputsFile, String dataMode, Vtl private void parseParadata(ModeInputs modeInputs, SurveyRawData data) throws NullException { Path paraDataFolder = modeInputs.getParadataFolder(); if (paraDataFolder != null) { - ParadataParser paraDataParser = new ParadataParser(); + ParadataParser paraDataParser = new ParadataParser(fileUtilsInterface); Paradata paraData = new Paradata(paraDataFolder); paraDataParser.parseParadata(paraData, data); } @@ -73,11 +76,11 @@ private void parseReportingData(ModeInputs modeInputs, SurveyRawData data) throw if (reportingDataFile != null) { ReportingData reportingData = new ReportingData(reportingDataFile); if (reportingDataFile.toString().contains(".xml")) { - XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(); + XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(fileUtilsInterface); xMLReportingDataParser.parseReportingData(reportingData, data, withAllReportingData); } else if (reportingDataFile.toString().contains(".csv")) { - CSVReportingDataParser cSVReportingDataParser = new CSVReportingDataParser(); + CSVReportingDataParser cSVReportingDataParser = new CSVReportingDataParser(fileUtilsInterface); cSVReportingDataParser.parseReportingData(reportingData, data, withAllReportingData); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java index 4835efc7..f473f490 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java @@ -16,11 +16,10 @@ import fr.insee.kraftwerk.core.rawdata.GroupInstance; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; -import java.io.File; import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -28,9 +27,11 @@ public class BuildBindingsSequenceGenesis { VtlExecute vtlExecute; + FileUtilsInterface fileUtilsInterface; - public BuildBindingsSequenceGenesis() { - vtlExecute = new VtlExecute(); + public BuildBindingsSequenceGenesis(FileUtilsInterface fileUtilsInterface) { + vtlExecute = new VtlExecute(fileUtilsInterface); + this.fileUtilsInterface = fileUtilsInterface; } public void buildVtlBindings(String dataMode, VtlBindings vtlBindings, Map metadataModels, List surveyUnits, Path inDirectory) throws KraftwerkException { @@ -70,39 +71,37 @@ public void buildVtlBindings(String dataMode, VtlBindings vtlBindings, Map listFiles = FileUtils.listFiles(reportingDataFile.toString()); + if (fileUtilsInterface.isFileExists(reportingDataFile.toString())) { + List listFiles = fileUtilsInterface.listFileNames(reportingDataFile.toString()); for (String file : listFiles) { ReportingData reportingData = new ReportingData(reportingDataFile.resolve(file)); if (file.contains(".xml")) { - XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(); + XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(fileUtilsInterface); xMLReportingDataParser.parseReportingData(reportingData, data, true); } else if (file.contains(".csv")) { - CSVReportingDataParser cSVReportingDataParser = new CSVReportingDataParser(); + CSVReportingDataParser cSVReportingDataParser = new CSVReportingDataParser(fileUtilsInterface); cSVReportingDataParser.parseReportingData(reportingData, data, true); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/ControlInputSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/ControlInputSequence.java index df381158..f0e51ab8 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/ControlInputSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/ControlInputSequence.java @@ -1,38 +1,40 @@ package fr.insee.kraftwerk.core.sequence; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; @Log4j2 public class ControlInputSequence { String defaultDirectory; + FileUtilsInterface fileUtilsInterface; - public ControlInputSequence(String defaultDirectory) { + public ControlInputSequence(String defaultDirectory, FileUtilsInterface fileUtilsInterface) { super(); this.defaultDirectory = defaultDirectory; + this.fileUtilsInterface = fileUtilsInterface; } - public UserInputsFile getUserInputs(Path inDirectory) throws KraftwerkException { - return new UserInputsFile(inDirectory.resolve(Constants.USER_INPUT_FILE), inDirectory); + public UserInputsFile getUserInputs(Path inDirectory, FileUtilsInterface fileUtilsInterface) throws KraftwerkException { + return new UserInputsFile(inDirectory.resolve(Constants.USER_INPUT_FILE), inDirectory, fileUtilsInterface); } public Path getInDirectory(String inDirectoryParam) throws KraftwerkException { - Path inDirectory = Paths.get(inDirectoryParam); + Path inDirectory = Path.of(inDirectoryParam); if (!verifyInDirectory(inDirectory)) inDirectory = Paths.get(defaultDirectory, "in", inDirectoryParam); if (!verifyInDirectory(inDirectory)) throw new KraftwerkException(400, "Configuration file not found"); return inDirectory; } - + private boolean verifyInDirectory(Path inDirectory) { Path userInputFile = inDirectory.resolve(Constants.USER_INPUT_FILE); - if (Files.exists(userInputFile)) { + if (fileUtilsInterface.isFileExists(userInputFile.toString())) { log.info(String.format("Found configuration file in campaign folder: %s", userInputFile)); } else { log.info("No configuration file found in campaign folder: " + inDirectory); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/ControlInputSequenceGenesis.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/ControlInputSequenceGenesis.java index 988feed7..d82b8f63 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/ControlInputSequenceGenesis.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/ControlInputSequenceGenesis.java @@ -1,10 +1,10 @@ package fr.insee.kraftwerk.core.sequence; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import fr.insee.kraftwerk.core.Constants; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.Getter; import lombok.extern.log4j.Log4j2; @@ -12,25 +12,27 @@ public class ControlInputSequenceGenesis { String defaultDirectory; - private static final String INPUT_FOLDER = "in"; + private static final String SPEC_FOLDER = "specs"; + private final FileUtilsInterface fileUtilsInterface; @Getter boolean hasConfigFile = true; - public ControlInputSequenceGenesis(String defaultDirectory) { + public ControlInputSequenceGenesis(String defaultDirectory, FileUtilsInterface fileUtilsInterface) { super(); this.defaultDirectory = defaultDirectory; + this.fileUtilsInterface = fileUtilsInterface; } public Path getInDirectory(String inDirectoryParam) { - Path inDirectory = Paths.get(defaultDirectory, INPUT_FOLDER, inDirectoryParam); + Path inDirectory = Paths.get(defaultDirectory,SPEC_FOLDER, inDirectoryParam); hasConfigFile = verifyInDirectory(inDirectory); return inDirectory; } private boolean verifyInDirectory(Path inDirectory) { Path userInputFile = inDirectory.resolve(Constants.USER_INPUT_FILE); - if (Files.exists(userInputFile)) { + if (fileUtilsInterface.isFileExists(userInputFile.toString())) { log.info(String.format("Found configuration file in campaign folder: %s", userInputFile)); } else { log.info("No configuration file found in campaign folder: " + inDirectory); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java index 9bca1d79..4e88876d 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java @@ -12,38 +12,38 @@ import fr.insee.kraftwerk.core.dataprocessing.ReconciliationProcessing; import fr.insee.kraftwerk.core.inputs.UserInputs; import fr.insee.kraftwerk.core.metadata.MetadataModel; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.NoArgsConstructor; @NoArgsConstructor public class MultimodalSequence { - - public void multimodalProcessing(UserInputs userInputs, VtlBindings vtlBindings, List errors, Map metadataModels){ + + public void multimodalProcessing(UserInputs userInputs, VtlBindings vtlBindings, List errors, Map metadataModels, FileUtilsInterface fileUtilsInterface) { String multimodeDatasetName = Constants.MULTIMODE_DATASET_NAME; /* Step 3.1 : aggregate unimodal datasets into a multimodal unique dataset */ - DataProcessing reconciliationProcessing = new ReconciliationProcessing(vtlBindings); + DataProcessing reconciliationProcessing = new ReconciliationProcessing(vtlBindings, fileUtilsInterface); String vtlGenerate = reconciliationProcessing.applyVtlTransformations(multimodeDatasetName, userInputs.getVtlReconciliationFile(), errors); - TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, "ReconciliationProcessing",multimodeDatasetName), vtlGenerate); + TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "ReconciliationProcessing",multimodeDatasetName), vtlGenerate, fileUtilsInterface); /* Step 3.1.b : clean up processing */ - CleanUpProcessing cleanUpProcessing = new CleanUpProcessing(vtlBindings, metadataModels); + CleanUpProcessing cleanUpProcessing = new CleanUpProcessing(vtlBindings, metadataModels, fileUtilsInterface); vtlGenerate = cleanUpProcessing.applyVtlTransformations(multimodeDatasetName, null, errors); - TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, "CleanUpProcessing",multimodeDatasetName), vtlGenerate); + TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "CleanUpProcessing",multimodeDatasetName), vtlGenerate, fileUtilsInterface); /* Step 3.2 : treatments on the multimodal dataset */ - DataProcessing multimodeTransformations = new MultimodeTransformations(vtlBindings); + DataProcessing multimodeTransformations = new MultimodeTransformations(vtlBindings, fileUtilsInterface); vtlGenerate = multimodeTransformations.applyVtlTransformations(multimodeDatasetName, userInputs.getVtlTransformationsFile(), errors); - TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, "MultimodeTransformations",multimodeDatasetName), vtlGenerate); + TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "MultimodeTransformations",multimodeDatasetName), vtlGenerate, fileUtilsInterface); /* Step 3.3 : create datasets on each information level (i.e. each group) */ - DataProcessing informationLevelsProcessing = new InformationLevelsProcessing(vtlBindings); + DataProcessing informationLevelsProcessing = new InformationLevelsProcessing(vtlBindings, fileUtilsInterface); vtlGenerate = informationLevelsProcessing.applyVtlTransformations(multimodeDatasetName, userInputs.getVtlInformationLevelsFile(), errors); - TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, "InformationLevelsProcessing",multimodeDatasetName), vtlGenerate); + TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "InformationLevelsProcessing",multimodeDatasetName), vtlGenerate, fileUtilsInterface); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java index d7263549..330e22b7 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/UnimodalSequence.java @@ -6,7 +6,7 @@ import fr.insee.kraftwerk.core.inputs.ModeInputs; import fr.insee.kraftwerk.core.inputs.UserInputs; import fr.insee.kraftwerk.core.metadata.*; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.TextFileWriter; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.NoArgsConstructor; @@ -21,7 +21,7 @@ public class UnimodalSequence { public void applyUnimodalSequence(UserInputs userInputs, String dataMode, VtlBindings vtlBindings, - List errors, Map metadataModels) { + List errors, Map metadataModels, FileUtilsInterface fileUtilsInterface) { ModeInputs modeInputs = userInputs.getModeInputs(dataMode); String vtlGenerate; @@ -41,40 +41,40 @@ public void applyUnimodalSequence(UserInputs userInputs, String dataMode, VtlBin /* Step 2.4b : Apply VTL expression for calculated variables (if any) */ if (modeInputs.getLunaticFile() != null) { CalculatedVariables calculatedVariables = LunaticReader - .getCalculatedFromLunatic(modeInputs.getLunaticFile()); - CalculatedProcessing calculatedProcessing = new CalculatedProcessing(vtlBindings, calculatedVariables); + .getCalculatedFromLunatic(modeInputs.getLunaticFile(), fileUtilsInterface); + CalculatedProcessing calculatedProcessing = new CalculatedProcessing(vtlBindings, calculatedVariables, fileUtilsInterface); vtlGenerate = calculatedProcessing.applyCalculatedVtlTransformations(dataMode, modeInputs.getModeVtlFile(), errors); - TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, "CalculatedProcessing", dataMode), - vtlGenerate); + TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "CalculatedProcessing", dataMode), + vtlGenerate, fileUtilsInterface); } else { log.info(String.format("No Lunatic questionnaire file for mode \"%s\". Calculated variables will not be evaluated", dataMode)); } /* Step 2.4c : Prefix variable names with their belonging group names */ - vtlGenerate = new GroupProcessing(vtlBindings, metadataModels.get(dataMode)).applyVtlTransformations(dataMode, null, errors); - TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, "GroupProcessing", dataMode), vtlGenerate); + vtlGenerate = new GroupProcessing(vtlBindings, metadataModels.get(dataMode), fileUtilsInterface).applyVtlTransformations(dataMode, null, errors); + TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "GroupProcessing", dataMode), vtlGenerate, fileUtilsInterface); /* Step 2.5 : Apply standard mode-specific VTL transformations */ UnimodalDataProcessing dataProcessing = DataProcessingManager.getProcessingClass(modeInputs.getDataFormat(), - vtlBindings, metadataModels.get(dataMode)); + vtlBindings, metadataModels.get(dataMode), fileUtilsInterface); vtlGenerate = dataProcessing.applyVtlTransformations( dataMode, Path.of(Constants.VTL_FOLDER_PATH) .resolve("unimode") .resolve(dataMode+".vtl"), errors); - TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, "StandardVtl", dataMode), vtlGenerate); + TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "StandardVtl", dataMode), vtlGenerate, fileUtilsInterface); /* Step 2.5b : Apply TCM VTL transformations */ - TCMSequencesProcessing tcmSequencesProcessing = new TCMSequencesProcessing(vtlBindings,metadataModels.get(dataMode) , Constants.VTL_FOLDER_PATH); + TCMSequencesProcessing tcmSequencesProcessing = new TCMSequencesProcessing(vtlBindings,metadataModels.get(dataMode) , Constants.VTL_FOLDER_PATH, fileUtilsInterface); vtlGenerate = tcmSequencesProcessing.applyAutomatedVtlInstructions(dataMode, errors); - TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, "TCMSequenceVTL", dataMode), vtlGenerate); + TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, "TCMSequenceVTL", dataMode), vtlGenerate, fileUtilsInterface); /* Step 2.5c : Apply user specified mode-specific VTL transformations */ vtlGenerate = dataProcessing.applyVtlTransformations(dataMode, modeInputs.getModeVtlFile(), errors); - TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, dataProcessing.getStepName(), dataMode), - vtlGenerate); + TextFileWriter.writeFile(fileUtilsInterface.getTempVtlFilePath(userInputs, dataProcessing.getStepName(), dataMode), + vtlGenerate, fileUtilsInterface); } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/VtlReaderWriterSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/VtlReaderWriterSequence.java index 7356e6f5..d72e9db6 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/VtlReaderWriterSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/VtlReaderWriterSequence.java @@ -4,7 +4,7 @@ import java.nio.file.Path; import fr.insee.kraftwerk.core.dataprocessing.StepEnum; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; @@ -14,9 +14,12 @@ public class VtlReaderWriterSequence { VtlExecute vtlExecute; private static final String JSON = ".json"; + FileUtilsInterface fileUtilsInterface; - public VtlReaderWriterSequence() { - vtlExecute = new VtlExecute(); + + public VtlReaderWriterSequence(FileUtilsInterface fileUtilsInterface) { + vtlExecute = new VtlExecute(fileUtilsInterface); + this.fileUtilsInterface = fileUtilsInterface; } public void readDataset(String path,String bindingName, StepEnum previousStep, VtlBindings vtlBindings) { @@ -29,7 +32,7 @@ public void readDataset(String pathBindings,String bindingName,VtlBindings vtlBi } public void writeTempBindings(Path inDirectory, String dataMode, VtlBindings vtlBindings, StepEnum step) { - Path tempOutputPath = FileUtils.transformToTemp(inDirectory).resolve(dataMode+"_"+step.getStepLabel()+JSON); + Path tempOutputPath = FileUtilsInterface.transformToTemp(inDirectory).resolve(dataMode+"_"+step.getStepLabel()+JSON); vtlExecute.writeJsonDataset(dataMode, tempOutputPath, vtlBindings); } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java index 15948f90..c8e7313c 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java @@ -7,7 +7,7 @@ import fr.insee.kraftwerk.core.outputs.OutputFiles; import fr.insee.kraftwerk.core.outputs.csv.CsvOutputFiles; import fr.insee.kraftwerk.core.outputs.parquet.ParquetOutputFiles; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; import fr.insee.kraftwerk.core.vtl.VtlBindings; import lombok.NoArgsConstructor; @@ -22,38 +22,52 @@ @NoArgsConstructor public class WriterSequence { - public void writeOutputFiles(Path inDirectory, LocalDateTime executionDateTime, VtlBindings vtlBindings, - Map modeInputsMap, Map metadataModels, - List errors, KraftwerkExecutionLog kraftwerkExecutionLog, - Statement database) throws KraftwerkException { - Path outDirectory = FileUtils.transformToOut(inDirectory, executionDateTime); - - writeCsvFiles(outDirectory, vtlBindings, modeInputsMap, metadataModels, errors, kraftwerkExecutionLog, - database); - writeParquetFiles(outDirectory, vtlBindings, modeInputsMap, metadataModels, errors, database); - } - - //Write CSV - private void writeCsvFiles(Path outDirectory, VtlBindings vtlBindings, Map modeInputsMap, - Map metadataModels, List errors, - KraftwerkExecutionLog kraftwerkExecutionLog, Statement databaseConnection) throws KraftwerkException { - /* Step 5.1 : write csv output tables */ - OutputFiles csvOutputFiles = new CsvOutputFiles(outDirectory, vtlBindings, kraftwerkExecutionLog, - new ArrayList<>(modeInputsMap.keySet()), databaseConnection); - csvOutputFiles.writeOutputTables(); - - /* Step 5.2 : write scripts to import csv tables in several languages */ - csvOutputFiles.writeImportScripts(metadataModels, errors); - } - - //Write Parquet - private void writeParquetFiles(Path outDirectory, VtlBindings vtlBindings, Map modeInputsMap, - Map metadataModels, List errors, - Statement databaseConnection) throws KraftwerkException { - /* Step 5.3 : write parquet output tables */ - OutputFiles parquetOutputFiles = new ParquetOutputFiles(outDirectory, vtlBindings, - new ArrayList<>(modeInputsMap.keySet()), databaseConnection); - parquetOutputFiles.writeOutputTables(); - parquetOutputFiles.writeImportScripts(metadataModels, errors); - } + public void writeOutputFiles(Path inDirectory, + LocalDateTime executionDateTime, + VtlBindings vtlBindings, + Map modeInputsMap, + Map metadataModels, + List errors, + KraftwerkExecutionLog kraftwerkExecutionLog, + Statement database, + FileUtilsInterface fileUtilsInterface) throws KraftwerkException { + Path outDirectory = FileUtilsInterface.transformToOut(inDirectory,executionDateTime); + + writeCsvFiles(inDirectory, executionDateTime,vtlBindings, modeInputsMap, metadataModels, errors, null, database, fileUtilsInterface); + writeParquetFiles(outDirectory, vtlBindings, modeInputsMap, metadataModels, errors, database, fileUtilsInterface); + } + + public void writeCsvFiles(Path inDirectory, + LocalDateTime executionDateTime, + VtlBindings vtlBindings, + Map modeInputsMap, + Map metadataModels, + List errors, + KraftwerkExecutionLog kraftwerkExecutionLog, + Statement database, + FileUtilsInterface fileUtilsInterface) throws KraftwerkException { + //Write CSV + Path outDirectory = FileUtilsInterface.transformToOut(inDirectory,executionDateTime); + /* Step 5.1 : write csv output tables */ + OutputFiles csvOutputFiles = new CsvOutputFiles(outDirectory, vtlBindings, kraftwerkExecutionLog, new ArrayList<>(modeInputsMap.keySet()), + database, fileUtilsInterface); + csvOutputFiles.writeOutputTables(); + /* Step 5.2 : write scripts to import csv tables in several languages */ + csvOutputFiles.writeImportScripts(metadataModels, errors); + } + + + //Write Parquet + private void writeParquetFiles(Path outDirectory, + VtlBindings vtlBindings, + Map modeInputsMap, + Map metadataModels, + List errors, + Statement database, + FileUtilsInterface fileUtilsInterface) throws KraftwerkException { + /* Step 5.3 : write parquet output tables */ + OutputFiles parquetOutputFiles = new ParquetOutputFiles(outDirectory, vtlBindings, new ArrayList<>(modeInputsMap.keySet()), database, fileUtilsInterface); + parquetOutputFiles.writeOutputTables(); + parquetOutputFiles.writeImportScripts(metadataModels, errors); + } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/CsvUtils.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/CsvUtils.java index e04f1767..7a414b88 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/CsvUtils.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/CsvUtils.java @@ -1,10 +1,10 @@ package fr.insee.kraftwerk.core.utils; -import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.nio.file.Path; import com.opencsv.CSVParser; import com.opencsv.CSVParserBuilder; @@ -17,28 +17,27 @@ /** Encapsulate org.opencsv features that we use in Kraftwerk. */ public class CsvUtils { - private CsvUtils() { //Utility class } - public static CSVReader getReader(Path filePath) throws IOException { + public static CSVReader getReader(InputStream inputStream){ CSVParser parser = new CSVParserBuilder() .withSeparator(Constants.CSV_OUTPUTS_SEPARATOR) //.withQuoteChar(Constants.CSV_OUTPUTS_QUOTE_CHAR) //.withEscapeChar(CSVWriter.DEFAULT_ESCAPE_CHARACTER) .build(); - return new CSVReaderBuilder(new FileReader(filePath.toFile(), StandardCharsets.UTF_8)) + return new CSVReaderBuilder(new InputStreamReader(inputStream, StandardCharsets.UTF_8)) //.withSkipLines(1) // (uncomment to ignore header) .withCSVParser(parser) .build(); } - public static CSVReader getReaderWithSeparator(String filePath, char separator) throws IOException { + public static CSVReader getReaderWithSeparator(InputStream inputStream, char separator){ CSVParser csvParser= new CSVParserBuilder() .withSeparator(separator) .build(); - return new CSVReaderBuilder(new FileReader(filePath, StandardCharsets.UTF_8)) + return new CSVReaderBuilder(new InputStreamReader(inputStream, StandardCharsets.UTF_8)) .withCSVParser(csvParser) .build(); } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/JsonFileReader.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/JsonReader.java similarity index 54% rename from kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/JsonFileReader.java rename to kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/JsonReader.java index 684994a6..b3e77027 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/JsonFileReader.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/JsonReader.java @@ -1,14 +1,16 @@ package fr.insee.kraftwerk.core.utils; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; -public class JsonFileReader { +public class JsonReader { - private JsonFileReader(){ + private JsonReader(){ throw new IllegalStateException("Utility class"); } @@ -19,9 +21,11 @@ private JsonFileReader(){ * * @return A jackson.databind.JsonNode. */ - public static JsonNode read(Path filePath) throws IOException { + public static JsonNode read(Path filePath, FileUtilsInterface fileUtilsInterface) throws IOException { ObjectMapper mapper = new ObjectMapper(); - return mapper.readTree(filePath.toFile()); + try (InputStream inputStream = fileUtilsInterface.readFile(filePath.toString())){ + return mapper.readTree(inputStream); + } } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileReader.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileReader.java index d177ba28..7cddcbd5 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileReader.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileReader.java @@ -1,11 +1,13 @@ package fr.insee.kraftwerk.core.utils; import java.io.BufferedReader; -import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.Reader; import java.nio.file.Path; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; @Log4j2 @@ -26,10 +28,10 @@ private TextFileReader(){ * @return * The content of the file in a string. */ - public static String readFromPath(Path filePath){ - try { - FileReader fileReader = new FileReader(filePath.toFile()); - return readTextContent(fileReader); + public static String readFromPath(Path filePath, FileUtilsInterface fileUtilsInterface){ + try (InputStream inputStream = fileUtilsInterface.readFile(filePath.toString())){ + InputStreamReader inputStreamReader = new InputStreamReader(inputStream); + return readTextContent(inputStreamReader); } catch (IOException e) { log.warn(String.format("Unable to read the text file %s.", filePath), e); @@ -37,14 +39,14 @@ public static String readFromPath(Path filePath){ } } - public static String readTextContent(Reader fileReader) throws IOException { - BufferedReader br = new BufferedReader(fileReader); + public static String readTextContent(Reader inputStreamReader) throws IOException { + BufferedReader br = new BufferedReader(inputStreamReader); StringBuilder content = new StringBuilder(); String line; while ((line = br.readLine()) != null) { content.append(line); } - fileReader.close(); + inputStreamReader.close(); br.close(); return content.toString(); } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileWriter.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileWriter.java index 22b1073d..67a195bc 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileWriter.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileWriter.java @@ -1,6 +1,5 @@ package fr.insee.kraftwerk.core.utils; -import java.io.FileWriter; import java.io.IOException; import java.nio.file.Path; import java.time.LocalDateTime; @@ -8,6 +7,8 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.Constants; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog; import lombok.extern.log4j.Log4j2; @@ -27,44 +28,31 @@ private TextFileWriter() { * @param filePath Path to the file. * @param fileContent Content of the text file. */ - public static void writeFile(Path filePath, String fileContent){ - try (FileWriter myWriter = new FileWriter(filePath.toFile())){ - myWriter.write(fileContent); - log.info(String.format("Text file: %s successfully written", filePath)); - } catch (IOException e) { - log.warn(String.format("Error occurred when trying to write text file: %s", filePath), e); - } - } + public static void writeFile(Path filePath, String fileContent, FileUtilsInterface fileUtilsInterface){ + fileUtilsInterface.writeFile(String.valueOf(filePath), fileContent, true); + log.info(String.format("Text file: %s successfully written", filePath)); + } - public static void writeErrorsFile(Path inDirectory, LocalDateTime localDateTime, List errors) { - Path tempOutputPath = FileUtils.transformToOut(inDirectory,localDateTime) + public static void writeErrorsFile(Path inDirectory, LocalDateTime localDateTime, List errors, FileUtilsInterface fileUtilsInterface) { + Path tempOutputPath = FileUtilsInterface.transformToOut(inDirectory,localDateTime) .resolve(Constants.ERRORS_FILE_NAME); - FileUtils.createDirectoryIfNotExist(tempOutputPath.getParent()); + fileUtilsInterface.createDirectoryIfNotExist(tempOutputPath.getParent()); //Write errors file if (!errors.isEmpty()) { - try (FileWriter myWriter = new FileWriter(tempOutputPath.toFile(),true)){ - for (KraftwerkError error : errors){ - myWriter.write(error.toString()); - } - log.info(String.format("Text file: %s successfully written", tempOutputPath)); - } catch (IOException e) { - log.warn(String.format("Error occurred when trying to write text file: %s", tempOutputPath), e); + for (KraftwerkError error : errors) { + fileUtilsInterface.writeFile(tempOutputPath.toString(), error.toString(), false); } + log.info(String.format("Text file: %s successfully written", tempOutputPath)); } else { log.debug("No error found during VTL transformations"); } } - public static void writeLogFile(Path inDirectory, LocalDateTime localDateTime, KraftwerkExecutionLog kraftwerkExecutionLog){ - Path tempOutputPath = FileUtils.transformToOut(inDirectory,localDateTime); + public static void writeLogFile(Path inDirectory, LocalDateTime localDateTime, KraftwerkExecutionLog kraftwerkExecutionLog, FileUtilsInterface fileUtilsInterface){ + Path tempOutputPath = FileUtilsInterface.transformToOut(inDirectory,localDateTime); tempOutputPath = tempOutputPath.resolve(inDirectory.getFileName() + "_LOG_" + kraftwerkExecutionLog.getStartTimeStamp() +".txt"); - FileUtils.createDirectoryIfNotExist(tempOutputPath.getParent()); - try (FileWriter myWriter = new FileWriter(tempOutputPath.toFile(),true)){ - myWriter.write(kraftwerkExecutionLog.getFormattedString()); - }catch (IOException e) { - log.warn(String.format("Error occurred when trying to write log file: %s", tempOutputPath), e); - } + fileUtilsInterface.writeFile(tempOutputPath.toString(), kraftwerkExecutionLog.getFormattedString(), false); } } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/XMLSplitter.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/XMLSplitter.java deleted file mode 100644 index deea20d9..00000000 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/XMLSplitter.java +++ /dev/null @@ -1,131 +0,0 @@ -package fr.insee.kraftwerk.core.utils; - -import javax.xml.stream.*; -import javax.xml.stream.events.EndDocument; -import javax.xml.stream.events.StartDocument; -import javax.xml.stream.events.StartElement; -import javax.xml.stream.events.XMLEvent; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; - -public class XMLSplitter { - - // We use StAX in this class to deal with memory issues on huge XML files - private XMLSplitter() { - throw new IllegalStateException("Utility class"); - } - - public static void split(String inputfolder, String xmlfile, String outputFolder, String condition, int nbElementsByFile) throws XMLStreamException, IOException { - - FileUtils.createDirectoryIfNotExist(Paths.get(outputFolder)); - String xmlResource = inputfolder + xmlfile; - List header = getHeader(xmlResource, condition); - - XMLEventFactory xef = XMLEventFactory.newFactory(); - XMLInputFactory xif = XMLInputFactory.newInstance(); - xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); - xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); - XMLEventReader xer = xif.createXMLEventReader(new FileReader(xmlResource)); - StartElement rootStartElement = xer.nextTag().asStartElement(); - StartDocument startDocument = xef.createStartDocument(); - EndDocument endDocument = xef.createEndDocument(); - - XMLOutputFactory xof = XMLOutputFactory.newFactory(); - int fileCount = 1; - while(xer.hasNext() && !xer.peek().isEndDocument()) { - XMLEvent xmlEvent = xer.nextEvent(); - - if (isStartElementWithName(condition, xmlEvent)) { - // Create a file for the fragment, the name is derived from the value of the id attribute - FileWriter fileWriter = new FileWriter(outputFolder + "split" + fileCount + ".xml"); - - // A StAX XMLEventWriter will be used to write the XML fragment - XMLEventWriter xew = xof.createXMLEventWriter(fileWriter); - xew.add(startDocument); - - // Add the elements which are common to all split files - addHeadersToNewFile(header, xew); - - // Write the XMLEvents that are part of SurveyUnit element - xew.add(xmlEvent); - xmlEvent = xer.nextEvent(); - int nbResponses = 1; - // We loop until we reach the end tag Survey units indicating the near end of the document - iterateOnSurveyUnits(condition, nbElementsByFile, xer, xmlEvent, xew, nbResponses); - - // Write the file, close everything we opened and update the file's counter - xew.add(xef.createEndElement(rootStartElement.getName(), null)); - xew.add(endDocument); - fileWriter.close(); - - fileCount++; - - } - } - } - - private static void iterateOnSurveyUnits(String condition, int nbElementsByFile, XMLEventReader xer, - XMLEvent xmlEvent, XMLEventWriter xew, int nbResponses) throws XMLStreamException { - while (xer.hasNext() && !(isEndElementWithName(xmlEvent,"SurveyUnits"))) { - // We exit the specified number of elements by file is reached - if (isEndElementWithName(xmlEvent, condition) && nbResponses >= nbElementsByFile) { - break; - } - xew.add(xmlEvent); - xmlEvent = xer.nextEvent(); - // We count the number of survey unit in the file - if (isStartElementWithName(condition, xmlEvent)) { - nbResponses++; - } - } - } - - private static void addHeadersToNewFile(List header, XMLEventWriter xew) throws XMLStreamException { - for (XMLEvent headerEvents : header) { - xew.add(headerEvents); - } - } - - private static boolean isStartElementWithName(String condition, XMLEvent xmlEvent) { - return xmlEvent.isStartElement() && xmlEvent.asStartElement().getName().getLocalPart().equals(condition); - } - - private static boolean isEndElementWithName(XMLEvent xmlEvent, String condition) { - return xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().getLocalPart().equals(condition); - } - - private static List getHeader(String xmlResource, String condition) throws FileNotFoundException, XMLStreamException { - XMLInputFactory xif = XMLInputFactory.newInstance(); - xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); - xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); - XMLEventReader xer = xif.createXMLEventReader(new FileReader(xmlResource)); - - List cachedXMLEvents = new ArrayList<>(); - while(xer.hasNext() && !xer.peek().isEndDocument()) { - XMLEvent xmlEvent = xer.nextTag(); - if (!xmlEvent.isStartElement()) { - break; - } - StartElement breakStartElement = xmlEvent.asStartElement(); - - cachedXMLEvents.add(breakStartElement); - xmlEvent = xer.nextEvent(); - while (!(xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().equals(breakStartElement.getName()))) { - if (isStartElementWithName(condition, xmlEvent)) { - xer.close(); - return cachedXMLEvents; - } - cachedXMLEvents.add(xmlEvent); - xmlEvent = xer.nextEvent(); - } - } - xer.close(); - return List.of(); - } - -} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/FileUtils.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/FileSystemImpl.java similarity index 57% rename from kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/FileUtils.java rename to kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/FileSystemImpl.java index 6b83ea3e..43e6218e 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/FileUtils.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/FileSystemImpl.java @@ -1,69 +1,41 @@ -package fr.insee.kraftwerk.core.utils; +package fr.insee.kraftwerk.core.utils.files; -import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.ModeInputs; import fr.insee.kraftwerk.core.inputs.UserInputs; import fr.insee.kraftwerk.core.inputs.UserInputsFile; +import fr.insee.kraftwerk.core.utils.DateUtils; +import lombok.NoArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.springframework.util.FileSystemUtils; +import org.apache.commons.io.FileUtils; +import javax.annotation.Nullable; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Stream; @Log4j2 -public class FileUtils { - - private FileUtils(){ - throw new IllegalStateException("Utility class"); - } +@NoArgsConstructor +public class FileSystemImpl implements FileUtilsInterface{ private static final String ARCHIVE = "Archive"; - /** - * Change /some/path/in/campaign-name to /some/path/out/campaign-name - */ - public static Path transformToOut(Path inDirectory) { - return transformToOther(inDirectory, "out"); - } - public static Path transformToOut(Path inDirectory, LocalDateTime localDateTime) { - return transformToOther(inDirectory, "out").resolve(localDateTime.format(DateTimeFormatter.ofPattern(Constants.OUTPUT_FOLDER_DATETIME_PATTERN))); - } - - /** - * Change /some/path/in/campaign-name to /some/path/temp/campaign-name - */ - public static Path transformToTemp(Path inDirectory) { - return transformToOther(inDirectory, "temp"); - } - /** - * Change /some/path/in/campaign-name to /some/path/__other__/campaign-name - */ - private static Path transformToOther(Path inDirectory, String other) { - return "in".equals(inDirectory.getFileName().toString()) ? inDirectory.getParent().resolve(other) - : transformToOther(inDirectory.getParent(),other).resolve(inDirectory.getFileName()); - } - - /** - * Move the input file to another directory to archive it - */ - public static void renameInputFile(Path inDirectory) { - + @Override + public void renameInputFile(Path inDirectory) { File file = inDirectory.resolve("kraftwerk.json").toFile(); String fileWithTime = "kraftwerk-" + DateUtils.getCurrentTimeStamp() + ".json"; File file2 = inDirectory.resolve(fileWithTime).toFile(); @@ -81,7 +53,8 @@ public static void renameInputFile(Path inDirectory) { } } - public static void archiveInputFiles(UserInputsFile userInputsFile) throws KraftwerkException { + @Override + public void archiveInputFiles(UserInputsFile userInputsFile) throws KraftwerkException { // Path inputFolder = userInputsFile.getInputDirectory(); String[] directories = inputFolder.toString().split(Pattern.quote(File.separator)); @@ -98,6 +71,149 @@ public static void archiveInputFiles(UserInputsFile userInputsFile) throws Kraft } } + @Override + public void deleteDirectory(Path directoryPath) throws KraftwerkException { + try { + org.springframework.util.FileSystemUtils.deleteRecursively(directoryPath); + } catch (IOException e) { + throw new KraftwerkException(500, "IOException when deleting temp folder : "+e.getMessage()); + } + } + + @Override + public List listFileNames(String dir) { + return Stream.of(new File(dir).listFiles()) + .filter(file -> !file.isDirectory()) + .map(File::getName) + .toList(); + } + + @Override + public List listFilePaths(String dir) { + return Stream.of(new File(dir).listFiles()) + .filter(file -> !file.isDirectory()) + .map(File::getAbsolutePath) + .toList(); + } + + @Override + public Path getTempVtlFilePath(UserInputs userInputs, String step, String dataset) { + createDirectoryIfNotExist(FileUtilsInterface.transformToTemp(userInputs.getInputDirectory())); + return FileUtilsInterface.transformToTemp(userInputs.getInputDirectory()).resolve(step+ dataset+".vtl"); + } + + @Override + public void createDirectoryIfNotExist(Path path) { + try { + Files.createDirectories(path); + log.info(String.format("Created folder: %s", path.toFile().getAbsolutePath())); + } catch (IOException e) { + log.error("Permission refused to create folder: " + path.getParent(), e); + } + } + + @Override + @Nullable + public Boolean isDirectory(String path) { + File file = new File(path); + if(file.isDirectory()){ + return true; + } + if(!Files.isRegularFile(Path.of(path))){ + return null; + } + return false; + } + + @Override + public long getSizeOf(String path) { + File file = new File(path); + return FileUtils.sizeOf(file); + } + + @Override + public void writeFile(String path, String toWrite, boolean replace) { + createDirectoryIfNotExist(Path.of(path).getParent()); + StandardOpenOption standardOpenOption = replace || !isFileExists(path) ? StandardOpenOption.CREATE : StandardOpenOption.APPEND; + try { + Files.write(Path.of(path), toWrite.getBytes(), standardOpenOption); + }catch (IOException e){ + log.error(e.toString()); + } + } + + @Override + public String findFile(String directory, String regex) throws KraftwerkException { + try (Stream files = Files.find(Path.of(directory), 1, (path, basicFileAttributes) -> path.toFile().getName().toLowerCase().matches(regex))) { + return files.findFirst() + .orElseThrow(() -> new KraftwerkException(404, "No file (%s) found in ".formatted(regex) + directory)).toString(); + }catch (IOException e){ + log.error(e.toString()); + return null; + } + } + + @Override + public InputStream readFile(String path) { + try{ + return Files.newInputStream(Path.of(path)); + }catch (IOException e){ + log.error(e.toString()); + return null; + } + } + + @Override + public boolean isFileExists(String path) { + return Files.exists(Path.of(path)); + } + + @Override + public void moveFile(String srcPath, String dstPath) throws KraftwerkException { + try { + createDirectoryIfNotExist(Path.of(dstPath).getParent()); + Files.move(Path.of(srcPath), Path.of(dstPath), StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new KraftwerkException(500, "Can't move file " + srcPath + " to " + dstPath); + } + } + + @Override + public void moveFile(Path fileSystemPath, String dstPath) throws KraftwerkException { + //Same than other moveFile + moveFile(fileSystemPath.toString(), dstPath); + } + + @Override + public Path convertToPath(String userField, Path inputDirectory) throws KraftwerkException { + if (userField != null && !"null".equals(userField) && !userField.isEmpty()) { + Path inputPath = inputDirectory.resolve(userField); + if (!new File(inputPath.toUri()).exists()) { + throw new KraftwerkException(400, String.format("The input folder \"%s\" does not exist in \"%s\".", userField, inputDirectory)); + } + return inputPath; + } else { + return null; + } + } + + @Override + public String convertToUrl(String userField, Path inputDirectory) { + if (userField == null) { + return null; + } + try { + if (userField.startsWith("http")) { + return new URI(userField).toURL().toString(); + } + return inputDirectory.resolve(userField).toFile().toString(); + } catch (MalformedURLException | URISyntaxException e) { + return null; + } + } + + // Utilities + /** * If reporting data, we move reporting data files * @param inputFolder @@ -105,7 +221,7 @@ public static void archiveInputFiles(UserInputsFile userInputsFile) throws Kraft * @param modeInputs * @throws KraftwerkException */ - private static void archiveReportingData(Path inputFolder, String campaignName, ModeInputs modeInputs) + private void archiveReportingData(Path inputFolder, String campaignName, ModeInputs modeInputs) throws KraftwerkException { if (modeInputs.getReportingDataFile() != null) { moveDirectory(modeInputs.getReportingDataFile().toFile(), inputFolder.resolve(ARCHIVE) @@ -120,7 +236,7 @@ private static void archiveReportingData(Path inputFolder, String campaignName, * @param modeInputs * @throws KraftwerkException */ - private static void archiveParadata(Path inputFolder, String campaignName, ModeInputs modeInputs) + private void archiveParadata(Path inputFolder, String campaignName, ModeInputs modeInputs) throws KraftwerkException { if (modeInputs.getParadataFolder() != null) { moveDirectory(modeInputs.getParadataFolder().toFile(), inputFolder.resolve(ARCHIVE) @@ -128,7 +244,7 @@ private static void archiveParadata(Path inputFolder, String campaignName, ModeI } } - private static void archiveData(Path inputFolder, String campaignName, ModeInputs modeInputs) + private void archiveData(Path inputFolder, String campaignName, ModeInputs modeInputs) throws KraftwerkException { Path dataPath = modeInputs.getDataFile(); Path newDataPath = inputFolder.resolve(ARCHIVE).resolve(getRoot(dataPath, campaignName)); @@ -137,7 +253,7 @@ private static void archiveData(Path inputFolder, String campaignName, ModeInput new File(newDataPath.getParent().toString()).mkdirs(); } if (Files.isRegularFile(dataPath)) { - moveFile(dataPath, newDataPath); + moveFile(dataPath.toString(), newDataPath.toString()); } else if (Files.isDirectory(dataPath)) { moveDirectory(dataPath.toFile(),newDataPath.toFile()); } else { @@ -145,22 +261,14 @@ private static void archiveData(Path inputFolder, String campaignName, ModeInput } } - private static void createArchiveDirectoryIfNotExists(Path inputFolder) { + private void createArchiveDirectoryIfNotExists(Path inputFolder) { if (!Files.exists(inputFolder.resolve(ARCHIVE))) { inputFolder.resolve(ARCHIVE).toFile().mkdir(); } } - private static void moveFile(Path dataPath, Path newDataPath) throws KraftwerkException { - try { - Files.move(dataPath, newDataPath); - } catch (IOException e) { - throw new KraftwerkException(500, "Can't move file " + dataPath + " to " + newDataPath); - } - } - - private static void moveDirectory(File sourceFile, File destFile) throws KraftwerkException { + private void moveDirectory(File sourceFile, File destFile) throws KraftwerkException { if (sourceFile.isDirectory()) { File[] files = sourceFile.listFiles(); assert files != null : "List of files in sourceFile is null"; @@ -178,8 +286,8 @@ private static void moveDirectory(File sourceFile, File destFile) throws Kraftwe throw new KraftwerkException(500, "Can't rename file " + sourceFile + " to " + destFile); } } - - private static String getRoot(Path path, String campaignName) { + + private String getRoot(Path path, String campaignName) { String[] directories = path.toString().split(Pattern.quote(File.separator)); int campaignIndex = Arrays.asList(directories).indexOf(campaignName); String[] newDirectories = Arrays.copyOfRange(directories, campaignIndex + 1, directories.length); @@ -191,68 +299,5 @@ private static String getRoot(Path path, String campaignName) { } return result.toString(); } - - - public static void deleteDirectory(Path directoryPath) throws KraftwerkException { - try { - FileSystemUtils.deleteRecursively(directoryPath); - } catch (IOException e) { - throw new KraftwerkException(500, "IOException when deleting temp folder : "+e.getMessage()); - } - } - - /** - * List the files in the directory - * @param dir - * @return - */ - public static List listFiles(String dir) { - return Stream.of(new File(dir).listFiles()) - .filter(file -> !file.isDirectory()) - .map(File::getName) - .toList(); - } - - public static Path getTempVtlFilePath(UserInputs userInputs, String step, String dataset) { - createDirectoryIfNotExist(FileUtils.transformToTemp(userInputs.getInputDirectory())); - return FileUtils.transformToTemp(userInputs.getInputDirectory()).resolve(step+ dataset+".vtl"); - } - - public static void createDirectoryIfNotExist(Path path) { - try { - Files.createDirectories(path); - log.info(String.format("Created folder: %s", path.toFile().getAbsolutePath())); - } catch (IOException e) { - log.error("Permission refused to create folder: " + path.getParent(), e); - } - } - public static Path convertToPath(String userField, Path inputDirectory) throws KraftwerkException { - if (userField != null && !"null".equals(userField) && !userField.isEmpty()) { - Path inputPath = inputDirectory.resolve(userField); - if (!new File(inputPath.toUri()).exists()) { - throw new KraftwerkException(400, String.format("The input folder \"%s\" does not exist in \"%s\".", userField, inputDirectory)); - } - return inputPath; - } else { - return null; - } - } - - public static URL convertToUrl(String userField, Path inputDirectory) { - if (userField == null) { - log.debug("null value out of method that reads DDI field (should not happen)."); - return null; - } - try { - if (userField.startsWith("http")) { - return new URI(userField).toURL(); - } - return inputDirectory.resolve(userField).toFile().toURI().toURL(); - } catch (MalformedURLException | URISyntaxException e) { - log.error("Unable to convert URL from user input: " + userField); - return null; - } - } - } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/FileSystemType.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/FileSystemType.java new file mode 100644 index 00000000..aa29915b --- /dev/null +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/FileSystemType.java @@ -0,0 +1,6 @@ +package fr.insee.kraftwerk.core.utils.files; + +public enum FileSystemType { + OS_FILESYSTEM, + MINIO +} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/FileUtilsInterface.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/FileUtilsInterface.java new file mode 100644 index 00000000..2bbc072f --- /dev/null +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/FileUtilsInterface.java @@ -0,0 +1,143 @@ +package fr.insee.kraftwerk.core.utils.files; + +import fr.insee.kraftwerk.core.Constants; +import fr.insee.kraftwerk.core.exceptions.KraftwerkException; +import fr.insee.kraftwerk.core.inputs.UserInputs; +import fr.insee.kraftwerk.core.inputs.UserInputsFile; + +import javax.annotation.Nullable; +import java.io.InputStream; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; + +/** + * Interface to use to interact with storage + */ +public interface FileUtilsInterface { + //Common methods + static Path transformToOut(Path inDirectory) { + return transformToOther(inDirectory, "out"); + } + + static Path transformToOut(Path inDirectory, LocalDateTime localDateTime) { + return transformToOther(inDirectory, "out").resolve(localDateTime.format(DateTimeFormatter.ofPattern(Constants.OUTPUT_FOLDER_DATETIME_PATTERN))); + } + + /** + * Change /some/path/in/campaign-name to /some/path/temp/campaign-name + */ + static Path transformToTemp(Path inDirectory) { + return transformToOther(inDirectory, "temp"); + } + + /** + * Change /some/path/in/campaign-name to /some/path/__other__/campaign-name + */ + private static Path transformToOther(Path inDirectory, String other) { + if(("in".equals(inDirectory.getFileName().toString()) || "specs".equals(inDirectory.getFileName().toString())) && inDirectory.getParent() != null){ + return inDirectory.getParent().resolve(other); + } + return "in".equals(inDirectory.getFileName().toString()) || "specs".equals(inDirectory.getFileName().toString()) ? Path.of(other) + : transformToOther(inDirectory.getParent(), other).resolve(inDirectory.getFileName()); + } + + //Methods to implement + //File system interactions + /** + * Read a file + * @param path path of file + * @return an InputStream of content of file + */ + InputStream readFile(String path); + /** + * Write string into a file + * @param path path of the file + * @param toWrite String to write + * @param replace true if overwrite, false if append + */ + void writeFile(String path, String toWrite, boolean replace); + /** + * Move file from source to destination (depends on the file system) + * @param srcPath source path + * @param dstPath destination path + * @throws KraftwerkException + */ + void moveFile(String srcPath, String dstPath) throws KraftwerkException; + /** + * Move file from local file system to another path + * @param fileSystemPath local path + * @param dstPath destination path + * @throws KraftwerkException + */ + void moveFile(Path fileSystemPath, String dstPath) throws KraftwerkException; + + /** + * Deletes a directory recursively + * @param directoryPath path of directory + * @throws KraftwerkException if exception + */ + void deleteDirectory(Path directoryPath) throws KraftwerkException; + /** + * Find the file in the folder of a campaign + * @param directory directory where the file should be + * @param fileRegex regex of the file to match + * @return Path of the file + * @throws KraftwerkException if no file found + */ + String findFile(String directory, String fileRegex) throws KraftwerkException; + + //File listing + /** + * List file names in a directory + * @param dir directory to list files from + * @return a list of file names + */ + List listFileNames(String dir); + + /** + * List file paths in a directory + * @param dir directory to list files from + * @return a list of file paths + */ + List listFilePaths(String dir); + + /** + * Create parent directories of the file + * @param path path of the file + */ + void createDirectoryIfNotExist(Path path); + + //Checks + /** + * Check if file exists + * @param path path of file + * @return true if file exists, false otherwise + */ + boolean isFileExists(String path); + /** + * Checks if path is a file or directory or neither + * @param path path of file + * @return true if directory, false if file, null if neither + */ + @Nullable + Boolean isDirectory(String path); + + /** + * Returns the size of a file + * @param path path of file + * @return size of file + */ + long getSizeOf(String path); + + //Misc. + Path convertToPath(String userField, Path inputDirectory) throws KraftwerkException; + String convertToUrl(String userField, Path inputDirectory); + /** + * Move the input file to another directory to archive it + */ + void renameInputFile(Path inDirectory); + void archiveInputFiles(UserInputsFile userInputsFile) throws KraftwerkException; + Path getTempVtlFilePath(UserInputs userInputs, String step, String dataset); +} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/MinioImpl.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/MinioImpl.java new file mode 100644 index 00000000..4adb2d4e --- /dev/null +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/files/MinioImpl.java @@ -0,0 +1,336 @@ +package fr.insee.kraftwerk.core.utils.files; + +import fr.insee.kraftwerk.core.exceptions.KraftwerkException; +import fr.insee.kraftwerk.core.inputs.ModeInputs; +import fr.insee.kraftwerk.core.inputs.UserInputs; +import fr.insee.kraftwerk.core.inputs.UserInputsFile; +import fr.insee.kraftwerk.core.utils.DateUtils; +import io.minio.CopyObjectArgs; +import io.minio.CopySource; +import io.minio.GetObjectArgs; +import io.minio.ListObjectsArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import io.minio.Result; +import io.minio.StatObjectArgs; +import io.minio.StatObjectResponse; +import io.minio.errors.ErrorResponseException; +import io.minio.messages.Item; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +@Slf4j +@AllArgsConstructor +public class MinioImpl implements FileUtilsInterface { + + private static final String ARCHIVE = "Archive"; + + MinioClient minioClient; + String bucketName; + + // Interface methods + + @Override + public void renameInputFile(Path inDirectory) { + String file1MinioPathString = inDirectory.resolve("/kraftwerk.json").toString(); + String fileWithTime = "kraftwerk-" + DateUtils.getCurrentTimeStamp() + ".json"; + String file2MinioPathString = inDirectory.resolve(fileWithTime).toString(); + if (isFileExists(file2MinioPathString)) { + log.warn(String.format("Trying to rename '%s' to '%s', but second file already exists.", Path.of(file1MinioPathString).getFileName(), Path.of(file2MinioPathString).getFileName())); + log.warn("Timestamped input file will be over-written."); + deleteFile(file2MinioPathString); + } + try { + moveFile(file1MinioPathString, file2MinioPathString); + }catch (Exception e){ + log.error(e.toString()); + } + } + + @Override + public void archiveInputFiles(UserInputsFile userInputsFile) throws KraftwerkException { + // + Path inputFolder = userInputsFile.getInputDirectory(); + String[] directories = inputFolder.toString().split("/"); + String campaignName = directories[directories.length - 1]; + + // + for (String mode : userInputsFile.getModes()) { + ModeInputs modeInputs = userInputsFile.getModeInputs(mode); + archiveData(inputFolder, campaignName, modeInputs); + archiveParadata(inputFolder, campaignName, modeInputs); + archiveReportingData(inputFolder, campaignName, modeInputs); + } + } + + @Override + public void deleteDirectory(Path directoryPath) throws KraftwerkException { + try { + for (String filePath : listFileNames(directoryPath.toString())) { + deleteFile(filePath); + } + } catch (Exception e) { + log.error(e.toString()); + } + } + + @Override + public List listFileNames(String dir) { + try { + ArrayList filePaths = new ArrayList<>(); + Iterable> results = minioClient.listObjects( + ListObjectsArgs.builder().bucket(bucketName).prefix(dir.replace("\\","/")).recursive(true).build()); + + for (Result result : results) { + filePaths.add(result.get().objectName()); + } + return filePaths; + } catch (Exception e) { + log.error(e.toString()); + return null; + } + } + + @Override + public List listFilePaths(String dir) { + return listFileNames(dir.replace("\\","/")); + } + + @Override + public void createDirectoryIfNotExist(Path path) { + //MinIO creates automatically the folders + } + + @Override + public Path getTempVtlFilePath(UserInputs userInputs, String step, String dataset) { + return FileUtilsInterface.transformToTemp(userInputs.getInputDirectory()).resolve(step+ dataset+".vtl"); + } + + @Override + public Boolean isDirectory(String path){ + try { + //List files of parent to check if directory + Iterable> results = minioClient.listObjects( + ListObjectsArgs.builder().bucket(bucketName).prefix(Path.of(path).getParent().toString().replace("\\","/")).recursive(true).build()); + + for (Result result : results) { + if(result.get().objectName().startsWith(path)){ + return true; + } + } + log.warn("S3 File or folder {} not found in {}", Path.of(path).getFileName().toString().replace("\\","/"), Path.of(path).getParent().toString().replace("\\","/")); + return null; + } catch (Exception e) { + log.error(e.toString()); + return null; + } + } + + @Override + public long getSizeOf(String path) { + try { + StatObjectResponse objectStat = minioClient.statObject(StatObjectArgs.builder().bucket(bucketName).object(path.replace("\\","/")).build()); + return objectStat.size(); + }catch (Exception e){ + log.error(e.toString()); + return 0; + } + } + + @Override + public void writeFile(String path, String toWrite, boolean replace) { + InputStream inputStream = new ByteArrayInputStream(toWrite.getBytes()); + writeFileOnMinio(path.replace("\\","/"), inputStream, toWrite.length()); + } + + @Override + public String findFile(String directory, String fileRegex) throws KraftwerkException { + //Stream of files with filename matching fileRegex + try (Stream files = listFileNames(directory.replace("\\","/")) + .stream().filter(s -> s.split("/")[s.split("/").length - 1].matches(fileRegex))) { + return files.findFirst() + .orElseThrow(() -> new KraftwerkException(404, "No file with regex " + fileRegex + " found in " + directory.replace("\\","/"))); + } + } + + @Override + public InputStream readFile(String minioPath) { + try { + return minioClient.getObject(GetObjectArgs.builder().bucket(bucketName).object(minioPath.replace("\\","/")).build()); + } catch (Exception e) { + log.error(e.toString()); + return null; + } + } + + @Override + public boolean isFileExists(String objectPath) { + try { + minioClient.statObject(StatObjectArgs.builder().bucket(bucketName).object(objectPath.replace("\\","/")).build()); + return true; + } catch (ErrorResponseException e) { + return false; + } catch (Exception e) { + log.error(e.toString()); + return false; + } + } + + @Override + public void moveFile(String srcMinioPath, String dstMinioPath) throws KraftwerkException { + try { + copyFile(srcMinioPath.replace("\\","/"), dstMinioPath.replace("\\","/")); + deleteFile(srcMinioPath); + } catch (Exception e) { + throw new KraftwerkException(500, "Can't move file " + srcMinioPath.replace("\\","/") + " to " + dstMinioPath.replace("\\","/")); + } + } + + @Override + public void moveFile(Path fileSystemPath, String dstMinioPath) throws KraftwerkException { + try (InputStream inputStream = new FileInputStream(fileSystemPath.toFile())){ + writeFileOnMinio(dstMinioPath.replace("\\","/"), inputStream, Files.size(fileSystemPath)); + } catch (Exception e) { + throw new KraftwerkException(500, "Can't move file " + fileSystemPath + " to " + dstMinioPath.replace("\\","/")); + } + try { + Files.deleteIfExists(fileSystemPath); + }catch (Exception e) { + log.error("Error during file system file deletion : " + e); + } + } + + @Override + public Path convertToPath(String userField, Path inputDirectory) throws KraftwerkException { + if (userField != null && !"null".equals(userField) && !userField.isEmpty()) { + Path inputPath = inputDirectory.resolve(userField); + if (Boolean.FALSE.equals(isDirectory(inputPath.toString().replace("\\","/"))) + || isDirectory(inputPath.toString().replace("\\","/")) == null + ) { + throw new KraftwerkException(400, String.format("The input folder \"%s\" does not exist in \"%s\".", userField, inputDirectory)); + } + return inputPath; + } else { + return null; + } + } + + @Override + public String convertToUrl(String userField, Path inputDirectory) { + if (userField == null) { + return null; + } + try { + if (userField.startsWith("http")) { + return new URI(userField).toURL().toString(); + } + return inputDirectory.resolve(userField).toString(); + } catch (MalformedURLException | URISyntaxException e) { + return null; + } + } + + //Utilities + + private void writeFileOnMinio(String minioPath, InputStream inputStream, long fileSize) { + try { + minioClient.putObject(PutObjectArgs.builder().bucket(bucketName).stream(inputStream, fileSize, -1).object(minioPath.replace("\\","/")).build()); + } catch (Exception e) { + log.error(e.toString()); + } + } + + private void copyFile(String srcMinioPath, String dstMinioPath) { + try { + CopySource copySource = CopySource.builder().bucket(bucketName).object(srcMinioPath.replace("\\","/")).build(); + minioClient.copyObject(CopyObjectArgs.builder().bucket(bucketName).object(dstMinioPath).source(copySource).build()); + } catch (Exception e) { + log.error(e.toString()); + } + } + + private void moveDirectory(String srcMinioPath, String dstMinioPath) { + try { + for (String filePath : listFileNames(srcMinioPath.replace("\\","/"))) { + moveFile(filePath, dstMinioPath + "/" + extractFileName(filePath)); + } + } catch (Exception e) { + log.error(e.toString()); + } + } + + private String extractFileName(String filePath) { + if (filePath == null || filePath.isEmpty()) { + return ""; + } + return Path.of(filePath).getFileName().toString().replace("\\","/"); + } + + private void deleteFile(String minioPath) { + try { + minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(minioPath.replace("\\","/")).build()); + } catch (Exception e) { + log.error(e.toString()); + } + } + + + + private void archiveData(Path inputFolder, String campaignName, ModeInputs modeInputs) throws KraftwerkException{ + Path dataPath = modeInputs.getDataFile(); + Path newDataPath = inputFolder.resolve(ARCHIVE).resolve(getRoot(dataPath, campaignName)); + + if (!isDirectory(dataPath.toString())) { + moveFile(dataPath.toString(), newDataPath.toString()); + } else { + moveDirectory(dataPath.toString(), newDataPath.toString()); + } + } + + /** + * If paradata, we move the paradata folder + */ + private void archiveParadata(Path inputFolder, String campaignName, ModeInputs modeInputs){ + if (modeInputs.getParadataFolder() != null) { + moveDirectory(modeInputs.getParadataFolder().toString(), inputFolder.resolve(ARCHIVE) + .resolve(getRoot(modeInputs.getParadataFolder(), campaignName)).toString().replace("\\","/")); + } + } + + /** + * If reporting data, we move reporting data files + */ + private void archiveReportingData(Path inputFolder, String campaignName, ModeInputs modeInputs){ + if (modeInputs.getReportingDataFile() != null) { + moveDirectory(modeInputs.getReportingDataFile().toString(), inputFolder.resolve(ARCHIVE) + .resolve(getRoot(modeInputs.getReportingDataFile(), campaignName)).toString().replace("\\","/")); + } + } + + private String getRoot(Path path, String campaignName) { + String[] directories = path.toString().split("/"); + int campaignIndex = Arrays.asList(directories).indexOf(campaignName); + String[] newDirectories = Arrays.copyOfRange(directories, campaignIndex + 1, directories.length); + StringBuilder result = new StringBuilder(); + String sep = ""; + for (String directory : newDirectories) { + result.append(sep).append(directory); + sep = "/"; + } + return result.toString().replace("\\","/"); + } +} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/XmlFileReader.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xml/XmlFileReader.java similarity index 73% rename from kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/XmlFileReader.java rename to kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xml/XmlFileReader.java index ff41b980..52eebf8f 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/XmlFileReader.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xml/XmlFileReader.java @@ -1,9 +1,10 @@ -package fr.insee.kraftwerk.core.utils; +package fr.insee.kraftwerk.core.utils.xml; -import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import nu.xom.Builder; import nu.xom.Document; @@ -18,12 +19,19 @@ public class XmlFileReader { private boolean xsdValidation = false; + private final FileUtilsInterface fileUtilsInterface; + + public XmlFileReader(FileUtilsInterface fileUtilsInterface) { + this.fileUtilsInterface = fileUtilsInterface; + } public Document readXmlFile(Path filePath) { - try { - File file = filePath.toFile(); + try (InputStream inputStream = fileUtilsInterface.readFile(filePath.toString())){ + if(inputStream == null){ + return null; + } Builder parser = new Builder(xsdValidation); - return parser.build(file); + return parser.build(inputStream); } catch (ValidityException ex) { log.warn("XSD validation error.", ex); log.warn("See following INFO log for details."); diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xml/XmlSplitter.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xml/XmlSplitter.java new file mode 100644 index 00000000..1870e04b --- /dev/null +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xml/XmlSplitter.java @@ -0,0 +1,141 @@ +package fr.insee.kraftwerk.core.utils.xml; + +import fr.insee.kraftwerk.core.exceptions.KraftwerkException; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; + +import javax.xml.stream.*; +import javax.xml.stream.events.EndDocument; +import javax.xml.stream.events.StartDocument; +import javax.xml.stream.events.StartElement; +import javax.xml.stream.events.XMLEvent; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +public class XmlSplitter { + + // We use StAX in this class to deal with memory issues on huge XML files + private XmlSplitter() { + throw new IllegalStateException("Utility class"); + } + + public static void split(String inputfolder, String xmlfile, String outputFolder, String condition, int nbElementsByFile, FileUtilsInterface fileUtilsInterface) throws XMLStreamException, IOException, KraftwerkException { + String xmlResource = inputfolder + xmlfile; + List header = getHeader(xmlResource, condition, fileUtilsInterface); + + XMLEventFactory xef = XMLEventFactory.newFactory(); + XMLInputFactory xif = XMLInputFactory.newInstance(); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + + try (InputStream inputStream = fileUtilsInterface.readFile(xmlResource)) { + XMLEventReader xer = xif.createXMLEventReader(inputStream); + StartElement rootStartElement = xer.nextTag().asStartElement(); + StartDocument startDocument = xef.createStartDocument(); + EndDocument endDocument = xef.createEndDocument(); + + XMLOutputFactory xof = XMLOutputFactory.newFactory(); + int fileCount = 1; + while (xer.hasNext() && !xer.peek().isEndDocument()) { + XMLEvent xmlEvent = xer.nextEvent(); + + if (isStartElementWithName(condition, xmlEvent)) { + // Create a temp file for the fragment, the name is derived from the value of the id attribute + String tempFileName = "split" + fileCount; + File tempFile = File.createTempFile(outputFolder + tempFileName,null); + FileWriter fileWriter = new FileWriter(tempFile); + + // A StAX XMLEventWriter will be used to write the XML fragment + XMLEventWriter xew = xof.createXMLEventWriter(fileWriter); + xew.add(startDocument); + + // Add the elements which are common to all split files + addHeadersToNewFile(header, xew); + + // Write the XMLEvents that are part of SurveyUnit element + xew.add(xmlEvent); + xmlEvent = xer.nextEvent(); + int nbResponses = 1; + // We loop until we reach the end tag Survey units indicating the near end of the document + iterateOnSurveyUnits(condition, nbElementsByFile, xer, xmlEvent, xew, nbResponses); + + // Write the file, close everything we opened and + xew.add(xef.createEndElement(rootStartElement.getName(), null)); + xew.add(endDocument); + fileWriter.close(); + + //Move to outputFolder + fileUtilsInterface.moveFile(tempFile.getAbsolutePath(), outputFolder + "/" + tempFileName + ".xml"); + + //Update the file's counter + fileCount++; + } + } + } + } + + private static void iterateOnSurveyUnits(String condition, int nbElementsByFile, XMLEventReader xer, + XMLEvent xmlEvent, XMLEventWriter xew, int nbResponses) throws XMLStreamException { + while (xer.hasNext() && !(isEndElementWithName(xmlEvent,"SurveyUnits"))) { + // We exit the specified number of elements by file is reached + if (isEndElementWithName(xmlEvent, condition) && nbResponses >= nbElementsByFile) { + break; + } + xew.add(xmlEvent); + xmlEvent = xer.nextEvent(); + // We count the number of survey unit in the file + if (isStartElementWithName(condition, xmlEvent)) { + nbResponses++; + } + } + } + + private static void addHeadersToNewFile(List header, XMLEventWriter xew) throws XMLStreamException { + for (XMLEvent headerEvents : header) { + xew.add(headerEvents); + } + } + + private static boolean isStartElementWithName(String condition, XMLEvent xmlEvent) { + return xmlEvent.isStartElement() && xmlEvent.asStartElement().getName().getLocalPart().equals(condition); + } + + private static boolean isEndElementWithName(XMLEvent xmlEvent, String condition) { + return xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().getLocalPart().equals(condition); + } + + private static List getHeader(String xmlResource, String condition, FileUtilsInterface fileUtilsInterface) throws IOException, XMLStreamException { + try(InputStream inputStream = fileUtilsInterface.readFile(xmlResource)) { + XMLInputFactory xif = XMLInputFactory.newInstance(); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + XMLEventReader xer = xif.createXMLEventReader(inputStream); + + List cachedXMLEvents = new ArrayList<>(); + while (xer.hasNext() && !xer.peek().isEndDocument()) { + XMLEvent xmlEvent = xer.nextTag(); + if (!xmlEvent.isStartElement()) { + break; + } + StartElement breakStartElement = xmlEvent.asStartElement(); + + cachedXMLEvents.add(breakStartElement); + xmlEvent = xer.nextEvent(); + while (!(xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().equals(breakStartElement.getName()))) { + if (isStartElementWithName(condition, xmlEvent)) { + xer.close(); + return cachedXMLEvents; + } + cachedXMLEvents.add(xmlEvent); + xmlEvent = xer.nextEvent(); + } + } + xer.close(); + } + return List.of(); + } + +} diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformer.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformer.java index 99def2a6..5267ae5f 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformer.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformer.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.nio.file.Path; @@ -13,34 +14,37 @@ import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import lombok.AllArgsConstructor; import lombok.extern.log4j.Log4j2; @Log4j2 +@AllArgsConstructor public class SaxonTransformer { + private FileUtilsInterface fileUtilsInterface; /** * XSL transformation method using Saxon. * - * @param inputXmlURL : URL of the XML input file + * @param inputXmlPath : Path of the XML input file * @param inputXslPath : Path to the XSL file from the resources folder of the * application * @param outputXmlPath : Path to the XML output file which will be created */ - public void xslTransform(URL inputXmlURL, String inputXslPath, Path outputXmlPath) { - log.info("About to transform the file from URL: " + inputXmlURL); + public void xslTransform(String inputXmlPath, String inputXslPath, Path outputXmlPath) throws IOException { + log.info("About to transform the file: " + inputXmlPath); log.info("using the XSL file " + inputXslPath); // Get the XML input file StreamSource xmlSource; InputStream xmlInput; - try { - xmlInput = inputXmlURL.openStream(); - xmlSource = new StreamSource(xmlInput); - xmlSource.setSystemId(inputXmlURL.toString()); - } catch (IOException e) { - log.error(String.format("IOException when trying to read file from URL: %s", inputXmlURL), e); - return; // to break here if the xml input file is not found + if(inputXmlPath.startsWith("http")){ + xmlInput = URI.create(inputXmlPath).toURL().openStream(); + }else{ + xmlInput = fileUtilsInterface.readFile(inputXmlPath); } + xmlSource = new StreamSource(xmlInput); + xmlSource.setSystemId(inputXmlPath); // Get the XSL file StreamSource xslSource; @@ -82,17 +86,10 @@ public void xslTransform(URL inputXmlURL, String inputXslPath, Path outputXmlPat * application * @param outputXmlPath : Path to the XML output file which will be created */ - public void xslTransform(Path inputXmlPath, String inputXslPath, Path outputXmlPath) { + public void xslTransform(Path inputXmlPath, String inputXslPath, Path outputXmlPath) throws IOException { log.info("About to transform the file " + inputXmlPath); log.info("using the XSL file " + inputXslPath); - try { - URL inputXmlUrl = inputXmlPath.toUri().toURL(); - xslTransform(inputXmlUrl, inputXslPath, outputXmlPath); - } catch (MalformedURLException e) { - log.error(String.format("Error when converting file path '%s' to an URL.", inputXmlPath), e); - } - + xslTransform(inputXmlPath.toString(), inputXslPath, outputXmlPath); } - } \ No newline at end of file diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlExecute.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlExecute.java index 51b540d5..6e0128e0 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlExecute.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/vtl/VtlExecute.java @@ -5,8 +5,9 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.TextFileWriter; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.vtl.jackson.TrevasModule; import fr.insee.vtl.model.Dataset; import lombok.extern.log4j.Log4j2; @@ -36,14 +37,16 @@ public class VtlExecute { private final ObjectMapper mapper; /** Engine that will execute VTL instructions */ private final ScriptEngine engine; + private final FileUtilsInterface fileUtilsInterface; - public VtlExecute(){ + public VtlExecute(FileUtilsInterface fileUtilsInterface){ mapper = new ObjectMapper(); mapper.registerModule(new TrevasModule()); // Engine engine = new ScriptEngineManager() .getEngineByName("vtl"); + this.fileUtilsInterface = fileUtilsInterface; } /** @@ -118,12 +121,12 @@ public void putVtlDataset(String filePath, String bindingName, VtlBindings bindi * @param jsonOutFile Path to write the output json file. * */ public void writeJsonDataset(String bindingName, Path jsonOutFile, VtlBindings bindings) { - FileUtils.createDirectoryIfNotExist(jsonOutFile.getParent()); + fileUtilsInterface.createDirectoryIfNotExist(jsonOutFile.getParent()); //Write file if (bindings.containsKey(bindingName)) { try { - TextFileWriter.writeFile(jsonOutFile, mapper.writeValueAsString(bindings.getDataset(bindingName))); + TextFileWriter.writeFile(jsonOutFile, mapper.writeValueAsString(bindings.getDataset(bindingName)), fileUtilsInterface); } catch (JsonProcessingException e) { log.debug(String.format("Unable to serialize dataset stored under name '%s'.", bindingName), e); } diff --git a/kraftwerk-core/src/main/resources/application.properties b/kraftwerk-core/src/main/resources/application.properties index f9107f66..e4cbcfe2 100644 --- a/kraftwerk-core/src/main/resources/application.properties +++ b/kraftwerk-core/src/main/resources/application.properties @@ -1,4 +1,8 @@ # LOGS logging.level.root= INFO logging.level.fr.insee.kraftwerk=INFO -simpleLogger.defaultLogLevel=INFO \ No newline at end of file +simpleLogger.defaultLogLevel=INFO + +# S3 +# Disabled by default +fr.insee.postcollecte.minio.enable = false \ No newline at end of file diff --git a/kraftwerk-core/src/test/java/cucumber/unit_tests/AggregateDefinitions.java b/kraftwerk-core/src/test/java/cucumber/unit_tests/AggregateDefinitions.java index f7ea2db3..00ed2efb 100644 --- a/kraftwerk-core/src/test/java/cucumber/unit_tests/AggregateDefinitions.java +++ b/kraftwerk-core/src/test/java/cucumber/unit_tests/AggregateDefinitions.java @@ -7,6 +7,7 @@ import fr.insee.kraftwerk.core.dataprocessing.ReconciliationProcessing; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import io.cucumber.java.en.Given; @@ -25,7 +26,7 @@ public class AggregateDefinitions { public List errors = new ArrayList<>(); - VtlExecute vtlExecute = new VtlExecute(); + VtlExecute vtlExecute = new VtlExecute(new FileSystemImpl()); @Given("We have some VTLBindings named {string} and {string}") public void initialize(String firstDataset, String secondDataset){ @@ -35,9 +36,9 @@ public void initialize(String firstDataset, String secondDataset){ vtlExecute.convertToVtlDataset(fakeCawiData, firstDataset, vtlBindings); vtlExecute.convertToVtlDataset(fakePapiData, secondDataset, vtlBindings); // add group prefixes - GroupProcessing groupProcessing = new GroupProcessing(vtlBindings, fakeCawiData.getMetadataModel()); + GroupProcessing groupProcessing = new GroupProcessing(vtlBindings, fakeCawiData.getMetadataModel(), new FileSystemImpl()); groupProcessing.applyVtlTransformations(firstDataset, null,errors); - GroupProcessing groupProcessing2 = new GroupProcessing(vtlBindings, fakePapiData.getMetadataModel()); + GroupProcessing groupProcessing2 = new GroupProcessing(vtlBindings, fakePapiData.getMetadataModel(), new FileSystemImpl()); groupProcessing2.applyVtlTransformations(secondDataset, null,errors); // @@ -47,7 +48,7 @@ public void initialize(String firstDataset, String secondDataset){ @When("I try to aggregate the bindings") public void collect_variables() { - DataProcessing reconciliationProcessing = new ReconciliationProcessing(vtlBindings); + DataProcessing reconciliationProcessing = new ReconciliationProcessing(vtlBindings, new FileSystemImpl()); reconciliationProcessing.applyVtlTransformations( "MULTIMODE", null,errors); } diff --git a/kraftwerk-core/src/test/java/cucumber/unit_tests/EvalScriptDefinitions.java b/kraftwerk-core/src/test/java/cucumber/unit_tests/EvalScriptDefinitions.java index d80db367..02108eef 100644 --- a/kraftwerk-core/src/test/java/cucumber/unit_tests/EvalScriptDefinitions.java +++ b/kraftwerk-core/src/test/java/cucumber/unit_tests/EvalScriptDefinitions.java @@ -8,6 +8,7 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import io.cucumber.java.en.Given; @@ -18,7 +19,7 @@ public class EvalScriptDefinitions { public VtlBindings vtlBindings = new VtlBindings(); - VtlExecute vtlExecute = new VtlExecute(); + VtlExecute vtlExecute = new VtlExecute(new FileSystemImpl()); @Given("We have some simple VTLBindings") public void initialize() throws Exception { diff --git a/kraftwerk-core/src/test/java/cucumber/unit_tests/ExportDatasetDefinitions.java b/kraftwerk-core/src/test/java/cucumber/unit_tests/ExportDatasetDefinitions.java index 2ff6c225..7e4c9adc 100644 --- a/kraftwerk-core/src/test/java/cucumber/unit_tests/ExportDatasetDefinitions.java +++ b/kraftwerk-core/src/test/java/cucumber/unit_tests/ExportDatasetDefinitions.java @@ -4,6 +4,7 @@ import fr.insee.kraftwerk.core.dataprocessing.GroupProcessing; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import fr.insee.kraftwerk.core.vtl.VtlJsonDatasetWriter; @@ -22,7 +23,7 @@ public class ExportDatasetDefinitions { public String tempDatasetPath = ""; public SurveyRawData survey = null; - VtlExecute vtlExecute = new VtlExecute(); + VtlExecute vtlExecute = new VtlExecute(new FileSystemImpl()); @Given("We have some SurveyRawData named {string}") public void initialize(String nameDataset) throws Exception { @@ -44,7 +45,7 @@ public void exportDataset(String nameDataset) throws Exception { public void importDataset(String nameDataset) throws Exception { vtlExecute.putVtlDataset(tempDatasetPath, "OUTPUT_TEST_EXPORT", vtlBindings); // add group prefixes - GroupProcessing groupProcessing = new GroupProcessing(vtlBindings, survey.getMetadataModel()); + GroupProcessing groupProcessing = new GroupProcessing(vtlBindings, survey.getMetadataModel(), new FileSystemImpl()); groupProcessing.applyVtlTransformations("OUTPUT_TEST_EXPORT", null, new ArrayList<>()); } diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java index d9eca49a..ba51b4c5 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CalculatedProcessingTest.java @@ -6,6 +6,8 @@ import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.Variable; import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlScript; import fr.insee.vtl.model.Dataset; @@ -27,6 +29,7 @@ class CalculatedProcessingTest { private static MetadataModel fooMetadataModel; private static VtlBindings vtlBindings; private static List errors; + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); @BeforeAll static void setFooCalculated() { @@ -52,7 +55,7 @@ static void setFooCalculated() { @Test void testIfCalculatedAreCorrectlyResolved() { // - CalculatedProcessing processing = new CalculatedProcessing(vtlBindings, fooCalculated); + CalculatedProcessing processing = new CalculatedProcessing(vtlBindings, fooCalculated, fileUtilsInterface); VtlScript vtlScript = processing.generateVtlInstructions("TEST"); // @@ -68,7 +71,7 @@ void testIfCalculatedAreProcessed() { // VtlBindings vtlBindings = getVtlBindings(); // - CalculatedProcessing processing = new CalculatedProcessing(vtlBindings, fooCalculated); + CalculatedProcessing processing = new CalculatedProcessing(vtlBindings, fooCalculated, fileUtilsInterface); processing.applyAutomatedVtlInstructions("TEST", errors); // Dataset outDataset = vtlBindings.getDataset("TEST"); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java index d7887cf2..223b0728 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/CleanUpProcessingTest.java @@ -3,6 +3,8 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.metadata.*; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; @@ -17,6 +19,8 @@ class CleanUpProcessingTest { + FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + Dataset cawiDataset = new InMemoryDataset( List.of( List.of("T01", "foo", "2"), @@ -95,7 +99,7 @@ void applyCleanUp() { vtlBindings.put("MULTIMODE", multimodeDataset); // Apply clean up - CleanUpProcessing cleanUpProcessing = new CleanUpProcessing(vtlBindings, metadatas); + CleanUpProcessing cleanUpProcessing = new CleanUpProcessing(vtlBindings, metadatas, fileUtilsInterface); cleanUpProcessing.applyVtlTransformations("MULTIMODE", null,errors); // Are paper indicator variables removed in VTL multimode dataset ? diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java index 309c1d91..bc7a3eed 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/GroupProcessingTest.java @@ -2,6 +2,8 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.metadata.*; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; @@ -16,6 +18,8 @@ class GroupProcessingTest { + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + @Test void addPrefixes() { // @@ -41,7 +45,7 @@ void addPrefixes() { metadata.getVariables().putVariable(new Variable("FOO1", metadata.getGroup("DEPTH1"), VariableType.NUMBER)); metadata.getVariables().putVariable(new Variable("FOO2", metadata.getGroup("DEPTH2"), VariableType.NUMBER)); // - new GroupProcessing(vtlBindings, metadata).applyAutomatedVtlInstructions("TEST",errors); + new GroupProcessing(vtlBindings, metadata, fileUtilsInterface).applyAutomatedVtlInstructions("TEST",errors); Dataset outDataset = vtlBindings.getDataset("TEST"); // diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsTest.java index 661c848c..c347bbf8 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/InformationLevelsTest.java @@ -8,6 +8,8 @@ import java.util.Map; import java.util.Set; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.junit.jupiter.api.Test; import fr.insee.kraftwerk.core.Constants; @@ -18,7 +20,7 @@ import fr.insee.vtl.model.InMemoryDataset; class InformationLevelsTest { - + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); private final String rootId = Constants.ROOT_IDENTIFIER_NAME; Dataset testDataset = new InMemoryDataset( @@ -52,7 +54,7 @@ void applyInformationLevelsProcessing() { VtlBindings vtlBindings = new VtlBindings(); vtlBindings.put("MULTIMODE", testDataset); // - InformationLevelsProcessing processing = new InformationLevelsProcessing(vtlBindings); + InformationLevelsProcessing processing = new InformationLevelsProcessing(vtlBindings, fileUtilsInterface); processing.applyAutomatedVtlInstructions("MULTIMODE",errors); // Dataset rootDataset = vtlBindings.getDataset(Constants.ROOT_GROUP_NAME); @@ -92,7 +94,7 @@ void applyInformationLevelsProcessingWithPartialIdNUll() { VtlBindings vtlBindings = new VtlBindings(); vtlBindings.put("MULTIMODE", testDatasetWithPartialIdNull); // - InformationLevelsProcessing processing = new InformationLevelsProcessing(vtlBindings); + InformationLevelsProcessing processing = new InformationLevelsProcessing(vtlBindings, fileUtilsInterface); processing.applyAutomatedVtlInstructions("MULTIMODE",errors); // Dataset rootDataset = vtlBindings.getDataset(Constants.ROOT_GROUP_NAME); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java index 936ec8a7..d118a392 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/PaperDataProcessingTest.java @@ -2,6 +2,8 @@ import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.metadata.*; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; @@ -16,7 +18,7 @@ import static org.junit.Assert.assertNotNull; class PaperDataProcessingTest { - + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); Dataset paperDataset = new InMemoryDataset( List.of( // Correct answers @@ -70,7 +72,7 @@ void testPaperDataProcessing() { VtlBindings vtlBindings = new VtlBindings(); vtlBindings.put("TEST", paperDataset); // - PaperDataProcessing paperDataProcessing = new PaperDataProcessing(vtlBindings, metadataModel); + PaperDataProcessing paperDataProcessing = new PaperDataProcessing(vtlBindings, metadataModel, fileUtilsInterface); paperDataProcessing.applyAutomatedVtlInstructions("TEST", errors); // Dataset paperDsModified = vtlBindings.getDataset("TEST"); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationTest.java index 1c45c0d2..58bf71c0 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/ReconciliationTest.java @@ -2,6 +2,8 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; @@ -23,6 +25,7 @@ class ReconciliationTest { private VtlBindings vtlBindings; private List errors; + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); InMemoryDataset capiDataset = new InMemoryDataset( List.of( @@ -80,7 +83,7 @@ void applyReconciliation_singleMode(String dsName) { // vtlBindings.put("SINGLE_MODE", testDatasets.get(dsName)); // - ReconciliationProcessing reconciliation = new ReconciliationProcessing(vtlBindings); + ReconciliationProcessing reconciliation = new ReconciliationProcessing(vtlBindings, fileUtilsInterface); reconciliation.applyVtlTransformations("MULTIMODE", null,errors); // Dataset multimodeDataset = vtlBindings.getDataset("MULTIMODE"); @@ -92,7 +95,7 @@ private Dataset applyReconciliation_twoModes(String mode1, String mode2) { vtlBindings.put(mode1, testDatasets.get(mode1)); vtlBindings.put(mode2, testDatasets.get(mode2)); // - ReconciliationProcessing reconciliation = new ReconciliationProcessing(vtlBindings); + ReconciliationProcessing reconciliation = new ReconciliationProcessing(vtlBindings, fileUtilsInterface); reconciliation.applyVtlTransformations("MULTIMODE", null,errors); // return vtlBindings.getDataset("MULTIMODE"); @@ -136,7 +139,7 @@ void applyReconciliation_threeModes() { vtlBindings.put("CAWI", capiDataset); vtlBindings.put("PAPI", papiDataset); // - ReconciliationProcessing reconciliation = new ReconciliationProcessing(vtlBindings); + ReconciliationProcessing reconciliation = new ReconciliationProcessing(vtlBindings, fileUtilsInterface); reconciliation.applyVtlTransformations("MULTIMODE", null,errors); // Dataset multimodeDataset = vtlBindings.getDataset("MULTIMODE"); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java index 60196fad..9e40b6f4 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/dataprocessing/TCMSequenceProcessingTest.java @@ -4,6 +4,8 @@ import fr.insee.kraftwerk.core.TestConstants; import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.Sequence; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; @@ -26,6 +28,8 @@ class TCMSequenceProcessingTest { static final String FORMAT_INSTRUCTION = "/* Instruction %s */"; static final List modules = new ArrayList<>(); + FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + //Given static Dataset unimodalDataset = new InMemoryDataset( List.of( @@ -77,7 +81,7 @@ void check_standard_vtl_execution(){ String expectedScript = expectedScriptBuilder.toString(); //WHEN - TCMSequencesProcessing processing = new TCMSequencesProcessing(vtlBindings, metadataModel, Path.of(TestConstants.UNIT_TESTS_DIRECTORY).resolve("vtl").toString()); + TCMSequencesProcessing processing = new TCMSequencesProcessing(vtlBindings, metadataModel, Path.of(TestConstants.UNIT_TESTS_DIRECTORY).resolve("vtl").toString(), fileUtilsInterface); String scriptString = processing.applyAutomatedVtlInstructions("TEST", errors); //THEN diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java index 27d580e9..ec501205 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java @@ -7,6 +7,8 @@ import fr.insee.kraftwerk.core.extradata.reportingdata.ReportingDataUE; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -16,9 +18,11 @@ class CSVReportingDataParserTest { + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + @Test void parseReportingDataTest() { - CSVReportingDataParser csvReportingDataParser = new CSVReportingDataParser(); + CSVReportingDataParser csvReportingDataParser = new CSVReportingDataParser(fileUtilsInterface); SurveyRawData data = SurveyRawDataTest.createFakePapiSurveyRawData(); ReportingData reportingData = new ReportingData( @@ -48,7 +52,7 @@ void parseReportingDataTest() { @Test void controlHeaderTest() { - CSVReportingDataParser csvReportingDataParser = new CSVReportingDataParser(); + CSVReportingDataParser csvReportingDataParser = new CSVReportingDataParser(fileUtilsInterface); String[] validHeaderToControl = new String [] {"statut", "dateInfo", "idUe", "idContact", "nom", "prenom", "adresse", "numeroDeLot"}; String[] invalidHeaderWrongValues = new String [] {"statut", "dateInfo", "idUe2", "idContact", "nom", "prenom", "adresse2", "numeroDeLot2"}; String[] headerToControlWrongSize = new String [] {"statut", "dateInfo", "idUe", "idContact", "nom", "prenom", "adresse", "numeroDeLot", "ninth"}; @@ -60,7 +64,7 @@ void controlHeaderTest() { @Test void convertDateTest() { - CSVReportingDataParser csvReportingDataParser = new CSVReportingDataParser(); + CSVReportingDataParser csvReportingDataParser = new CSVReportingDataParser(fileUtilsInterface); assertEquals(1645007098, csvReportingDataParser.convertToTimestamp("16/02/2022 11:24:58")); assertEquals(1566544132, csvReportingDataParser.convertToTimestamp("23/08/2019 09:08:52")); assertEquals(1111111111, csvReportingDataParser.convertToTimestamp("18/03/2005 02:58:31")); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/ParaDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/ParaDataParserTest.java index a78413a1..efbc56d1 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/ParaDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/ParaDataParserTest.java @@ -8,6 +8,7 @@ import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -21,7 +22,7 @@ class ParaDataParserTest { - ParadataParser paradataParser = new ParadataParser(); + ParadataParser paradataParser = new ParadataParser(new FileSystemImpl()); @Test void catchExceptionAndErrors() { diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/XMLReportingDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/XMLReportingDataParserTest.java index 31211074..209419d4 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/XMLReportingDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/XMLReportingDataParserTest.java @@ -9,6 +9,8 @@ import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.junit.jupiter.api.Test; import java.nio.file.Paths; @@ -18,9 +20,11 @@ class XMLReportingDataParserTest { + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + @Test void parseReportingDataTest() { - XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(); + XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(fileUtilsInterface); SurveyRawData data = SurveyRawDataTest.createFakePapiSurveyRawData(); ReportingData reportingData = new ReportingData( @@ -62,7 +66,7 @@ void parseReportingDataTest() { @Test void parseMoogReportingDataTest(){ - XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(); + XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(fileUtilsInterface); SurveyRawData data = SurveyRawDataTest.createFakePapiSurveyRawData(); ReportingData reportingData = new ReportingData( @@ -154,7 +158,7 @@ void parseMoogReportingDataTest(){ @Test void maxTest() { - XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(); + XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(fileUtilsInterface); ReportingData reportingData = new ReportingData(); reportingData.putReportingDataUE(ReportingDataUETest.createFakeReportingDataUEs()); assertEquals(5, xMLReportingDataParser.countMaxStates(reportingData)); //remove double @@ -162,7 +166,7 @@ void maxTest() { @Test void maxTest_noData() { - XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(); + XMLReportingDataParser xMLReportingDataParser = new XMLReportingDataParser(fileUtilsInterface); ReportingData reportingData = new ReportingData(); assertEquals(0, xMLReportingDataParser.countMaxStates(reportingData)); assertEquals(0, xMLReportingDataParser.countMaxAttempts(reportingData)); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/inputs/UserInputsFileTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/inputs/UserInputsFileTest.java index 5eb1dc44..1d4e4811 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/inputs/UserInputsFileTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/inputs/UserInputsFileTest.java @@ -10,6 +10,8 @@ import java.nio.file.Path; import java.util.Set; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.junit.jupiter.api.Test; import fr.insee.kraftwerk.core.TestConstants; @@ -21,12 +23,14 @@ class UserInputsFileTest { private static final Path inputSamplesDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "user_inputs"); + private static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + @Test void testReadValidUserInput_singleMode() throws KraftwerkException { UserInputsFile userInputsFile = new UserInputsFile( inputSamplesDirectory.resolve("inputs_valid.json"), - inputSamplesDirectory); + inputSamplesDirectory, fileUtilsInterface); // ModeInputs modeInputs = userInputsFile.getModeInputs("CAPI"); assertNotNull(modeInputs.getDataFile()); @@ -46,14 +50,14 @@ void testReadValidUserInput_singleMode() throws KraftwerkException { void testReadValidUserInput_missingOptionalFields() { assertDoesNotThrow(() -> new UserInputsFile( inputSamplesDirectory.resolve("inputs_valid_missing_fields.json"), - inputSamplesDirectory)); + inputSamplesDirectory, fileUtilsInterface)); } @Test void testReadValidUserInput_severalModes() throws KraftwerkException { UserInputsFile userInputsFile = new UserInputsFile( inputSamplesDirectory.resolve("inputs_valid_several_modes.json"), - inputSamplesDirectory); + inputSamplesDirectory, fileUtilsInterface); // assertTrue(userInputsFile.getModes().containsAll(Set.of("CAPI", "CAWI", "PAPI"))); // @@ -75,7 +79,7 @@ void testReadValidUserInput_severalModes() throws KraftwerkException { void testReadInvalidUserInput_wrongDataFormat() { Path path = inputSamplesDirectory.resolve("inputs_invalid_data_format.json"); assertThrows(UnknownDataFormatException.class, () -> { - new UserInputsFile(path,inputSamplesDirectory); + new UserInputsFile(path,inputSamplesDirectory, fileUtilsInterface); }); } @@ -83,7 +87,7 @@ void testReadInvalidUserInput_wrongDataFormat() { void testReadInvalidUserInput_wrongFieldNames() { Path path = inputSamplesDirectory.resolve("inputs_invalid_field_names.json"); assertThrows(MissingMandatoryFieldException.class, () -> { - new UserInputsFile( path,inputSamplesDirectory); + new UserInputsFile( path,inputSamplesDirectory, fileUtilsInterface); }); } @@ -91,7 +95,7 @@ void testReadInvalidUserInput_wrongFieldNames() { void testReadMalformedInput() throws KraftwerkException { Path path = inputSamplesDirectory.resolve("inputs_invalid_malformed.json"); assertThrows(UnknownDataFormatException.class, () -> { - new UserInputsFile(path,inputSamplesDirectory); + new UserInputsFile(path,inputSamplesDirectory, fileUtilsInterface); }); } @@ -99,7 +103,7 @@ void testReadMalformedInput() throws KraftwerkException { void testReadInputMissingFile() throws KraftwerkException { Path path = inputSamplesDirectory.resolve("inputs_invalid_several_modes_fileNotExist.json"); assertThrows(KraftwerkException.class, () -> { - new UserInputsFile(path,inputSamplesDirectory); + new UserInputsFile(path,inputSamplesDirectory, fileUtilsInterface); }); } @@ -107,7 +111,7 @@ void testReadInputMissingFile() throws KraftwerkException { void testReadInputCompletePath() throws KraftwerkException { Path path = inputSamplesDirectory.resolve("inputs_valid_several_modes_completePath.json"); assertDoesNotThrow(() -> { - new UserInputsFile(path,inputSamplesDirectory); + new UserInputsFile(path,inputSamplesDirectory, fileUtilsInterface); }); } diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/inputs/UserInputsGenesisTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/inputs/UserInputsGenesisTest.java index 1b77e367..85b575d6 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/inputs/UserInputsGenesisTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/inputs/UserInputsGenesisTest.java @@ -4,6 +4,7 @@ import fr.insee.kraftwerk.core.data.model.Mode; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.parsers.DataFormat; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -22,7 +23,7 @@ void testReadValidUserInput_singleMode() throws KraftwerkException, IOException List modes = new ArrayList<>(); modes.add(Mode.F2F); - UserInputsGenesis userInputsGenesis = new UserInputsGenesis(false, inputSamplesDirectory.resolve("Valid"), modes); + UserInputsGenesis userInputsGenesis = new UserInputsGenesis(false, inputSamplesDirectory.resolve("Valid"), modes, new FileSystemImpl()); // ModeInputs modeInputs = userInputsGenesis.getModeInputs("F2F"); assertNotNull(modeInputs.getDdiUrl()); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/DDIReaderTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/DDIReaderTest.java index dcb80462..fbc72ae3 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/DDIReaderTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/DDIReaderTest.java @@ -3,6 +3,8 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.TestConstants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.junit.jupiter.api.Test; import java.net.MalformedURLException; @@ -27,6 +29,8 @@ class DDIReaderTest { static final String DDI_LOG_X21 = "ddi-log-2021-x21-web.xml"; static final String DDI_LOG_X22 = "S2_WEB.xml"; + static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + @Test void readSimpsonsV1Variables() throws MalformedURLException, KraftwerkException, URISyntaxException { @@ -41,7 +45,7 @@ void readSimpsonsV1Variables() throws MalformedURLException, KraftwerkException, "SURVEY_COMMENT"); MetadataModel simpsonsMetadata = DDIReader - .getMetadataFromDDI(Constants.convertToUrl(DDI_FOLDER + "/" + DDI_SIMPSONS_V1)); + .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_SIMPSONS_V1, fileUtilsInterface); // assertNotNull(simpsonsMetadata); @@ -79,7 +83,7 @@ void readSimpsonsV2Variables() throws MalformedURLException, KraftwerkException, "FEELCHAREV4", "LEAVDURATION11", "LEAVDURATION52", "NB_CHAR", "SURVEY_COMMENT"); MetadataModel simpsonsMetadata = DDIReader - .getMetadataFromDDI(Constants.convertToUrl(DDI_FOLDER + "/" + DDI_SIMPSONS_V2)); + .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_SIMPSONS_V2, fileUtilsInterface); // assertNotNull(simpsonsMetadata); @@ -107,7 +111,7 @@ void readVqsWebVariables() throws MalformedURLException, KraftwerkException, URI "RESIDM", "NHAB"); MetadataModel vqsMetadata = DDIReader - .getMetadataFromDDI(Constants.convertToUrl(DDI_FOLDER + "/" + DDI_VQS_WEB)); + .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_VQS_WEB.toString(), fileUtilsInterface); // assertNotNull(vqsMetadata); @@ -147,7 +151,7 @@ void readVqsPapVariables() throws MalformedURLException, KraftwerkException, URI "AIDREG_B", "AIDREG_C", "AIDREG_D", "RESID", "RESIDANCIEN", "NBQUEST"); MetadataModel vqsMetadata = DDIReader - .getMetadataFromDDI(Constants.convertToUrl(DDI_FOLDER + "/" + DDI_VQS_PAP)); + .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_VQS_PAP, fileUtilsInterface); // assertNotNull(vqsMetadata); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/LunaticReaderTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/LunaticReaderTest.java index fce0af37..ae480ce3 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/LunaticReaderTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/LunaticReaderTest.java @@ -1,6 +1,8 @@ package fr.insee.kraftwerk.core.metadata; import fr.insee.kraftwerk.core.TestConstants; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.junit.jupiter.api.Test; import java.nio.file.Path; @@ -11,12 +13,13 @@ class LunaticReaderTest { static final Path lunaticSamplesPath = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "lunatic"); + static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); @Test void readLogX21TelLunaticFile() { // CalculatedVariables calculatedVariables = LunaticReader.getCalculatedFromLunatic( - lunaticSamplesPath.resolve("log2021x21_tel.json")); + lunaticSamplesPath.resolve("log2021x21_tel.json"), fileUtilsInterface); // assertNotNull(calculatedVariables); @@ -41,7 +44,7 @@ void readLogX21TelLunaticFile() { void readLogX22WebLunaticFile() { // CalculatedVariables calculatedVariables = LunaticReader.getCalculatedFromLunatic( - lunaticSamplesPath.resolve("log2021x22_web.json")); + lunaticSamplesPath.resolve("log2021x22_web.json"), fileUtilsInterface); // assertNotNull(calculatedVariables); @@ -55,7 +58,7 @@ void readLogX22WebLunaticFile() { void readVariablesFromLogX21WebLunaticFile() { // MetadataModel variables = LunaticReader.getMetadataFromLunatic( - lunaticSamplesPath.resolve("log2021x21_web.json")); + lunaticSamplesPath.resolve("log2021x21_web.json"), fileUtilsInterface); // assertNotNull(variables); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/VariableTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/VariableTest.java index 37b9db3e..1e2941a1 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/VariableTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/metadata/VariableTest.java @@ -3,6 +3,8 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.TestConstants; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.junit.jupiter.api.Test; import java.net.MalformedURLException; @@ -19,6 +21,7 @@ class VariableTest { static final String DDI_SIMPSONS_V2 = "ddi-simpsons-v2.xml"; static final String DDI_VQS_WEB = "vqs-2021-x00-xforms-ddi.xml"; static final String DDI_VQS_PAP = "vqs-2021-x00-fo-ddi.xml"; + static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); @Test void readSimpsonsV1Variables() throws MalformedURLException, KraftwerkException, URISyntaxException { @@ -34,7 +37,7 @@ void readSimpsonsV1Variables() throws MalformedURLException, KraftwerkException, "SURVEY_COMMENT"); MetadataModel simpsonsMetadata = DDIReader - .getMetadataFromDDI(Constants.convertToUrl(DDI_FOLDER + "/" + DDI_SIMPSONS_V1)); + .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_SIMPSONS_V1, fileUtilsInterface); // assertNotNull(simpsonsMetadata); @@ -72,7 +75,7 @@ void readSimpsonsV2Variables() throws MalformedURLException, KraftwerkException, "FEELCHAREV4", "LEAVDURATION11", "LEAVDURATION52", "NB_CHAR", "SURVEY_COMMENT"); MetadataModel simpsonsMetadata = DDIReader - .getMetadataFromDDI(Constants.convertToUrl(DDI_FOLDER + "/" + DDI_SIMPSONS_V2)); + .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_SIMPSONS_V2.toString(), fileUtilsInterface); // assertNotNull(simpsonsMetadata); @@ -100,7 +103,7 @@ void readVqsWebVariables() throws MalformedURLException, KraftwerkException, URI "RESIDM", "NHAB"); MetadataModel vqsMetadata = DDIReader - .getMetadataFromDDI(Constants.convertToUrl(DDI_FOLDER + "/" + DDI_VQS_WEB)); + .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_VQS_WEB, fileUtilsInterface); // assertNotNull(vqsMetadata); @@ -140,7 +143,7 @@ void readVqsPapVariables() throws MalformedURLException, KraftwerkException, URI "AIDREG_B", "AIDREG_C", "AIDREG_D", "RESID", "RESIDANCIEN", "NBQUEST"); MetadataModel vqsMetadata = DDIReader - .getMetadataFromDDI(Constants.convertToUrl(DDI_FOLDER + "/" + DDI_VQS_PAP)); + .getMetadataFromDDI(DDI_FOLDER + "/" + DDI_VQS_PAP, fileUtilsInterface); // assertNotNull(vqsMetadata); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/CsvOutputFilesTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/CsvOutputFilesTest.java index ac7dc8fe..6b5b4b93 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/CsvOutputFilesTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/CsvOutputFilesTest.java @@ -9,8 +9,9 @@ import fr.insee.kraftwerk.core.metadata.Variable; import fr.insee.kraftwerk.core.metadata.VariableType; import fr.insee.kraftwerk.core.outputs.csv.CsvOutputFiles; -import fr.insee.kraftwerk.core.utils.FileUtils; import fr.insee.kraftwerk.core.utils.SqlUtils; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; @@ -23,6 +24,8 @@ import org.junit.jupiter.api.TestMethodOrder; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; @@ -36,6 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; + @TestMethodOrder(OrderAnnotation.class) class CsvOutputFilesTest { @@ -43,6 +47,8 @@ class CsvOutputFilesTest { private static OutputFiles outputFiles; private static Connection database; + private static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + Dataset fooDataset = new InMemoryDataset(List.of(), List.of(new Structured.Component("FOO", String.class, Dataset.Role.IDENTIFIER))); @@ -53,7 +59,7 @@ void createInstance() { // testUserInputsFile = new UserInputsFile( Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "user_inputs/inputs_valid_several_modes.json"), - Path.of(TestConstants.UNIT_TESTS_DIRECTORY,"user_inputs")); + Path.of(TestConstants.UNIT_TESTS_DIRECTORY,"user_inputs"), fileUtilsInterface); // VtlBindings vtlBindings = new VtlBindings(); for (String mode : testUserInputsFile.getModes()) { @@ -66,7 +72,7 @@ void createInstance() { // database = SqlUtils.openConnection(); SqlUtils.convertVtlBindingsIntoSqlDatabase(vtlBindings, database.createStatement()); - outputFiles = new CsvOutputFiles(Paths.get(TestConstants.UNIT_TESTS_DUMP), vtlBindings, testUserInputsFile.getModes(), database.createStatement()); + outputFiles = new CsvOutputFiles(Paths.get(TestConstants.UNIT_TESTS_DUMP), vtlBindings, testUserInputsFile.getModes(), database.createStatement(), fileUtilsInterface); }); } @@ -86,8 +92,8 @@ void testGetDatasetOutputNames() { @Test @Order(3) - void testWriteCsv() throws KraftwerkException, SQLException { - FileUtils.createDirectoryIfNotExist(outputFiles.getOutputFolder()); + void testWriteCsv() throws KraftwerkException, IOException { + Files.createDirectories(outputFiles.getOutputFolder()); Map metaModels = new HashMap<>(); MetadataModel metMod = new MetadataModel(); @@ -103,7 +109,7 @@ void testWriteCsv() throws KraftwerkException, SQLException { Path racinePath = Path.of(outputFiles.getOutputFolder().toString(), outputFiles.outputFileName("RACINE")); racinePath = racinePath.resolveSibling(racinePath.getFileName()); File f = racinePath.toFile(); - Assertions.assertTrue(f.exists()); + assertTrue(f.exists()); Assertions.assertNotEquals(0, f.length()); } diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/ParquetOutputFilesTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/ParquetOutputFilesTest.java index 598d5896..652dae42 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/ParquetOutputFilesTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/ParquetOutputFilesTest.java @@ -9,7 +9,8 @@ import fr.insee.kraftwerk.core.metadata.Variable; import fr.insee.kraftwerk.core.metadata.VariableType; import fr.insee.kraftwerk.core.outputs.parquet.ParquetOutputFiles; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; @@ -38,6 +39,7 @@ class ParquetOutputFilesTest { private static UserInputsFile testUserInputs; private static ParquetOutputFiles outputFiles; + private static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); private static Statement testDatabase; @@ -62,7 +64,7 @@ static void createInstance() throws SQLException { // testUserInputs = new UserInputsFile( Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "user_inputs/inputs_valid_several_modes.json"), - Path.of(TestConstants.UNIT_TESTS_DIRECTORY,"user_inputs")); + Path.of(TestConstants.UNIT_TESTS_DIRECTORY,"user_inputs"), fileUtilsInterface); // VtlBindings vtlBindings = new VtlBindings(); for (String mode : testUserInputs.getModes()) { @@ -74,7 +76,7 @@ static void createInstance() throws SQLException { vtlBindings.put("FROM_USER", testDataset); // SqlUtils.convertVtlBindingsIntoSqlDatabase(vtlBindings, testDatabase); - outputFiles = new ParquetOutputFiles(Paths.get(TestConstants.UNIT_TESTS_DUMP), vtlBindings, testUserInputs.getModes(), testDatabase); + outputFiles = new ParquetOutputFiles(Paths.get(TestConstants.UNIT_TESTS_DUMP), vtlBindings, testUserInputs.getModes(), testDatabase, fileUtilsInterface); }); } @@ -97,7 +99,7 @@ void writeParquetFromDatasetTest() throws KraftwerkException { // Clean the existing file // Files.deleteIfExists(outputFiles.getOutputFolder()); - FileUtils.createDirectoryIfNotExist(outputFiles.getOutputFolder()); + fileUtilsInterface.createDirectoryIfNotExist(outputFiles.getOutputFolder()); Map metaModels = new HashMap<>(); MetadataModel metMod = new MetadataModel(); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java index 5d9c2513..b3e48d64 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/outputs/scripts/ImportScriptTest.java @@ -10,6 +10,8 @@ import fr.insee.kraftwerk.core.outputs.TableScriptInfo; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import fr.insee.vtl.model.Dataset; @@ -36,8 +38,9 @@ class ImportScriptTest { TableScriptInfo tableScriptInfo; Map metadata; + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); - VtlExecute vtlExecute = new VtlExecute(); + VtlExecute vtlExecute = new VtlExecute(fileUtilsInterface); @BeforeEach public void initMetadata() { @@ -57,9 +60,9 @@ private void instantiateMap() { // add group prefixes List errors = new ArrayList<>(); - GroupProcessing groupProcessing = new GroupProcessing(vtlBindings, srdWeb.getMetadataModel()); + GroupProcessing groupProcessing = new GroupProcessing(vtlBindings, srdWeb.getMetadataModel(), fileUtilsInterface); groupProcessing.applyVtlTransformations("CAWI", null, errors); - GroupProcessing groupProcessing2 = new GroupProcessing(vtlBindings, srdPaper.getMetadataModel()); + GroupProcessing groupProcessing2 = new GroupProcessing(vtlBindings, srdPaper.getMetadataModel(), fileUtilsInterface); groupProcessing2.applyVtlTransformations("PAPI", null, errors); dataStructure = vtlBindings.getDataset("CAWI").getDataStructure(); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParserTest.java index 9909320d..ae00e02b 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/LunaticXmlDataParserTest.java @@ -9,6 +9,8 @@ import fr.insee.kraftwerk.core.rawdata.GroupData; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.junit.jupiter.api.Test; import java.nio.file.Path; @@ -23,11 +25,12 @@ class LunaticXmlDataParserTest { private final String dataSamplesFolder = TestConstants.UNIT_TESTS_DIRECTORY + "/data"; + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); @Test void readXmlFile_null() { SurveyRawData data = new SurveyRawData("TEST"); - LunaticXmlDataParser parser = new LunaticXmlDataParser(data); + LunaticXmlDataParser parser = new LunaticXmlDataParser(data, fileUtilsInterface); assertThrows(NullPointerException.class,() ->parser.parseDataFile(null)); parser.parseDataFile(Path.of("notfound.xml")); @@ -43,7 +46,7 @@ void parseLunaticDataFolder() throws NullException { metadataModel.getVariables().putVariable(new Variable("FOO", metadataModel.getRootGroup(), VariableType.STRING)); data.setMetadataModel(metadataModel); Path dataPath = Paths.get(dataSamplesFolder + "/lunatic_xml/fake-multiple-files"); - LunaticXmlDataParser parser = new LunaticXmlDataParser(data); + LunaticXmlDataParser parser = new LunaticXmlDataParser(data, fileUtilsInterface); parser.parseSurveyData(dataPath,null); // @@ -63,7 +66,7 @@ void parseLunaticXml_rootOnly() throws NullException { SurveyRawData data = new SurveyRawData("TEST"); data.setMetadataModel(MetadataModelTest.createVariablesMap_rootOnly()); Path dataPath = Paths.get(dataSamplesFolder + "/lunatic_xml/fake-lunatic-data-root-only.xml"); - LunaticXmlDataParser parser = new LunaticXmlDataParser(data); + LunaticXmlDataParser parser = new LunaticXmlDataParser(data, fileUtilsInterface); parser.parseSurveyData(dataPath,null); // @@ -96,7 +99,7 @@ void parseLunaticXml_oneLevel() throws NullException { SurveyRawData data = new SurveyRawData("TEST"); data.setMetadataModel(MetadataModelTest.createVariablesMap_oneLevel()); Path dataPath = Paths.get(dataSamplesFolder + "/lunatic_xml/fake-lunatic-data-1.xml"); - LunaticXmlDataParser parser = new LunaticXmlDataParser(data); + LunaticXmlDataParser parser = new LunaticXmlDataParser(data, fileUtilsInterface); parser.parseSurveyData(dataPath,null); // @@ -130,7 +133,7 @@ void parseLunaticXml_noCollected() throws NullException { SurveyRawData data = new SurveyRawData("TEST"); data.setMetadataModel(MetadataModelTest.createVariablesMap_oneLevel()); Path dataPath = Paths.get(dataSamplesFolder + "/lunatic_xml/fake-lunatic-data-3.xml"); - LunaticXmlDataParser parser = new LunaticXmlDataParser(data); + LunaticXmlDataParser parser = new LunaticXmlDataParser(data, fileUtilsInterface); //When parser.parseSurveyData(dataPath,null); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/PaperDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/PaperDataParserTest.java index 798f6dfe..5cfd4adc 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/PaperDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/parsers/PaperDataParserTest.java @@ -7,6 +7,8 @@ import fr.insee.kraftwerk.core.metadata.VariableType; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -18,12 +20,13 @@ class PaperDataParserTest { private final String dataSamplesFolder = TestConstants.UNIT_TESTS_DIRECTORY + "/data"; + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); @Test void readPaperFile_null() { //Given SurveyRawData data = new SurveyRawData("TEST"); - PaperDataParser parser = new PaperDataParser(data); + PaperDataParser parser = new PaperDataParser(data, fileUtilsInterface); //When + Then assertThrows(NullPointerException.class,() ->parser.parseDataFile(null)); @@ -39,7 +42,7 @@ void parsePaperDataFile() throws NullException, NumberFormatException { metadataModel.getVariables().putVariable(new Variable("TESTINT1", metadataModel.getRootGroup(), VariableType.INTEGER)); data.setMetadataModel(metadataModel); Path dataPath = Paths.get(dataSamplesFolder + "/paper_csv/fake-paper-data.csv"); - PaperDataParser parser = new PaperDataParser(data); + PaperDataParser parser = new PaperDataParser(data, fileUtilsInterface); //When parser.parseSurveyData(dataPath,null); @@ -72,7 +75,7 @@ void parsePaperDataFolder() throws NullException, NumberFormatException { metadataModel.getVariables().putVariable(new Variable("TESTINT1", metadataModel.getRootGroup(), VariableType.INTEGER)); data.setMetadataModel(metadataModel); Path dataPath = Paths.get(dataSamplesFolder + "/paper_csv/fake-multiple-files"); - PaperDataParser parser = new PaperDataParser(data); + PaperDataParser parser = new PaperDataParser(data, fileUtilsInterface); //When parser.parseSurveyData(dataPath,null); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBIndingsSequenceGenesisTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBIndingsSequenceGenesisTest.java new file mode 100644 index 00000000..5472f7c0 --- /dev/null +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBIndingsSequenceGenesisTest.java @@ -0,0 +1,98 @@ +package fr.insee.kraftwerk.core.sequence; + +import fr.insee.kraftwerk.core.Constants; +import fr.insee.kraftwerk.core.TestConstants; +import fr.insee.kraftwerk.core.data.model.ExternalVariable; +import fr.insee.kraftwerk.core.data.model.Mode; +import fr.insee.kraftwerk.core.data.model.SurveyUnitUpdateLatest; +import fr.insee.kraftwerk.core.data.model.VariableState; +import fr.insee.kraftwerk.core.metadata.Group; +import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.kraftwerk.core.metadata.Variable; +import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.vtl.VtlBindings; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +class BuildBIndingsSequenceGenesisTest { + private static final List surveyUnits = new ArrayList<>(); + + @BeforeAll + static void init(){ + String dataMode = "WEB"; + //Create one document + surveyUnits.clear(); + SurveyUnitUpdateLatest surveyUnitUpdateLatest = new SurveyUnitUpdateLatest(); + surveyUnitUpdateLatest.setIdUE("0000001"); + surveyUnitUpdateLatest.setIdCampaign("TESTCAMPAIGN"); + surveyUnitUpdateLatest.setIdQuest("TESTQUEST"); + surveyUnitUpdateLatest.setMode(Mode.valueOf(dataMode)); + + surveyUnitUpdateLatest.setVariablesUpdate(new ArrayList<>()); + VariableState variableState = new VariableState(); + variableState.setIdVar("COLLVAR1"); + variableState.setIdLoop(Constants.ROOT_GROUP_NAME); + variableState.setValues(new ArrayList<>()); + variableState.getValues().add("TEST1"); + surveyUnitUpdateLatest.getVariablesUpdate().add(variableState); + variableState = new VariableState(); + variableState.setIdVar("COLLVAR2"); + variableState.setIdLoop("LOOP1"); + variableState.setIdParent(Constants.ROOT_GROUP_NAME); + variableState.setValues(new ArrayList<>()); + variableState.getValues().add("TEST3"); + surveyUnitUpdateLatest.getVariablesUpdate().add(variableState); + + surveyUnitUpdateLatest.setExternalVariables(new ArrayList<>()); + ExternalVariable externalVariable = new ExternalVariable(); + externalVariable.setIdVar("EXTVAR1"); + externalVariable.setValues(new ArrayList<>()); + externalVariable.getValues().add("TEST2"); + surveyUnitUpdateLatest.getExternalVariables().add(externalVariable); + + surveyUnits.add(surveyUnitUpdateLatest); + } + + @Test + void buildVtlBindings_errorWithoutMetadata(){ + //GIVEN + String dataMode = "WEB"; + + VtlBindings vtlBindings = new VtlBindings(); + BuildBindingsSequenceGenesis bbsg = new BuildBindingsSequenceGenesis(new FileSystemImpl()); + //WHEN + THEN + assertThrows(NullPointerException.class, () -> bbsg.buildVtlBindings(dataMode, vtlBindings, null, surveyUnits, null)); + + } + + @Test + void buildVtlBindings_success(){ + //GIVEN + String dataMode = "WEB"; + + VtlBindings vtlBindings = new VtlBindings(); + BuildBindingsSequenceGenesis bbsg = new BuildBindingsSequenceGenesis(new FileSystemImpl()); + MetadataModel metadata = new MetadataModel(); + metadata.getVariables().putVariable(new Variable("COLLVAR1", metadata.getRootGroup(), VariableType.STRING)); + Group group = new Group("TESTLOOP",Constants.ROOT_GROUP_NAME); + metadata.getVariables().putVariable(new Variable("COLLVAR2", group, VariableType.STRING)); + metadata.getVariables().putVariable(new Variable("EXTVAR1", metadata.getRootGroup(), VariableType.STRING)); + metadata.getGroups().put("TESTLOOP", group); + Map modeMetadataMap = new HashMap<>(); + modeMetadataMap.put(dataMode,metadata); + + //WHEN + THEN + assertDoesNotThrow(() -> bbsg.buildVtlBindings(dataMode, vtlBindings, modeMetadataMap, surveyUnits, + Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "genesis"))); + } +} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceTest.java index 0ccc6381..d17b2a03 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceTest.java @@ -7,6 +7,8 @@ import fr.insee.kraftwerk.core.metadata.UcqVariable; import fr.insee.kraftwerk.core.metadata.Variable; import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.vtl.VtlBindings; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -20,6 +22,8 @@ class BuildBindingsSequenceTest { private static final Path inputSamplesDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "user_inputs"); + private static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + @Test @@ -27,12 +31,12 @@ void buildVtlBindings_errorWithoutMetadata() throws KraftwerkException { //GIVEN UserInputsFile userInputsFile = new UserInputsFile( inputSamplesDirectory.resolve("inputs_valid.json"), - inputSamplesDirectory); + inputSamplesDirectory, fileUtilsInterface); String dataMode = "CAPI"; VtlBindings vtlBindings = new VtlBindings(); boolean withAllReportingData = false; boolean withDdi = true; - BuildBindingsSequence bbs = new BuildBindingsSequence(withAllReportingData); + BuildBindingsSequence bbs = new BuildBindingsSequence(withAllReportingData, new FileSystemImpl()); //WHEN MetadataModel metadata = null; @@ -47,10 +51,10 @@ void buildVtlBindings_success_changingDdi_and_reportingData(boolean withDdi, boo //GIVEN UserInputsFile userInputsFile = new UserInputsFile( inputSamplesDirectory.resolve("inputs_valid.json"), - inputSamplesDirectory); + inputSamplesDirectory, fileUtilsInterface); String dataMode = "CAPI"; VtlBindings vtlBindings = new VtlBindings(); - BuildBindingsSequence bbs = new BuildBindingsSequence(withAllReportingData); + BuildBindingsSequence bbs = new BuildBindingsSequence(withAllReportingData, new FileSystemImpl()); MetadataModel capiMetadata = new MetadataModel(); capiMetadata.getVariables().putVariable(new Variable("VAR1", capiMetadata.getRootGroup(), VariableType.STRING)); capiMetadata.getVariables().putVariable(new UcqVariable("PAYSNAIS", capiMetadata.getRootGroup(), VariableType.STRING)); diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/FileUtilsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/FileUtilsTest.java deleted file mode 100644 index dcb6a065..00000000 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/FileUtilsTest.java +++ /dev/null @@ -1,98 +0,0 @@ -package fr.insee.kraftwerk.core.utils; - -import fr.insee.kraftwerk.core.Constants; -import fr.insee.kraftwerk.core.TestConstants; -import fr.insee.kraftwerk.core.exceptions.KraftwerkException; -import fr.insee.kraftwerk.core.inputs.UserInputsFile; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.util.FileSystemUtils; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.util.Objects; - -import static org.junit.Assert.assertThrows; -import static org.junit.jupiter.api.Assertions.assertEquals; - -class FileUtilsTest { - - - @Test - void testTransformToOut() { - assertEquals(Paths.get("C://Users/in/Kraftwerk/src/test/resources/functional_tests/out/VQS"), - FileUtils.transformToOut(Paths.get("C://Users/in/Kraftwerk/src/test/resources/functional_tests/in/VQS"))); - } - - @Test - void testTransformToOut2() { - Path path = Paths.get("C://Users/in/Kraftwerk/src/test/resources/functional_tests/out/VQS"); - assertEquals(path,FileUtils.transformToOut(Paths.get("C://Users/in/Kraftwerk/src/test/resources/functional_tests/in/VQS"), LocalDateTime.now()).getParent()); - } - - @Test - void archiveInputFiles_failWhenNull() { - assertThrows(NullPointerException.class, () -> FileUtils.archiveInputFiles(null)); - } - - - @Test - void archiveInputFiles_ok() throws IOException, KraftwerkException{ - - //GIVEN - String campaignName = "move_files"; - Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, campaignName, "execute"); - FileSystemUtils.deleteRecursively(inputDirectory); - Files.createDirectories(inputDirectory); - Files.copy(Path.of(TestConstants.UNIT_TESTS_DIRECTORY, campaignName,"move_files.json"), - Path.of(inputDirectory.toString(),"move_files.json")); - - //WEB - Path webDirectory = Paths.get(inputDirectory.toString(), "web"); - Files.createDirectories(webDirectory); - new File(webDirectory+"/web.xml").createNewFile(); - new File(webDirectory+"/vqs-2021-x00-xforms-ddi.xml").createNewFile(); - new File(webDirectory+"/WEB.vtl").createNewFile(); - - //PAPER - Path paperDirectory = Paths.get(inputDirectory.toString(), "papier"); - Files.createDirectories(paperDirectory); - new File(paperDirectory+"/paper.txt").createNewFile(); - new File(paperDirectory+"/vqs-2021-x00-fo-ddi.xml").createNewFile(); - - //Reporting - Path reportingDirectory = Paths.get(inputDirectory.toString(), "suivi"); - Files.createDirectories(reportingDirectory); - new File(reportingDirectory+"/reportingdata.xml").createNewFile(); - - //Paradata - Path paradataDirectory = Paths.get(inputDirectory.toString(), "paradata"); - Files.createDirectories(paradataDirectory); - new File(Constants.getResourceAbsolutePath(paradataDirectory +"/L0000003.json")).createNewFile(); - new File(Constants.getResourceAbsolutePath(paradataDirectory +"/L0000004.json")).createNewFile(); - new File(Constants.getResourceAbsolutePath(paradataDirectory +"/L0000009.json")).createNewFile(); - new File(Constants.getResourceAbsolutePath(paradataDirectory +"/L0000010.json")).createNewFile(); - - UserInputsFile testUserInputsFile = new UserInputsFile(Path.of(inputDirectory.toString(), "move_files.json"),inputDirectory); - - - //WHEN - FileUtils.archiveInputFiles(testUserInputsFile); - - //THEN - Assertions.assertTrue(new File(inputDirectory.toString() + "/Archive/papier").exists()); - Assertions.assertTrue(new File(inputDirectory.toString() + "/Archive/web").exists()); - Assertions.assertTrue(new File(inputDirectory.toString() + "/Archive/paradata/L0000010.json").exists()); - Assertions.assertTrue(new File(inputDirectory.toString() + "/Archive/suivi/reportingdata.xml").exists()); - - //CLEAN - FileSystemUtils.deleteRecursively(inputDirectory); - } - - -} - diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/SqlUtilsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/SqlUtilsTest.java index 268dca57..b5fdeec4 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/SqlUtilsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/SqlUtilsTest.java @@ -5,6 +5,7 @@ import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; import fr.insee.kraftwerk.core.metadata.VariableType; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; @@ -14,6 +15,9 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.nio.file.FileSystemException; +import java.nio.file.Files; import java.nio.file.Path; import java.sql.Connection; import java.sql.ResultSet; @@ -38,7 +42,8 @@ void convertVTLBindingTest() throws SQLException, KraftwerkException { //Given UserInputsFile testUserInputsFile = new UserInputsFile( Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "user_inputs/inputs_valid_several_modes.json"), - Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "user_inputs")); + Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "user_inputs"), + new FileSystemImpl()); VtlBindings vtlBindings = new VtlBindings(); Dataset testDataset = new InMemoryDataset(List.of(), List.of(new Structured.Component("TestString", String.class, Dataset.Role.IDENTIFIER))); @@ -184,6 +189,28 @@ void getTableNamesTest() throws SQLException { } } + @Test + void openConnection_file_test() throws SQLException, IOException { + Path filePath = Path.of(TestConstants.UNIT_TESTS_DIRECTORY,"sql","testdb.duckdb"); + try(Connection testDatabaseFile = SqlUtils.openConnection(filePath)) { + Assertions.assertThat(testDatabaseFile).isNotNull(); + + Statement testDatabaseStatement = testDatabaseFile.createStatement(); + //Given + testDatabaseStatement.execute("CREATE TABLE testtable1(testint1 INT, teststring1 NVARCHAR)"); + testDatabaseStatement.execute("CREATE TABLE testtable2(testint2 INT, teststring2 NVARCHAR)"); + + //When + List tableNames = SqlUtils.getTableNames(testDatabaseStatement); + + //Then + Assertions.assertThat(tableNames).contains("testtable1","testtable2"); + testDatabaseStatement.close(); + } + Files.deleteIfExists(filePath); + Files.deleteIfExists(filePath.getParent().resolve(filePath.getFileName()+".wal")); + } + @AfterAll diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/XMLSplitterTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/XmlSplitterTest.java similarity index 76% rename from kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/XMLSplitterTest.java rename to kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/XmlSplitterTest.java index 189a918c..484b5e96 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/XMLSplitterTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/XmlSplitterTest.java @@ -1,6 +1,9 @@ package fr.insee.kraftwerk.core.utils; import fr.insee.kraftwerk.core.TestConstants; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; +import fr.insee.kraftwerk.core.utils.xml.XmlSplitter; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -20,15 +23,17 @@ import static org.junit.Assert.assertEquals; -class XMLSplitterTest { +class XmlSplitterTest { static final String outDirectory = TestConstants.UNIT_TESTS_DUMP+"/split/"; + static final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + @BeforeAll static void cleanUpBeforeTests() throws Exception { File file = new File(outDirectory); if (file.exists()) { - List splitFiles = FileUtils.listFiles(outDirectory); + List splitFiles = fileUtilsInterface.listFileNames(outDirectory); if (splitFiles != null){ for (String splitFile : splitFiles) { if(!Files.deleteIfExists(Paths.get(outDirectory+splitFile))){ @@ -37,21 +42,21 @@ static void cleanUpBeforeTests() throws Exception { } } } - XMLSplitter.split(TestConstants.UNIT_TESTS_DIRECTORY+"/data/lunatic_xml/", "fake-lunatic-data-1.xml", - outDirectory,"SurveyUnit",2); + XmlSplitter.split(TestConstants.UNIT_TESTS_DIRECTORY+"/data/lunatic_xml/", "fake-lunatic-data-1.xml", + outDirectory,"SurveyUnit",2, new FileSystemImpl()); } @Test @DisplayName("OutDirectory should contain 3 files") void splitInThreeTest() throws XMLStreamException, IOException { - List splitFiles = FileUtils.listFiles(outDirectory); + List splitFiles = fileUtilsInterface.listFileNames(outDirectory); assertEquals(3,splitFiles.size()); } @Test @DisplayName("File split1.xml file should contain 2 survey units") void contains2SuTest() throws XMLStreamException, IOException { - List splitFiles = FileUtils.listFiles(outDirectory); + List splitFiles = fileUtilsInterface.listFileNames(outDirectory); int count = 0; XMLInputFactory factory = XMLInputFactory.newInstance(); try { diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/files/FileSystemImplTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/files/FileSystemImplTest.java new file mode 100644 index 00000000..963d5b16 --- /dev/null +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/files/FileSystemImplTest.java @@ -0,0 +1,347 @@ +package fr.insee.kraftwerk.core.utils.files; + +import fr.insee.kraftwerk.core.Constants; +import fr.insee.kraftwerk.core.TestConstants; +import fr.insee.kraftwerk.core.exceptions.KraftwerkException; +import fr.insee.kraftwerk.core.inputs.UserInputsFile; +import org.assertj.core.api.Assertions; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.Assert.assertThrows; + +class FileSystemImplTest { + private static final FileSystemImpl fileSystemImpl = new FileSystemImpl(); + + @Test + void renameInputFileTest() throws IOException { + + //GIVEN + String campaignName = "rename_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY,"files", campaignName); + //backup + Files.copy(inputDirectory.resolve("kraftwerk.json"), inputDirectory.getParent().resolve("kraftwerk.json")); + + //WHEN + fileSystemImpl.renameInputFile(inputDirectory); + + //THEN + try(Stream stream = Files.list(inputDirectory).filter(path -> path.getFileName().toString().startsWith("kraftwerk-"))){ + Assertions.assertThat(stream).isNotEmpty(); + } + + //CLEAN + org.springframework.util.FileSystemUtils.deleteRecursively(inputDirectory); + Files.createDirectories(inputDirectory); + Files.move(Path.of(inputDirectory.getParent().toString(),"kraftwerk.json"),Path.of(inputDirectory.toString(),"kraftwerk.json")); + } + + @Test + void archiveInputFiles_failWhenNull() { + assertThrows(NullPointerException.class, () -> fileSystemImpl.archiveInputFiles(null)); + } + + @Test + void archiveInputFiles_ok() throws IOException, KraftwerkException { + + //GIVEN + String campaignName = "move_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + //backup + Files.copy(Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName,"move_files.json"), + Path.of(inputDirectory.getParent().toString(),"move_files.json")); + + //WEB + Path webDirectory = Paths.get(inputDirectory.toString(), "web"); + Files.createDirectories(webDirectory); + new File(webDirectory+"/web.xml").createNewFile(); + new File(webDirectory+"/vqs-2021-x00-xforms-ddi.xml").createNewFile(); + new File(webDirectory+"/WEB.vtl").createNewFile(); + + //PAPER + Path paperDirectory = Paths.get(inputDirectory.toString(), "papier"); + Files.createDirectories(paperDirectory); + new File(paperDirectory+"/paper.txt").createNewFile(); + new File(paperDirectory+"/vqs-2021-x00-fo-ddi.xml").createNewFile(); + + //Reporting + Path reportingDirectory = Paths.get(inputDirectory.toString(), "suivi"); + Files.createDirectories(reportingDirectory); + new File(reportingDirectory+"/reportingdata.xml").createNewFile(); + + //Paradata + Path paradataDirectory = Paths.get(inputDirectory.toString(), "paradata"); + Files.createDirectories(paradataDirectory); + new File(Constants.getResourceAbsolutePath(paradataDirectory +"/L0000003.json")).createNewFile(); + new File(Constants.getResourceAbsolutePath(paradataDirectory +"/L0000004.json")).createNewFile(); + new File(Constants.getResourceAbsolutePath(paradataDirectory +"/L0000009.json")).createNewFile(); + new File(Constants.getResourceAbsolutePath(paradataDirectory +"/L0000010.json")).createNewFile(); + + UserInputsFile testUserInputsFile = new UserInputsFile(Path.of(inputDirectory.toString(), "move_files.json"),inputDirectory, fileSystemImpl); + + + //WHEN + fileSystemImpl.archiveInputFiles(testUserInputsFile); + + //THEN + Assertions.assertThat(new File(inputDirectory + "/Archive/papier")).exists(); + Assertions.assertThat(new File(inputDirectory + "/Archive/web")).exists(); + Assertions.assertThat(new File(inputDirectory + "/Archive/paradata/L0000010.json")).exists(); + Assertions.assertThat(new File(inputDirectory + "/Archive/suivi/reportingdata.xml")).exists(); + + //CLEAN + org.springframework.util.FileSystemUtils.deleteRecursively(inputDirectory); + Files.createDirectories(inputDirectory); + Files.move(Path.of(inputDirectory.getParent().toString(),"move_files.json"),Path.of(inputDirectory.toString(),"move_files.json")); + } + + @Test + void deleteDirectoryTest() throws KraftwerkException, IOException { + //GIVEN + String campaignName = "delete_directory"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + Files.createDirectories(inputDirectory); + Files.createFile(inputDirectory.resolve("test.txt")); + + //WHEN + fileSystemImpl.deleteDirectory(inputDirectory); + + //THEN + Assertions.assertThat(new File(inputDirectory.toString())).doesNotExist(); + } + + @Test + void listFileNamesTest(){ + //GIVEN + String campaignName = "list_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN + List fileNames = fileSystemImpl.listFileNames(inputDirectory.toString()); + + //THEN + Assertions.assertThat(fileNames).contains("file1.txt","file2.json","file3.xml"); + } + + @Test + void listFilePathsTest() { + //GIVEN + String campaignName = "list_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN + List filePaths = fileSystemImpl.listFilePaths(inputDirectory.toString()); + + //THEN + Assertions.assertThat(filePaths).hasSize(3); + for (String filePath : filePaths) { + Assertions.assertThat(Path.of(filePath).getFileName().toString()).containsAnyOf("file1.txt", "file2.json", "file3.xml"); + } + } + + @Test + void createDirectoryTest() throws IOException { + //GIVEN + String campaignName = "create_directory"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN + fileSystemImpl.createDirectoryIfNotExist(inputDirectory); + + //THEN + Assertions.assertThat(inputDirectory.toFile()).exists().isDirectory(); + + //CLEAN + Files.deleteIfExists(inputDirectory); + } + + @Test + void isDirectoryTest(){ + //GIVEN + String campaignName = "move_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN + THEN + Assertions.assertThat(fileSystemImpl.isDirectory(String.valueOf(inputDirectory))).isTrue(); + Assertions.assertThat(fileSystemImpl.isDirectory(String.valueOf(inputDirectory.resolve("move_files.json")))).isFalse(); + Assertions.assertThat(fileSystemImpl.isDirectory(String.valueOf(inputDirectory.resolve("NULL.json")))).isNull(); + } + + @Test + void getSizeOfTest() throws IOException { + //GIVEN + String campaignName = "convert_path"; + Path file = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName).resolve("test.txt"); + + //WHEN + THEN + Assertions.assertThat(fileSystemImpl.getSizeOf(file.toString())).isEqualTo(Files.size(file)); + } + + @Test + void writeFileTest() throws IOException { + //GIVEN + String campaignName = "write_file"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN + fileSystemImpl.writeFile(inputDirectory.resolve("test.txt").toString(),"test",true); + + //THEN + Assertions.assertThat(new File(inputDirectory.resolve("test.txt").toString())).exists(); + Assertions.assertThat(Files.readAllBytes(inputDirectory.resolve("test.txt"))).contains("test".getBytes()); + + //CLEAN + Files.deleteIfExists(inputDirectory.resolve("test.txt")); + Files.deleteIfExists(inputDirectory); + } + + @Test + void writeFileTest_append() throws IOException { + //GIVEN + String campaignName = "write_file"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN + fileSystemImpl.writeFile(inputDirectory.resolve("test.txt").toString(),"test1",true); + fileSystemImpl.writeFile(inputDirectory.resolve("test.txt").toString(),"\ntest2",false); + + //THEN + Assertions.assertThat(new File(inputDirectory.resolve("test.txt").toString())).exists(); + Assertions.assertThat(Files.readAllBytes(inputDirectory.resolve("test.txt"))).contains("test".getBytes()).contains("test2".getBytes()); + + //CLEAN + Files.deleteIfExists(inputDirectory.resolve("test.txt")); + Files.deleteIfExists(inputDirectory); + } + @Test + void findFileTest() throws KraftwerkException { + //GIVEN + String campaignName = "list_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN + THEN + Assertions.assertThat(fileSystemImpl.findFile(inputDirectory.toString(), "[\\w,\\s-]+\\.xml")).isNotEmpty(); + } + + @Test + void findFileTest_notfound() throws KraftwerkException { + //GIVEN + String campaignName = "list_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN+THEN + assertThrows(KraftwerkException.class, () -> fileSystemImpl.findFile(inputDirectory.toString(),Constants.DDI_FILE_REGEX)); + } + + @Test + void readFileTest() throws IOException { + //GIVEN + String campaignName = "read_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN + THEN + try(InputStream inputStream = fileSystemImpl.readFile(inputDirectory.resolve("test.txt").toString())){ + Assertions.assertThat(inputStream).hasContent("hello !"); + } + } + + @Test + void isFileExistsTest(){ + //GIVEN + String campaignName = "read_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN + THEN + Assertions.assertThat(fileSystemImpl.isFileExists(inputDirectory.resolve("test.txt").toString())).isTrue(); + Assertions.assertThat(fileSystemImpl.isFileExists(inputDirectory.resolve("null.txt").toString())).isFalse(); + } + + @Test + void moveFileTest() throws KraftwerkException, IOException { + //GIVEN + String campaignName = "move_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + Files.copy(inputDirectory.resolve("move_files.json"),inputDirectory.resolve("test1.txt")); + + String campaignName2 = "move_files2"; + Path inputDirectory2 = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName2); + + //WHEN + fileSystemImpl.moveFile(inputDirectory.resolve("test1.txt").toString(), inputDirectory2.resolve("test2.txt").toString()); + + //THEN + Assertions.assertThat(inputDirectory2.resolve("test2.txt")).exists(); + + //CLEAN + Files.deleteIfExists(inputDirectory2.resolve("test2.txt")); + Files.deleteIfExists(inputDirectory.resolve("test1.txt")); + } + + @Test + void moveFileTest_FromPath() throws KraftwerkException, IOException { + //GIVEN + String campaignName = "move_files"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + Files.copy(inputDirectory.resolve("move_files.json"),inputDirectory.resolve("test1.txt")); + + String campaignName2 = "move_files2"; + Path inputDirectory2 = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName2); + + //WHEN + fileSystemImpl.moveFile(inputDirectory.resolve("test1.txt"), inputDirectory2.resolve("test2.txt").toString()); + + //THEN + Assertions.assertThat(inputDirectory2.resolve("test2.txt")).exists(); + + //CLEAN + Files.deleteIfExists(inputDirectory2.resolve("test2.txt")); + Files.deleteIfExists(inputDirectory.resolve("test1.txt")); + Files.deleteIfExists(inputDirectory2); + } + + + + @Test + void convertToPathTest_nullUserField() throws KraftwerkException { + Assertions.assertThat(fileSystemImpl.convertToPath(null,null)).isNull(); + } + + @Test + void convertToPathTest_directoryNotExists(){ + Assert.assertThrows(KraftwerkException.class, () -> fileSystemImpl.convertToPath("test", Path.of("NOT SUPPOSED TO EXIST"))); + } + + @Test + void convertToPathTest() throws KraftwerkException { + //GIVEN + String campaignName = "convert_path"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN+THEN + Assertions.assertThat(fileSystemImpl.convertToPath("test.txt", inputDirectory)).exists(); + } + + @Test + void convertToURLTest_nullUserField(){ + Assertions.assertThat(fileSystemImpl.convertToUrl(null,null)).isNull(); + } + + @Test + void convertToURLTest(){ + //GIVEN + String campaignName = "convert_path"; + Path inputDirectory = Path.of(TestConstants.UNIT_TESTS_DIRECTORY, "files", campaignName); + + //WHEN+THEN + Assertions.assertThat(fileSystemImpl.convertToUrl("test.txt", inputDirectory)).endsWith("test.txt"); + } +} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/files/FileSystemInterfaceTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/files/FileSystemInterfaceTest.java new file mode 100644 index 00000000..4a799379 --- /dev/null +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/files/FileSystemInterfaceTest.java @@ -0,0 +1,29 @@ +package fr.insee.kraftwerk.core.utils.files; + +import fr.insee.kraftwerk.core.TestConstants; +import fr.insee.kraftwerk.core.exceptions.KraftwerkException; +import org.assertj.core.api.Assertions; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FileSystemInterfaceTest { + @Test + void testTransformToOut() { + assertEquals(Paths.get("C://Users/in/Kraftwerk/src/test/resources/functional_tests/out/VQS"), + FileUtilsInterface.transformToOut(Paths.get("C://Users/in/Kraftwerk/src/test/resources/functional_tests/in/VQS"))); + } + + @Test + void testTransformToOut2() { + Path path = Paths.get("C://Users/in/Kraftwerk/src/test/resources/functional_tests/out/VQS"); + assertEquals(path, FileUtilsInterface.transformToOut(Paths.get("C://Users/in/Kraftwerk/src/test/resources/functional_tests/in/VQS"), LocalDateTime.now()).getParent()); + } + +} + diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionLogTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionLogTest.java new file mode 100644 index 00000000..ac181309 --- /dev/null +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/log/KraftwerkExecutionLogTest.java @@ -0,0 +1,35 @@ +package fr.insee.kraftwerk.core.utils.log; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.HashMap; + +class KraftwerkExecutionLogTest { + @Test + void getFormattedString_test(){ + //GIVEN + KraftwerkExecutionLog kraftwerkExecutionLog = new KraftwerkExecutionLog(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss.SSS"); + long start = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC); + long stop = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC); + + kraftwerkExecutionLog.setStartTimeStamp(start); + kraftwerkExecutionLog.setEndTimeStamp(stop); + kraftwerkExecutionLog.setOkFileNames(new ArrayList<>()); + kraftwerkExecutionLog.setLineCountByTableMap(new HashMap<>()); + + kraftwerkExecutionLog.getOkFileNames().add("TEST.xml"); + kraftwerkExecutionLog.getLineCountByTableMap().put("RACINE",1); + + //WHEN + String formattedString = kraftwerkExecutionLog.getFormattedString(); + + //THEN + Assertions.assertThat(formattedString).contains("TEST.xml", "RACINE", simpleDateFormat.format(start), simpleDateFormat.format(stop)); + } +} diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformerTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformerTest.java index c2947789..07b0d933 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformerTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/xsl/SaxonTransformerTest.java @@ -1,8 +1,11 @@ package fr.insee.kraftwerk.core.utils.xsl; +import java.io.IOException; import java.net.MalformedURLException; import java.nio.file.Path; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import org.junit.jupiter.api.Test; import org.xmlunit.assertj3.XmlAssert; @@ -11,17 +14,19 @@ class SaxonTransformerTest { + FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + @Test - void applyXsltScript() throws MalformedURLException { + void applyXsltScript() throws IOException { String xsltTestScript = TestConstants.UNIT_TESTS_DIRECTORY + "/utils/xsl/do-nothing.xsl"; String inXmlFile = TestConstants.UNIT_TESTS_DIRECTORY + "/utils/xsl/note.xml"; String outXmlFile = TestConstants.UNIT_TESTS_DUMP + "/xsl-output.xml"; // - SaxonTransformer saxonTransformer = new SaxonTransformer(); + SaxonTransformer saxonTransformer = new SaxonTransformer(fileUtilsInterface); saxonTransformer.xslTransform(Path.of(inXmlFile), xsltTestScript,Path.of(outXmlFile)); // - String inContent = TextFileReader.readFromPath(Path.of(inXmlFile)); - String outContent = TextFileReader.readFromPath(Path.of(outXmlFile)); + String inContent = TextFileReader.readFromPath(Path.of(inXmlFile), fileUtilsInterface); + String outContent = TextFileReader.readFromPath(Path.of(outXmlFile), fileUtilsInterface); // XmlAssert.assertThat(inContent).and(outContent).areSimilar(); } diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java index 963b58f7..93cf2fb8 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlBindingsTest.java @@ -5,6 +5,8 @@ import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; import fr.insee.vtl.model.InMemoryDataset; @@ -25,8 +27,8 @@ class VtlBindingsTest { private VtlBindings vtlBindings; private List errors; - - VtlExecute vtlExecute = new VtlExecute(); + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + VtlExecute vtlExecute = new VtlExecute(fileUtilsInterface); Dataset ds1 = new InMemoryDataset( List.of( diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriterTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriterTest.java index c4d92e53..12fee50a 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriterTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/vtl/VtlJsonDatasetWriterTest.java @@ -5,6 +5,8 @@ import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.rawdata.SurveyRawDataTest; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.vtl.model.Dataset; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.BeforeEach; @@ -19,8 +21,9 @@ class VtlJsonDatasetWriterTest { private VtlBindings vtlBindings; - - VtlExecute vtlExecute = new VtlExecute(); + + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + VtlExecute vtlExecute = new VtlExecute(fileUtilsInterface); @BeforeEach public void initVtlBindings() { diff --git a/kraftwerk-core/src/test/resources/unit_tests/files/convert_path/test.txt b/kraftwerk-core/src/test/resources/unit_tests/files/convert_path/test.txt new file mode 100644 index 00000000..30d74d25 --- /dev/null +++ b/kraftwerk-core/src/test/resources/unit_tests/files/convert_path/test.txt @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/kraftwerk-core/src/test/resources/unit_tests/files/list_files/file1.txt b/kraftwerk-core/src/test/resources/unit_tests/files/list_files/file1.txt new file mode 100644 index 00000000..e69de29b diff --git a/kraftwerk-core/src/test/resources/unit_tests/files/list_files/file2.json b/kraftwerk-core/src/test/resources/unit_tests/files/list_files/file2.json new file mode 100644 index 00000000..e69de29b diff --git a/kraftwerk-core/src/test/resources/unit_tests/files/list_files/file3.xml b/kraftwerk-core/src/test/resources/unit_tests/files/list_files/file3.xml new file mode 100644 index 00000000..e69de29b diff --git a/kraftwerk-core/src/test/resources/unit_tests/files/move_files/move_files.json b/kraftwerk-core/src/test/resources/unit_tests/files/move_files/move_files.json new file mode 100644 index 00000000..cbfa8755 --- /dev/null +++ b/kraftwerk-core/src/test/resources/unit_tests/files/move_files/move_files.json @@ -0,0 +1,34 @@ +{ + + "campaign": "Differential", + + "survey_data": [ + { + "data_mode": "WEB", + "data_file": "web/web.xml", + "DDI_file": "web/vqs-2021-x00-xforms-ddi.xml", + "data_format": "XFORMS", + "paradata_folder": "paradata", + "reporting_data_file": "suivi/reportingdata.xml", + "mode_specifications": "web/WEB.vtl" + }, + { + "data_mode": "PAPER", + "data_file": "papier/paper.txt", + "DDI_file": "papier/vqs-2021-x00-fo-ddi.xml", + "data_format": "PAPER", + "paradata_folder": "", + "reporting_data_file": "", + "mode_specifications": "" + } + ], + + "multimode_dataset_name": "MULTIMODE", + + "reconciliation_specifications": "", + + "transformation_specifications": "", + + "information_levels_specifications": "" + +} \ No newline at end of file diff --git a/kraftwerk-core/src/test/resources/unit_tests/files/read_files/test.txt b/kraftwerk-core/src/test/resources/unit_tests/files/read_files/test.txt new file mode 100644 index 00000000..84ac6f96 --- /dev/null +++ b/kraftwerk-core/src/test/resources/unit_tests/files/read_files/test.txt @@ -0,0 +1 @@ +hello ! \ No newline at end of file diff --git a/kraftwerk-core/src/test/resources/unit_tests/files/rename_files/kraftwerk.json b/kraftwerk-core/src/test/resources/unit_tests/files/rename_files/kraftwerk.json new file mode 100644 index 00000000..802eca67 --- /dev/null +++ b/kraftwerk-core/src/test/resources/unit_tests/files/rename_files/kraftwerk.json @@ -0,0 +1,24 @@ +{ + + "survey_data": [ + { + "data_mode": "CAPI", + "data_file": "sample_files_to_get/data.xml", + "DDI_file": "sample_files_to_get/ddi.xml", + "data_format": "LUNATIC_XML", + "lunatic_file": "sample_files_to_get/lunatic.json", + "paradata_folder": "", + "reporting_data_file": "sample_files_to_get/reportingdata.xml", + "mode_specifications": "" + } + ], + + "multimode_dataset_name": "MULTIMODE", + + "reconciliation_specifications": "", + + "transformation_specifications": "", + + "information_levels_specifications": "" + +} diff --git a/kraftwerk-core/src/test/resources/unit_tests/genesis/WEB/PARADATA/paradataTest.json b/kraftwerk-core/src/test/resources/unit_tests/genesis/WEB/PARADATA/paradataTest.json new file mode 100644 index 00000000..e18b95b2 --- /dev/null +++ b/kraftwerk-core/src/test/resources/unit_tests/genesis/WEB/PARADATA/paradataTest.json @@ -0,0 +1,159 @@ +{ + "idSu": "0000001", + "events": [ + [ + { + "idSurveyUnit": "0000001", + "typeParadataObject": "orchestrator", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "validationPage", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "click", + "idParadataObject": "validate-button-orchestrator-collect", + "timestamp": 1645807734894 + }, + { + "idSurveyUnit": "0000001", + "typeParadataObject": "orchestrator", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "validationPage", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "click", + "idParadataObject": "agree-sending-modal-button-orchestrator-collect", + "timestamp": 1645807741929 + } + ], + [ + { + "idSurveyUnit": "0000001", + "typeParadataObject": "session", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "session-started", + "idParadataObject": "init-session", + "timestamp": 1645807624027 + }, + { + "idSurveyUnit": "0000001", + "typeParadataObject": "orchestrator", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "orchestrator-create", + "idParadataObject": "init-orchestrator-collect", + "timestamp": 1645807624458 + }, + { + "idSurveyUnit": "0000001", + "typeParadataObject": "orchestrator", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "welcomePage", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "click", + "idParadataObject": "next-button-orchestrator-collect", + "timestamp": 1645807637176 + }, + { + "idSurveyUnit": "0000001", + "typeParadataObject": "orchestrator", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "1", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "click", + "idParadataObject": "next-button-orchestrator-collect", + "timestamp": 1645807674779 + } + ], + [ + { + "idSurveyUnit": "0000001", + "typeParadataObject": "lunatic-input", + "idOrchestrator": "orchestrator-collect", + "responseName": "CADR", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "1", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "selection", + "value": "3", + "idParadataObject": "radio-kb9hlpdc-3", + "timestamp": 1645807679901 + }, + { + "idSurveyUnit": "0000001", + "typeParadataObject": "orchestrator", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "2", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "click", + "idParadataObject": "next-button-orchestrator-collect", + "timestamp": 1645807684715 + }, + { + "idSurveyUnit": "0000001", + "typeParadataObject": "lunatic-input", + "idOrchestrator": "orchestrator-collect", + "responseName": "INDNVOCC", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "2", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "selection", + "value": "2", + "idParadataObject": "radio-kqweh61i-2", + "timestamp": 1645807690025 + }, + { + "idSurveyUnit": "0000001", + "typeParadataObject": "orchestrator", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "4", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "click", + "idParadataObject": "next-button-orchestrator-collect", + "timestamp": 1645807691450 + }, + { + "idSurveyUnit": "0000001", + "typeParadataObject": "orchestrator", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "184", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "click", + "idParadataObject": "next-button-orchestrator-collect", + "timestamp": 1645807730751 + } + ], + [ + { + "idSurveyUnit": "0000001", + "typeParadataObject": "orchestrator", + "idOrchestrator": "orchestrator-collect", + "idQuestionnaire": "TESTQUEST", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "page": "endPage", + "idSession": "593d88e6-3f11-49a2-9727-92a8e13fb39f", + "type": "click", + "idParadataObject": "logout-close-button-orchestrator-collect", + "timestamp": 1645807759129 + } + ] + ] +} \ No newline at end of file diff --git a/kraftwerk-core/src/test/resources/unit_tests/genesis/WEB/REPORTING_DATA/reportingdata.csv b/kraftwerk-core/src/test/resources/unit_tests/genesis/WEB/REPORTING_DATA/reportingdata.csv new file mode 100644 index 00000000..ffea39f6 --- /dev/null +++ b/kraftwerk-core/src/test/resources/unit_tests/genesis/WEB/REPORTING_DATA/reportingdata.csv @@ -0,0 +1,2 @@ +statut,dateInfo,idUe,idContact,nom,prenom,adresse,numeroDeLot +INITLA,14/02/2022 14:35:34,0000001,XXXXXXX,NAME,FirstName,13 rue du Général hulot Insee Grand Est 54042 - Nancy cedex,99 \ No newline at end of file diff --git a/kraftwerk-core/src/test/resources/unit_tests/genesis/WEB/REPORTING_DATA/reportingdata.xml b/kraftwerk-core/src/test/resources/unit_tests/genesis/WEB/REPORTING_DATA/reportingdata.xml new file mode 100644 index 00000000..af9cc6e8 --- /dev/null +++ b/kraftwerk-core/src/test/resources/unit_tests/genesis/WEB/REPORTING_DATA/reportingdata.xml @@ -0,0 +1,155 @@ + + + TESTCAMPAIGN + + + 1686292973000 + 1688884973000 + + + + + 0000001 + E9LC05 + OU-TEST1 + false + + M. TEST TESTMAN + + + 1 AVENUE DES TESTS + + 01000 TESTVILLE + + + + + false + false + + + 0 + 0 + 0 + 1 + 202 + 11 + 1 + 0 + 0 + + 0 + + + + Mister + TEST + TESTMAN + toto@titi.fr + true + false + 01/01/1970 + + + 06000000 + interviewer + true + + + + + + + NVM + 1686306249642 + + + ANV + 1686306426503 + + + VIN + 1686306436167 + + + FIN + 1686749244026 + + + VIC + 1686313807830 + + + PRC + 1686313812283 + + + INS + 1686314711901 + + + WFT + 1686315212517 + + + AOC + 1686315494389 + + + APS + 1686315494390 + + + WFT + 1686315494391 + + + INS + 1686315520098 + + + WFT + 1686315525950 + + + APS + 1686315539138 + + + WFT + 1686315539139 + + + WFS + 1686315551319 + + + TBR + 1686315557403 + + + + + INTERVIEWER + ras + + + MANAGEMENT + + + + + INA + 1686315536222 + 1 + + + + INA + TEL + 1686315468266 + + + + + diff --git a/kraftwerk-core/src/test/resources/unit_tests/move_files/move_files.json b/kraftwerk-core/src/test/resources/unit_tests/move_files/move_files.json deleted file mode 100644 index c1d5cbbc..00000000 --- a/kraftwerk-core/src/test/resources/unit_tests/move_files/move_files.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - - "campaign": "Differential", - - "survey_data": [ - { - "data_mode": "WEB", - "data_file": "web/web.xml", - "DDI_file": "web/vqs-2021-x00-xforms-ddi.xml", - "data_format": "XFORMS", - "paradata_folder": "paradata", - "reporting_data_file": "suivi/reportingdata.xml", - "mode_specifications": "web/WEB.vtl" - }, - { - "data_mode": "PAPER", - "data_file": "papier/paper.txt", - "DDI_file": "papier/vqs-2021-x00-fo-ddi.xml", - "data_format": "PAPER", - "paradata_folder": "", - "reporting_data_file": "", - "mode_specifications": "" - } - ], - - "multimode_dataset_name": "MULTIMODE", - - "reconciliation_specifications": "", - - "transformation_specifications": "", - - "information_levels_specifications": "" - -} diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java index 77819d51..5ca525fc 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/CalculatedProcessingDefinition.java @@ -13,6 +13,7 @@ import fr.insee.kraftwerk.core.parsers.DataParser; import fr.insee.kraftwerk.core.parsers.LunaticXmlDataParser; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.vtl.VtlBindings; import fr.insee.kraftwerk.core.vtl.VtlExecute; import fr.insee.vtl.model.Dataset; @@ -43,7 +44,7 @@ public class CalculatedProcessingDefinition { private Dataset outDataset; private List variableNamesList; - VtlExecute vtlExecute = new VtlExecute(); + VtlExecute vtlExecute = new VtlExecute(new FileSystemImpl()); List errors = new ArrayList<>(); @ParameterType("(?:[^,]*)(?:,\\s?[^,]*)*") @@ -64,11 +65,11 @@ public void getCampaignFiles(String campaignName, String dataMode) throws Malfor this.dataMode = dataMode; // metadataModel = DDIReader.getMetadataFromDDI( - Constants.convertToUrl(campaignPacks.get(campaignName).get(dataMode).get("ddi"))); + Constants.convertToUrl(campaignPacks.get(campaignName).get(dataMode).get("ddi")).toString(), new FileSystemImpl()); // SurveyRawData data = new SurveyRawData(); data.setMetadataModel(metadataModel); - DataParser parser = new LunaticXmlDataParser(data); + DataParser parser = new LunaticXmlDataParser(data, new FileSystemImpl()); parser.parseSurveyData(Paths.get(campaignPacks.get(campaignName).get(dataMode).get("data")),null); // vtlBindings = new VtlBindings(); @@ -79,8 +80,8 @@ public void getCampaignFiles(String campaignName, String dataMode) throws Malfor public void readCampaignData() { // CalculatedVariables calculatedVariables = LunaticReader.getCalculatedFromLunatic( - Path.of(campaignPacks.get(campaignName).get(dataMode).get("lunatic"))); - DataProcessing calculatedProcessing = new CalculatedProcessing(vtlBindings,calculatedVariables); + Path.of(campaignPacks.get(campaignName).get(dataMode).get("lunatic")), new FileSystemImpl()); + DataProcessing calculatedProcessing = new CalculatedProcessing(vtlBindings,calculatedVariables, new FileSystemImpl()); calculatedProcessing.applyVtlTransformations("TEST", null,errors); // outDataset = vtlBindings.getDataset("TEST"); diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/DDIGetterDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/DDIGetterDefinitions.java index 0ca3d85c..8646bc65 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/DDIGetterDefinitions.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/DDIGetterDefinitions.java @@ -2,6 +2,7 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.utils.TextFileReader; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import fr.insee.kraftwerk.core.utils.xsl.SaxonTransformer; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; @@ -42,10 +43,10 @@ public void compare_ddi(String nameDDI) throws IOException, URISyntaxException { tempFile.deleteOnExit(); Path tempPath = Paths.get(tempFile.getAbsolutePath()); URL url = new URI(linkDDI).toURL(); - SaxonTransformer transformer = new SaxonTransformer(); - transformer.xslTransform(url, Constants.XSLT_STRUCTURED_VARIABLES, tempPath); + SaxonTransformer transformer = new SaxonTransformer(new FileSystemImpl()); + transformer.xslTransform(url.toString(), Constants.XSLT_STRUCTURED_VARIABLES, tempPath); - actualString = TextFileReader.readFromPath(tempPath); + actualString = TextFileReader.readFromPath(tempPath, new FileSystemImpl()); if (actualString.trim().contains("")) { actualAnswer = "DDI collected"; diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java index 2a0778b1..287186a0 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java @@ -10,6 +10,7 @@ import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.KraftwerkError; import fr.insee.kraftwerk.core.exceptions.KraftwerkException; +import fr.insee.kraftwerk.core.exceptions.NullException; import fr.insee.kraftwerk.core.inputs.UserInputsFile; import fr.insee.kraftwerk.core.metadata.MetadataModel; import fr.insee.kraftwerk.core.metadata.MetadataUtils; @@ -17,7 +18,8 @@ import fr.insee.kraftwerk.core.outputs.OutputFiles; import fr.insee.kraftwerk.core.outputs.csv.CsvOutputFiles; import fr.insee.kraftwerk.core.sequence.*; -import fr.insee.kraftwerk.core.utils.FileUtils; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import fr.insee.kraftwerk.core.utils.SqlUtils; import fr.insee.kraftwerk.core.vtl.VtlBindings; import io.cucumber.java.AfterAll; @@ -72,7 +74,8 @@ public class MainDefinitions { @BeforeAll public static void clean() throws KraftwerkException, SQLException { - FileUtils.deleteDirectory(outDirectory); + FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); + fileUtilsInterface.deleteDirectory(outDirectory); database = SqlUtils.openConnection(); } @@ -83,7 +86,7 @@ public void launch_all_steps(String campaignDirectoryName) { this.campaignName = campaignDirectoryName; inDirectory = inDirectory.resolve(campaignName); outDirectory = outDirectory.resolve(campaignName); - controlInputSequence = new ControlInputSequence(inDirectory.toString()); + controlInputSequence = new ControlInputSequence(inDirectory.toString(), new FileSystemImpl()); // We clean the output and the temp directory deleteDirectory(outDirectory.toFile()); deleteDirectory(tempDirectory.toFile()); @@ -106,19 +109,19 @@ public void init_VTL_script(String vtlScriptName, String variableToCreate, Strin @When("Step 1 : We initialize the input files") public void initialize_input_files() throws KraftwerkException { System.out.println("InDirectory value : " + inDirectory); - userInputs = controlInputSequence.getUserInputs(inDirectory); + userInputs = controlInputSequence.getUserInputs(inDirectory, new FileSystemImpl()); vtlBindings = new VtlBindings(); } @When("Step 1 : We initialize with input file {string}") public void initialize_with_specific_input(String inputFileName) throws KraftwerkException { - userInputs = new UserInputsFile(inDirectory.resolve(inputFileName), inDirectory); + userInputs = new UserInputsFile(inDirectory.resolve(inputFileName), inDirectory, new FileSystemImpl()); vtlBindings = new VtlBindings(); } @When("Step 1 : We initialize metadata model with lunatic specification only") public void initialize_metadata_model_with_lunatic() throws KraftwerkException { - MainProcessing mp = new MainProcessing(inDirectory.toString(), false,false,false, "defaultDirectory", 419430400L); + MainProcessing mp = new MainProcessing(inDirectory.toString(), false,false,false, "defaultDirectory", 419430400L, new FileSystemImpl()); mp.init(); userInputs=mp.getUserInputsFile(); metadataModelMap=mp.getMetadataModels(); @@ -126,7 +129,7 @@ public void initialize_metadata_model_with_lunatic() throws KraftwerkException { @When("Step 1 : We initialize metadata model with DDI specification only") public void initialize_metadata_model_with_DDI() throws KraftwerkException { - MainProcessing mp = new MainProcessing(inDirectory.toString(), false,false,true, "defaultDirectory", 419430400L); + MainProcessing mp = new MainProcessing(inDirectory.toString(), false,false,true, "defaultDirectory", 419430400L, new FileSystemImpl()); mp.init(); userInputs=mp.getUserInputsFile(); metadataModelMap=mp.getMetadataModels(); @@ -137,7 +140,7 @@ public void launch_main() throws KraftwerkException { // We clean the output and the temp directory deleteDirectory(outDirectory.toFile()); deleteDirectory(tempDirectory.toFile()); - MainProcessing mp = new MainProcessing(inDirectory.toString(), false, "defaultDirectory", 419430400L); + MainProcessing mp = new MainProcessing(inDirectory.toString(), false, "defaultDirectory", 419430400L, new FileSystemImpl()); mp.runMain(); } @@ -146,10 +149,10 @@ public void launch_main_2() throws KraftwerkException { // We clean the output and the temp directory deleteDirectory(outDirectory.toFile()); deleteDirectory(tempDirectory.toFile()); - MainProcessing mp = new MainProcessing(inDirectory.toString(), false, "defaultDirectory", 419430400L); + MainProcessing mp = new MainProcessing(inDirectory.toString(), false, "defaultDirectory", 419430400L, new FileSystemImpl()); mp.runMain(); await().atMost(2, TimeUnit.SECONDS); - mp = new MainProcessing(inDirectory.toString(), false, "defaultDirectory", 419430400L); + mp = new MainProcessing(inDirectory.toString(), false, "defaultDirectory", 419430400L, new FileSystemImpl()); mp.runMain(); } @@ -159,20 +162,20 @@ public void launch_main_filebyfile() throws KraftwerkException { deleteDirectory(outDirectory.toFile()); deleteDirectory(tempDirectory.toFile()); MainProcessing mp = new MainProcessing(inDirectory.toString(), true, - Paths.get(FUNCTIONAL_TESTS_INPUT_DIRECTORY).resolve(campaignName).toString(), 419430400L); + Paths.get(FUNCTIONAL_TESTS_INPUT_DIRECTORY).resolve(campaignName).toString(), 419430400L, new FileSystemImpl()); mp.runMain(); } @When("Step 2 : We get each unimodal dataset") - public void unimodal_treatments() throws KraftwerkException, SQLException { + public void unimodal_treatments() throws KraftwerkException, SQLException, NullException { try (Statement statement = database.createStatement()) { - metadataModelMap = MetadataUtils.getMetadata(userInputs.getModeInputsMap()); - BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(true); + metadataModelMap = MetadataUtils.getMetadata(userInputs.getModeInputsMap(), new FileSystemImpl()); + BuildBindingsSequence buildBindingsSequence = new BuildBindingsSequence(true, new FileSystemImpl()); for (String dataMode : userInputs.getModeInputsMap().keySet()) { boolean withDDI = true; buildBindingsSequence.buildVtlBindings(userInputs, dataMode, vtlBindings, metadataModelMap.get(dataMode), withDDI, null); UnimodalSequence unimodal = new UnimodalSequence(); - unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, errors, metadataModelMap); + unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, errors, metadataModelMap, new FileSystemImpl()); } } } @@ -181,7 +184,7 @@ public void unimodal_treatments() throws KraftwerkException, SQLException { public void aggregate_datasets() throws SQLException { MultimodalSequence multimodalSequence = new MultimodalSequence(); try (Statement statement = database.createStatement()) { - multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModelMap); + multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModelMap, new FileSystemImpl()); } } @@ -190,9 +193,9 @@ public void export_results() throws KraftwerkException, SQLException { try (Statement statement = database.createStatement()) { WriterSequence writerSequence = new WriterSequence(); LocalDateTime localDateTime = LocalDateTime.now(); - writerSequence.writeOutputFiles(inDirectory, localDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModelMap, errors, null, statement); + writerSequence.writeOutputFiles(inDirectory, localDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModelMap, errors, null, statement, new FileSystemImpl()); writeErrorsFile(inDirectory, localDateTime, errors); - outputFiles = new CsvOutputFiles(outDirectory, vtlBindings, userInputs.getModes(), statement); + outputFiles = new CsvOutputFiles(outDirectory, vtlBindings, userInputs.getModes(), statement, new FileSystemImpl()); } } @@ -372,8 +375,8 @@ void deleteDirectory(File directoryToBeDeleted) { } private void writeErrorsFile(Path inDirectory,LocalDateTime localDateTime, List errors) { - Path tempOutputPath = FileUtils.transformToOut(inDirectory,localDateTime).resolve(Constants.ERRORS_FILE_NAME); - FileUtils.createDirectoryIfNotExist(tempOutputPath.getParent()); + Path tempOutputPath = FileUtilsInterface.transformToOut(inDirectory,localDateTime).resolve(Constants.ERRORS_FILE_NAME); + new FileSystemImpl().createDirectoryIfNotExist(tempOutputPath.getParent()); // Write errors file if (!errors.isEmpty()) { diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/ParadataDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/ParadataDefinitions.java index 90b31d60..664ec562 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/ParadataDefinitions.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/ParadataDefinitions.java @@ -13,6 +13,8 @@ import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.sequence.ControlInputSequence; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; +import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -33,21 +35,22 @@ public class ParadataDefinitions { String userInputFileName = Constants.USER_INPUT_FILE; ModeInputs modeInputs; private ControlInputSequence controlInputSequence; + private final FileUtilsInterface fileUtilsInterface = new FileSystemImpl(); @Given("We read data from input named {string}") public void launch_all_steps(String campaignName) throws KraftwerkException { Path campaignDirectory = Paths.get(FUNCTIONAL_TESTS_INPUT_DIRECTORY).resolve(campaignName); - controlInputSequence = new ControlInputSequence(campaignDirectory.toString()); - UserInputsFile userInputs = controlInputSequence.getUserInputs(campaignDirectory); + controlInputSequence = new ControlInputSequence(campaignDirectory.toString(), new FileSystemImpl()); + UserInputsFile userInputs = controlInputSequence.getUserInputs(campaignDirectory, fileUtilsInterface); // For now, only one file String modeName = userInputs.getModes().getFirst(); modeInputs = userInputs.getModeInputs(modeName); // parse data data = new SurveyRawData(); data.setDataMode(modeInputs.getDataMode()); - data.setMetadataModel(DDIReader.getMetadataFromDDI(modeInputs.getDdiUrl())); - DataParser parser = DataParserManager.getParser(modeInputs.getDataFormat(), data); + data.setMetadataModel(DDIReader.getMetadataFromDDI(modeInputs.getDdiUrl(), fileUtilsInterface)); + DataParser parser = DataParserManager.getParser(modeInputs.getDataFormat(), data, fileUtilsInterface); parser.parseSurveyData(modeInputs.getDataFile(),null); // get paradata folder paradataFolder = modeInputs.getParadataFolder(); @@ -57,7 +60,7 @@ public void launch_all_steps(String campaignName) throws KraftwerkException { public void collect_paradata_test() throws NullException { if (modeInputs.getParadataFolder() != null && !modeInputs.getParadataFolder().toString().contentEquals("")) { - ParadataParser paraDataParser = new ParadataParser(); + ParadataParser paraDataParser = new ParadataParser(fileUtilsInterface); paradata = new Paradata(paradataFolder); paraDataParser.parseParadata(paradata, data); paradata.getListParadataUE().get(0).getEvents().stream().forEach(e -> log.debug(e.getTimestamp() + ","+ e.getIdParadataObject())); diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/VariablesGetterDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/VariablesGetterDefinitions.java index 4d6adef4..9680470e 100644 --- a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/VariablesGetterDefinitions.java +++ b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/VariablesGetterDefinitions.java @@ -3,6 +3,7 @@ import fr.insee.kraftwerk.core.exceptions.KraftwerkException; import fr.insee.kraftwerk.core.metadata.DDIReader; import fr.insee.kraftwerk.core.metadata.MetadataModel; +import fr.insee.kraftwerk.core.utils.files.FileSystemImpl; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -31,7 +32,7 @@ public void set_linkDDI(String linkDDI) throws MalformedURLException, URISyntaxE @When("I try to collect the variables's infos") public void collect_variables() throws KraftwerkException { - metadataModel = DDIReader.getMetadataFromDDI(linkDDI); + metadataModel = DDIReader.getMetadataFromDDI(linkDDI.toString(), new FileSystemImpl()); } @Then("The variables I try to count should answer {int} and have {string} in it") diff --git a/pom.xml b/pom.xml index 3cc6a905..0d8ffb95 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,10 @@ + + org.springframework.boot + spring-boot-starter-actuator +