Merge pull request #99 from InseeFr/devDateTimeOutputFolder

Add date time output folder

CHANGELOG.md

@@ -1,8 +1,10 @@
 # Changelog
 
 
-## X.X.X [Not released yet] 
+## X.X.X [Not released yet]
 
+### Changed
+- Output files are stored in separate execution time folders
 
 
 

kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java

@@ -17,6 +17,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -43,6 +44,7 @@ public class MainProcessing {
 	private VtlBindings vtlBindings = new VtlBindings();
 	private KraftwerkExecutionLog kraftwerkExecutionLog;
 	private List<KraftwerkError> errors = new ArrayList<>();
+	private LocalDateTime executionDateTime;
 
 	/**
 	 * Map by mode
@@ -96,6 +98,7 @@ public void runMain() throws KraftwerkException {
 	/* Step 1 : Init */
 	public void init() throws KraftwerkException {
 		kraftwerkExecutionLog = new KraftwerkExecutionLog(); //Init logger
+		this.executionDateTime = LocalDateTime.now();
 		inDirectory = controlInputSequence.getInDirectory(inDirectoryParam);
 
 		String campaignName = inDirectory.getFileName().toString();
@@ -141,7 +144,7 @@ private void multimodalProcess() {
 	/* Step 4 : Write output files */
 	private void outputFileWriter() throws KraftwerkException {
 		WriterSequence writerSequence = new WriterSequence();
-		writerSequence.writeOutputFiles(inDirectory, vtlBindings, userInputsFile.getModeInputsMap(), metadataModels, errors, kraftwerkExecutionLog);
+		writerSequence.writeOutputFiles(inDirectory,executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModels, errors, kraftwerkExecutionLog);
 	}
 
 	/* Step 5 : Write errors */

kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java

@@ -22,6 +22,7 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -37,6 +38,7 @@ public class MainProcessingGenesis {
 	private List<KraftwerkError> errors = new ArrayList<>();
 	@Getter
 	private UserInputsGenesis userInputs;
+	private LocalDateTime executionDateTime;
 
 	/* SPECIFIC VARIABLES */
 	@Getter
@@ -55,6 +57,7 @@ public MainProcessingGenesis(ConfigProperties config) {
 
 	public void init(String idQuestionnaire) throws KraftwerkException, IOException {
 		log.info("Kraftwerk main service started for questionnaire: " + idQuestionnaire);
+		this.executionDateTime = LocalDateTime.now();
 		inDirectory = controlInputSequenceGenesis.getInDirectory(idQuestionnaire);
 		//First we check the modes present in database for the given questionnaire
 		//We build userInputs for the given questionnaire
@@ -101,7 +104,7 @@ private void multimodalProcess() {
 	/* Step 4 : Write output files */
 	private void outputFileWriter() throws KraftwerkException {
 		WriterSequence writerSequence = new WriterSequence();
-		writerSequence.writeOutputFiles(inDirectory, vtlBindings, userInputs.getModeInputsMap(), metadataModels, errors);
+		writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModels, errors);
 	}
 
 	/* Step 5 : Write errors */

kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java

@@ -20,6 +20,7 @@
 
 import java.io.File;
 import java.nio.file.Path;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -199,6 +200,7 @@ public ResponseEntity<String> writeOutputFiles(
 		} catch (KraftwerkException e) {
 			return ResponseEntity.status(e.getStatus()).body(e.getMessage());
 		}
+		LocalDateTime executionDateTime = LocalDateTime.now();
 		VtlBindings vtlBindings = new VtlBindings();
 		List<KraftwerkError> errors = new ArrayList<>();
 		// Read all bindings necessary to produce output
@@ -219,7 +221,7 @@ public ResponseEntity<String> writeOutputFiles(
 			return ResponseEntity.status(e.getStatus()).body(e.getMessage());
 		}
 		Map<String, MetadataModel> metadataModelMap = MetadataUtils.getMetadata(userInputsFile.getModeInputsMap());
-		writerSequence.writeOutputFiles(inDirectory, vtlBindings, userInputsFile.getModeInputsMap(), metadataModelMap, errors);
+		writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModelMap, errors);
 		return ResponseEntity.ok(inDirectoryParam);
 
 	}

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java

@@ -55,6 +55,7 @@ private Constants() {}
 	public static final String REPORTING_DATA_GROUP_NAME = "REPORTINGDATA";
 	public static final String REPORTING_DATA_INTERVIEWER_ID_NULL_PLACEHOLDER = "NON_AFFECTE_";
 	public static final String END_LINE = "\n";
+	public static final String OUTPUT_FOLDER_DATETIME_PATTERN = "yyyy_MM_dd_HH_mm_ss";
 
 
 	// ----- Explicit Variables Names

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/outputs/csv/CsvOutputFiles.java

@@ -74,7 +74,7 @@ public void writeImportScripts(Map<String, MetadataModel> metadataModels, List<K
 	 */
 	@Override
 	public String outputFileName(String datasetName) {
-		return getOutputFolder().getFileName() + "_" + datasetName + ".csv";
+		return getOutputFolder().getParent().getFileName() + "_" + datasetName + ".csv";
 	}
 
 }

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java

@@ -1,5 +1,6 @@
 package fr.insee.kraftwerk.core.sequence;
 
+import fr.insee.kraftwerk.core.Constants;
 import fr.insee.kraftwerk.core.KraftwerkError;
 import fr.insee.kraftwerk.core.exceptions.KraftwerkException;
 import fr.insee.kraftwerk.core.inputs.ModeInputs;
@@ -13,19 +14,22 @@
 import lombok.NoArgsConstructor;
 
 import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 @NoArgsConstructor
 public class WriterSequence {
 
-	public void writeOutputFiles(Path inDirectory, VtlBindings vtlBindings, Map<String, ModeInputs> modeInputsMap, Map<String, MetadataModel> metadataModels, List<KraftwerkError> errors) throws KraftwerkException {
-		writeOutputFiles(inDirectory,vtlBindings,modeInputsMap,metadataModels,errors,null);
+	public void writeOutputFiles(Path inDirectory,LocalDateTime executionDateTime, VtlBindings vtlBindings, Map<String, ModeInputs> modeInputsMap, Map<String, MetadataModel> metadataModels, List<KraftwerkError> errors) throws KraftwerkException {
+		writeOutputFiles(inDirectory,executionDateTime,vtlBindings,modeInputsMap,metadataModels,errors,null);
 	}
 
-	public void writeOutputFiles(Path inDirectory, VtlBindings vtlBindings, Map<String, ModeInputs> modeInputsMap, Map<String, MetadataModel> metadataModels, List<KraftwerkError> errors, KraftwerkExecutionLog kraftwerkExecutionLog) throws KraftwerkException {
+	public void writeOutputFiles(Path inDirectory, LocalDateTime localDateTime, VtlBindings vtlBindings, Map<String, ModeInputs> modeInputsMap, Map<String, MetadataModel> metadataModels, List<KraftwerkError> errors, KraftwerkExecutionLog kraftwerkExecutionLog) throws KraftwerkException {
 		Path outDirectory = FileUtils.transformToOut(inDirectory);
+		outDirectory = outDirectory.resolve(localDateTime.format(DateTimeFormatter.ofPattern(Constants.OUTPUT_FOLDER_DATETIME_PATTERN)));
 		/* Step 4.1 : write csv output tables */
 		OutputFiles csvOutputFiles = new CsvOutputFiles(outDirectory, vtlBindings,  new ArrayList<>(modeInputsMap.keySet()),kraftwerkExecutionLog);
 		csvOutputFiles.writeOutputTables(metadataModels);

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/TextFileWriter.java

@@ -1,9 +1,11 @@
 package fr.insee.kraftwerk.core.utils;
 
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.List;
+import java.util.Objects;
 
 import fr.insee.kraftwerk.core.KraftwerkError;
 import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog;
@@ -54,7 +56,9 @@ public static void writeErrorsFile(Path inDirectory, List<KraftwerkError> errors
 	}
 
 	public static void writeLogFile(Path inDirectory, KraftwerkExecutionLog kraftwerkExecutionLog){
-		Path tempOutputPath = FileUtils.transformToOut(inDirectory).resolve(inDirectory.getFileName() + "_LOG_" + kraftwerkExecutionLog.getStartTimeStamp() +".txt");
+		Path tempOutputPath = FileUtils.transformToOut(inDirectory);
+		tempOutputPath = tempOutputPath.resolve(Objects.requireNonNull(new File(tempOutputPath.toString()).listFiles(File::isDirectory))[0].getName())
+				.resolve(inDirectory.getFileName() + "_LOG_" + kraftwerkExecutionLog.getStartTimeStamp() +".txt");
 		FileUtils.createDirectoryIfNotExist(tempOutputPath.getParent());
 
 		try (FileWriter myWriter = new FileWriter(tempOutputPath.toFile(),true)){

kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/LogFileDefinitions.java

@@ -3,10 +3,12 @@
 import io.cucumber.java.en.Then;
 import org.assertj.core.api.Assertions;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Objects;
 import java.util.stream.Stream;
 
 import static cucumber.TestConstants.FUNCTIONAL_TESTS_OUTPUT_DIRECTORY;
@@ -15,7 +17,10 @@ public class LogFileDefinitions {
     static Path outDirectory = Paths.get(FUNCTIONAL_TESTS_OUTPUT_DIRECTORY);
     @Then("We should have a log file named in directory {string}")
     public void logFileExistenceCheck(String directory) throws IOException {
-        try(Stream<Path> folderStream = Files.list(outDirectory.resolve(directory))){
+        Path executionOutDirectory = outDirectory.resolve(directory);
+        executionOutDirectory = executionOutDirectory.resolve(Objects.requireNonNull(new File(executionOutDirectory.toString()).listFiles(File::isDirectory))[0].getName());
+
+        try(Stream<Path> folderStream = Files.list(executionOutDirectory)){
             Assertions.assertThat(folderStream.filter(path ->
                     path.getFileName().toString().startsWith(directory + "_LOG_")
             )).isNotEmpty();

kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java

@@ -28,10 +28,12 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static cucumber.TestConstants.*;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -140,16 +142,18 @@ public void aggregate_datasets() {
 	@When("Step 4 : We export the final version")
 	public void export_results() throws KraftwerkException {
 		WriterSequence writerSequence = new WriterSequence();
-		writerSequence.writeOutputFiles(inDirectory, vtlBindings, userInputs.getModeInputsMap(), metadataModelMap, errors);
+		writerSequence.writeOutputFiles(inDirectory, LocalDateTime.now(), vtlBindings, userInputs.getModeInputsMap(), metadataModelMap, errors);
 		writeErrorsFile(inDirectory, errors);
 		outputFiles = new CsvOutputFiles(outDirectory, vtlBindings, userInputs.getModes());
 	}
 
 	@Then("Step 5 : We check if we have {int} lines")
 	public void count_lines_in_root_tables(int expectedLineCount) throws CsvValidationException, IOException {
+		// Go to first datetime folder
+		Path executionOutDirectory = outDirectory.resolve(Objects.requireNonNull(new File(outDirectory.toString()).listFiles(File::isDirectory))[0].getName());
 		// Get reader to read the root table written in outputs
 		System.out.println("Check output file path : "
-				+ outDirectory.resolve(outputFiles.outputFileName(Constants.ROOT_GROUP_NAME)));
+				+ executionOutDirectory.resolve(outputFiles.outputFileName(Constants.ROOT_GROUP_NAME)));
 		CSVReader csvReader = CsvUtils
 				.getReader(outDirectory.resolve(outputFiles.outputFileName(Constants.ROOT_GROUP_NAME)));
 		// Count
@@ -166,8 +170,11 @@ public void count_lines_in_root_tables(int expectedLineCount) throws CsvValidati
 	@Then("Step 2 : We check root output file has {int} lines and {int} variables")
 	public void check_output_root_table(int expectedLineCount, int expectedVariablesCount)
 			throws IOException, CsvValidationException {
+		// Go to first datetime folder
+		Path executionOutDirectory = outDirectory.resolve(Objects.requireNonNull(new File(outDirectory.toString()).listFiles(File::isDirectory))[0].getName());
+
 		CSVReader csvReader = CsvUtils
-				.getReader(outDirectory.resolve(outDirectory.getFileName() + "_" + Constants.ROOT_GROUP_NAME + ".csv"));
+				.getReader(executionOutDirectory.resolve(executionOutDirectory.getFileName() + "_" + Constants.ROOT_GROUP_NAME + ".csv"));
 		// get header
 		String[] header = csvReader.readNext();
 		// Count
@@ -263,7 +270,10 @@ private void writeErrorsFile(Path inDirectory, List<KraftwerkError> errors) {
 
 	@Then("In a file named {string} there should be a {string} field")
 	public void check_field_existence(String fileName, String fieldName) throws IOException, CsvValidationException {
-		File outputReportingDataFile = new File(outDirectory + "/" + fileName);
+		// Go to first datetime folder
+		Path executionOutDirectory = outDirectory.resolve(Objects.requireNonNull(new File(outDirectory.toString()).listFiles(File::isDirectory))[0].getName());
+
+		File outputReportingDataFile = new File(executionOutDirectory + "/" + fileName);
 
 		// File existence assertion
 		assertThat(outputReportingDataFile).exists().isFile().canRead();