SQL insert sequence

InseeFr · Jul 5, 2024 · d22fddb · d22fddb
1 parent bec49dd
commit d22fddb
Show file tree

Hide file tree

Showing 8 changed files with 44 additions and 20 deletions.
diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java
@@ -88,8 +88,9 @@ public void runMain() throws KraftwerkException {
 				this.userInputsFile = userFile;
 				vtlBindings = new VtlBindings();
 				unimodalProcess();
+				multimodalProcess();
 				try(Statement writeDatabase = writeDatabaseConnection.createStatement()){
-					multimodalProcess(writeDatabase);
+					insertDatabase(writeDatabase);
 				}
 			}
 			//Export from database
@@ -148,12 +149,18 @@ private void unimodalProcess() throws KraftwerkException {
 	}
 
 	/* Step 3 : multimodal VTL data processing */
-	private void multimodalProcess(Statement database){
+	private void multimodalProcess(){
 		MultimodalSequence multimodalSequence = new MultimodalSequence();
-		multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModels, database);
+		multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModels);
 	}
 
-	/* Step 4 : Write output files */
+	/* Step 4 : Insert into SQL database */
+	private void insertDatabase(Statement database) throws SQLException {
+		InsertDatabaseSequence insertDatabaseSequence = new InsertDatabaseSequence();
+		insertDatabaseSequence.insertDatabaseProcessing(vtlBindings, database);
+	}
+
+	/* Step 5 : Write output files */
 	private void outputFileWriter(Statement database) throws KraftwerkException {
 		WriterSequence writerSequence = new WriterSequence();
 		writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputsFile.getModeInputsMap(), metadataModels, errors, database);

diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesis.java
@@ -11,6 +11,7 @@
 import fr.insee.kraftwerk.core.metadata.MetadataUtilsGenesis;
 import fr.insee.kraftwerk.core.sequence.BuildBindingsSequenceGenesis;
 import fr.insee.kraftwerk.core.sequence.ControlInputSequenceGenesis;
+import fr.insee.kraftwerk.core.sequence.InsertDatabaseSequence;
 import fr.insee.kraftwerk.core.sequence.MultimodalSequence;
 import fr.insee.kraftwerk.core.sequence.UnimodalSequence;
 import fr.insee.kraftwerk.core.sequence.WriterSequence;
@@ -96,6 +97,7 @@ public void runMain(String idCampaign) throws KraftwerkException, IOException {
 					vtlBindings = new VtlBindings();
 					unimodalProcess(suLatest);
 					multimodalProcess();
+					insertDatabase();
 					outputFileWriter();
 					writeErrors();
 				}
@@ -109,7 +111,7 @@ public void runMain(String idCampaign) throws KraftwerkException, IOException {
 	private void unimodalProcess(List<SurveyUnitUpdateLatest> suLatest) throws KraftwerkException {
 		BuildBindingsSequenceGenesis buildBindingsSequenceGenesis = new BuildBindingsSequenceGenesis();
 		for (String dataMode : userInputs.getModeInputsMap().keySet()) {
-			buildBindingsSequenceGenesis.buildVtlBindings(dataMode, vtlBindings, metadataModels, suLatest, inDirectory, database);
+			buildBindingsSequenceGenesis.buildVtlBindings(dataMode, vtlBindings, metadataModels, suLatest, inDirectory);
 			UnimodalSequence unimodal = new UnimodalSequence();
 			unimodal.applyUnimodalSequence(userInputs, dataMode, vtlBindings, errors, metadataModels);
 		}
@@ -118,10 +120,16 @@ private void unimodalProcess(List<SurveyUnitUpdateLatest> suLatest) throws Kraft
 	/* Step 3 : multimodal VTL data processing */
 	private void multimodalProcess() {
 		MultimodalSequence multimodalSequence = new MultimodalSequence();
-		multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModels, database);
+		multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModels);
 	}
 
-	/* Step 4 : Write output files */
+	/* Step 4 : Insert into SQL database */
+	private void insertDatabase(){
+		InsertDatabaseSequence insertDatabaseSequence = new InsertDatabaseSequence();
+		insertDatabaseSequence.insertDatabaseProcessing(vtlBindings, database);
+	}
+
+	/* Step 5 : Write output files */
 	private void outputFileWriter() throws KraftwerkException {
 		WriterSequence writerSequence = new WriterSequence();
 		writerSequence.writeOutputFiles(inDirectory, executionDateTime, vtlBindings, userInputs.getModeInputsMap(), metadataModels, errors, database);

diff --git a/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java b/kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/StepByStepService.java
@@ -180,7 +180,7 @@ public ResponseEntity<String> multimodalProcessing(
 		//Process
 		try(Connection database = SqlUtils.openConnection()) {
 			MultimodalSequence multimodalSequence = new MultimodalSequence();
-			multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModelMap, database.createStatement());
+			multimodalSequence.multimodalProcessing(userInputsFile, vtlBindings, errors, metadataModelMap);
 		}catch (SQLException e){
 			return ResponseEntity.status(500).body(e.getMessage());
 		}

diff --git a/...erk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java b/...erk-core/src/main/java/fr/insee/kraftwerk/core/sequence/BuildBindingsSequenceGenesis.java
@@ -22,7 +22,6 @@
 
 import java.io.File;
 import java.nio.file.Path;
-import java.sql.Statement;
 import java.util.List;
 import java.util.Map;
 
@@ -34,7 +33,7 @@ public BuildBindingsSequenceGenesis() {
 		vtlExecute = new VtlExecute();
 	}
 
-	public void buildVtlBindings(String dataMode, VtlBindings vtlBindings, Map<String, MetadataModel> metadataModels, List<SurveyUnitUpdateLatest> surveyUnits, Path inDirectory, Statement database) throws KraftwerkException {
+	public void buildVtlBindings(String dataMode, VtlBindings vtlBindings, Map<String, MetadataModel> metadataModels, List<SurveyUnitUpdateLatest> surveyUnits, Path inDirectory) throws KraftwerkException {
 		SurveyRawData data = new SurveyRawData();
 
 		/* Step 2.0 : Read the DDI file (and Lunatic Json for missing variables) to get survey variables */

diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/InsertDatabaseSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/InsertDatabaseSequence.java
@@ -0,0 +1,14 @@
+package fr.insee.kraftwerk.core.sequence;
+
+import fr.insee.kraftwerk.core.utils.SqlUtils;
+import fr.insee.kraftwerk.core.vtl.VtlBindings;
+import lombok.NoArgsConstructor;
+
+import java.sql.Statement;
+
+@NoArgsConstructor
+public class InsertDatabaseSequence {
+    public void insertDatabaseProcessing(VtlBindings vtlBindings, Statement database){
+        SqlUtils.convertVtlBindingsIntoSqlDatabase(vtlBindings, database);
+    }
+}
diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/MultimodalSequence.java
@@ -1,6 +1,5 @@
 package fr.insee.kraftwerk.core.sequence;
 
-import java.sql.Statement;
 import java.util.List;
 import java.util.Map;
 
@@ -14,15 +13,14 @@
 import fr.insee.kraftwerk.core.inputs.UserInputs;
 import fr.insee.kraftwerk.core.metadata.MetadataModel;
 import fr.insee.kraftwerk.core.utils.FileUtils;
-import fr.insee.kraftwerk.core.utils.SqlUtils;
 import fr.insee.kraftwerk.core.utils.TextFileWriter;
 import fr.insee.kraftwerk.core.vtl.VtlBindings;
 import lombok.NoArgsConstructor;
 
 @NoArgsConstructor
 public class MultimodalSequence {
 
-	public void multimodalProcessing(UserInputs userInputs, VtlBindings vtlBindings, List<KraftwerkError> errors, Map<String, MetadataModel> metadataModels, Statement database){
+	public void multimodalProcessing(UserInputs userInputs, VtlBindings vtlBindings, List<KraftwerkError> errors, Map<String, MetadataModel> metadataModels){
 		String multimodeDatasetName = Constants.MULTIMODE_DATASET_NAME;
 
 		/* Step 3.1 : aggregate unimodal datasets into a multimodal unique dataset */
@@ -47,8 +45,5 @@ public void multimodalProcessing(UserInputs userInputs, VtlBindings vtlBindings,
 		vtlGenerate = informationLevelsProcessing.applyVtlTransformations(multimodeDatasetName,
 				userInputs.getVtlInformationLevelsFile(), errors);
 		TextFileWriter.writeFile(FileUtils.getTempVtlFilePath(userInputs, "InformationLevelsProcessing",multimodeDatasetName), vtlGenerate);
-
-		/* Step 3.4: Convert VTL datasets into SQL tables for export */
-		SqlUtils.convertVtlBindingsIntoSqlDatabase(vtlBindings, database);
 	}
 }
diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/sequence/WriterSequence.java
@@ -8,6 +8,7 @@
 import fr.insee.kraftwerk.core.outputs.csv.CsvOutputFiles;
 import fr.insee.kraftwerk.core.outputs.parquet.ParquetOutputFiles;
 import fr.insee.kraftwerk.core.utils.FileUtils;
+import fr.insee.kraftwerk.core.utils.SqlUtils;
 import fr.insee.kraftwerk.core.utils.log.KraftwerkExecutionLog;
 import fr.insee.kraftwerk.core.vtl.VtlBindings;
 import lombok.NoArgsConstructor;
@@ -31,17 +32,17 @@ public void writeOutputFiles(Path inDirectory,LocalDateTime executionDateTime, V
 
 	//Write CSV
 	private void writeCsvFiles(Path outDirectory, VtlBindings vtlBindings, Map<String, ModeInputs> modeInputsMap, Map<String, MetadataModel> metadataModels, List<KraftwerkError> errors, KraftwerkExecutionLog kraftwerkExecutionLog, Statement databaseConnection) throws KraftwerkException {
-		/* Step 4.2 : write csv output tables */
+		/* Step 5.1 : write csv output tables */
 		OutputFiles csvOutputFiles = new CsvOutputFiles(outDirectory, vtlBindings, kraftwerkExecutionLog, new ArrayList<>(modeInputsMap.keySet()), databaseConnection);
 		csvOutputFiles.writeOutputTables(metadataModels);
 
-		/* Step 4.3 : write scripts to import csv tables in several languages */
+		/* Step 5.2 : write scripts to import csv tables in several languages */
 		csvOutputFiles.writeImportScripts(metadataModels, errors);
 	}
 
 	//Write Parquet
 	private void writeParquetFiles(Path outDirectory, VtlBindings vtlBindings, Map<String, ModeInputs> modeInputsMap, Map<String, MetadataModel> metadataModels, List<KraftwerkError> errors, Statement databaseConnection) throws KraftwerkException {
-		/* Step 4.4 : write parquet output tables */
+		/* Step 5.3 : write parquet output tables */
 		OutputFiles parquetOutputFiles = new ParquetOutputFiles(outDirectory, vtlBindings,  new ArrayList<>(modeInputsMap.keySet()), databaseConnection);
 		parquetOutputFiles.writeOutputTables(metadataModels);
 		parquetOutputFiles.writeImportScripts(metadataModels, errors);

diff --git a/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java b/kraftwerk-functional-tests/src/test/java/cucumber/functional_tests/MainDefinitions.java
@@ -181,7 +181,7 @@ public void unimodal_treatments() throws KraftwerkException, SQLException {
 	public void aggregate_datasets() throws SQLException {
 		MultimodalSequence multimodalSequence = new MultimodalSequence();
 		try (Statement statement = database.createStatement()) {
-			multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModelMap, statement);
+			multimodalSequence.multimodalProcessing(userInputs, vtlBindings, errors, metadataModelMap);
 		}
 	}