Merge branch 'main' into devDDIReader

kraftwerk-core/pom.xml

@@ -16,7 +16,7 @@
 
 	<properties>
 		<trevas.version>1.5.0</trevas.version>
-		<duckdb.version>1.0.0</duckdb.version>
+		<duckdb.version>1.1.0</duckdb.version>
 	</properties>
 
 

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java

@@ -57,6 +57,8 @@ private Constants() {}
 	public static final String MULTIMODE_DATASET_NAME = "MULTIMODE";
 	public static final String REPORTING_DATA_GROUP_NAME = "REPORTINGDATA";
 	public static final String REPORTING_DATA_INTERVIEWER_ID_NULL_PLACEHOLDER = "NON_AFFECTE_";
+	public static final String REPORTING_DATA_INPUT_DATE_FORMAT = "dd/MM/yyyy HH:mm:ss";
+	public static final String REPORTING_DATA_OUTPUT_DATE_FORMAT = "yyyy-MM-dd-hh-mm-ss";
 	public static final String END_LINE = "\n";
 	public static final String OUTPUT_FOLDER_DATETIME_PATTERN = "yyyy_MM_dd_HH_mm_ss";
 	public static final String ERRORS_FILE_NAME = "errors.txt";

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java

@@ -1,26 +1,25 @@
 package fr.insee.kraftwerk.core.extradata.reportingdata;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.file.Path;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
-
 import com.opencsv.CSVParser;
 import com.opencsv.CSVParserBuilder;
 import com.opencsv.CSVReader;
 import com.opencsv.CSVReaderBuilder;
 import com.opencsv.exceptions.CsvValidationException;
-
+import fr.insee.kraftwerk.core.Constants;
 import fr.insee.kraftwerk.core.exceptions.NullException;
 import fr.insee.kraftwerk.core.rawdata.SurveyRawData;
 import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface;
 import lombok.extern.log4j.Log4j2;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+
 @Log4j2
 public class CSVReportingDataParser extends ReportingDataParser {
 
@@ -67,20 +66,19 @@ public void parseReportingData(ReportingData reportingData, SurveyRawData data,
 		}
 	}
 
-	public long convertToTimestamp(String rowTimestamp) {
-		SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
-		dateFormat.setTimeZone(TimeZone.getTimeZone("CET"));
-		Date parsedDate = null;
+	public long convertToTimestamp(String dateString) {
+		DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern(Constants.REPORTING_DATA_INPUT_DATE_FORMAT);
+		LocalDateTime parsedDate = null;
 		try {
-			parsedDate = dateFormat.parse(rowTimestamp);
-		} catch (ParseException e1) {
-			log.error("Parsing error : {}", e1.getMessage());
+			parsedDate = LocalDateTime.parse(dateString, dateFormat);
+		} catch (DateTimeParseException e) {
+			log.error("Parsing error : {}", e.getMessage());
 		}
 		if (parsedDate == null) {
 			log.error("Parsing error : the parsed date is null");
 			return 0L;
 		}
-		return TimeUnit.MILLISECONDS.toSeconds(parsedDate.getTime());
+		return parsedDate.atZone(ZoneId.of("CET")).toInstant().toEpochMilli();
 	}
 
 	/**

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java

@@ -1,5 +1,8 @@
 package fr.insee.kraftwerk.core.extradata.reportingdata;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.Date;
 
 import lombok.Getter;
@@ -22,8 +25,8 @@ public ContactAttempt(String status, long timestamp) {
     this.timestamp = timestamp;
   }
 
-  public Date getDate() {
-	  return new Date(timestamp);
+  public LocalDateTime getDate() {
+	  return LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.of("CET"));
   }
 
 

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java

@@ -7,18 +7,22 @@
 import fr.insee.bpm.metadata.model.VariableType;
 import fr.insee.kraftwerk.core.rawdata.QuestionnaireData;
 import fr.insee.kraftwerk.core.rawdata.SurveyRawData;
-import fr.insee.kraftwerk.core.utils.DateUtils;
 import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface;
 import lombok.extern.log4j.Log4j2;
 
-import java.sql.Date;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
 
 @Log4j2
 public abstract class ReportingDataParser {
 
 	public static final String DATE_SUFFIX = "_DATE";
+	private final DateTimeFormatter reportingDataOutputDateFormat =
+			DateTimeFormatter.ofPattern(Constants.REPORTING_DATA_OUTPUT_DATE_FORMAT);
 	Group reportingGroup;
 	private int maxStates = 0;
 	private int maxAttempts = 0;
@@ -221,7 +225,7 @@ private void addReportingDataUEToQuestionnaire(SurveyRawData surveyRawData, Repo
 		}
 		if(reportingDataUE.getSurveyValidationDateTimeStamp() != null){
 			questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME).getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()).putValue(Constants.REPORTING_DATA_SURVEY_VALIDATION_NAME,
-					DateUtils.formatLongToString(reportingDataUE.getSurveyValidationDateTimeStamp()));
+					reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getSurveyValidationDateTimeStamp()),ZoneId.of("CET"))));
 		}
 	}
 
@@ -234,12 +238,12 @@ private void addContactAttempts(ReportingDataUE reportingDataUE, QuestionnaireDa
 			questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
 					.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
 					.putValue(Constants.OUTCOME_ATTEMPT_SUFFIX_NAME + "_" + k + DATE_SUFFIX,
-							DateUtils.formatDateToString(reportingDataUE.getContactAttempts().get(k - 1).getDate()));
+							reportingDataOutputDateFormat.format(reportingDataUE.getContactAttempts().get(k - 1).getDate()));
 		}
 		questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
 				.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
 				.putValue(Constants.LAST_ATTEMPT_DATE,
-						DateUtils.formatDateToString(getLastContactAttempt(reportingDataUE).getDate()));
+						reportingDataOutputDateFormat.format(getLastContactAttempt(reportingDataUE).getDate()));
 	}
 
 	private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireData questionnaire) {
@@ -251,7 +255,7 @@ private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireDat
 			questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
 					.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
 					.putValue(Constants.OUTCOME_DATE,
-							DateUtils.formatDateToString(new Date(contactOutcome.getDateEndContact())));
+							reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(contactOutcome.getDateEndContact()),ZoneId.of("CET"))));
 		}
 		questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
 				.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
@@ -267,8 +271,8 @@ private void addStates(ReportingDataUE reportingDataUE, QuestionnaireData questi
 							StateType.getStateType((reportingDataUE.getStates().get(k - 1)).getStateType()));
 			questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
 					.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
-					.putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX, DateUtils
-							.formatDateToString(new Date((reportingDataUE.getStates().get(k - 1)).getTimestamp())));
+					.putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX,
+							reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getStates().get(k - 1).getTimestamp()),ZoneId.of("CET"))));
 		}
 		questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
 				.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java

@@ -31,7 +31,8 @@ public enum StateType {
   STATE22("VALPAP", "Questionnaire validé sur papier"),
   STATE23("VALINT", "Questionnaire validé sur internet"),
   STATE24("REFUSAL", "Refus de répondre"),
-  STATE25("RELANCE", "RELANCE");
+  STATE25("RELANCE", "RELANCE"),
+  STATE26("FOLLOWUP", "RELANCE");
 
 	private final String key;
 	private final String value;

kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java

@@ -1,5 +1,6 @@
 package fr.insee.kraftwerk.core.utils;
 
+import fr.insee.kraftwerk.core.Constants;
 import lombok.extern.log4j.Log4j2;
 
 import java.sql.Timestamp;
@@ -22,8 +23,7 @@ public static String getCurrentTimeStamp() {
 		return sdf.format(timestamp);
 	}
 
-	public static long convertToTimestamp(String rowTimestamp) {
-		SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
+	public static long convertToTimestamp(String rowTimestamp, SimpleDateFormat dateFormat) {
 		dateFormat.setTimeZone(TimeZone.getTimeZone("CET"));
 		Date parsedDate;
 		try {
@@ -32,15 +32,6 @@ public static long convertToTimestamp(String rowTimestamp) {
 			log.error("Parsing error : {}", e1.getMessage());
 			return 0L;
 		}
-		return TimeUnit.MILLISECONDS.toSeconds(parsedDate.getTime());
-	}
-
-	public static String formatLongToString(long l) {
-		return formatDateToString(new Date(l));
-	}
-
-	public static String formatDateToString(Date date) {
-		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
-		return sdf.format(date);
+		return parsedDate.getTime();
 	}
 }

kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java

@@ -42,6 +42,13 @@ void parseReportingDataTest() throws NullException {
 		assertEquals("INITLA", reportingDataUE.getStates().get(0).getStateType());
 		assertEquals("PARTIELINT", reportingDataUE.getStates().get(1).getStateType());
 		assertEquals("VALINT", reportingDataUE.getStates().get(2).getStateType());
+
+		// Check the reporting data date is correctly parsed
+		assertEquals(1644845734000L, reportingDataUE.getStates().get(0).getTimestamp());
+		assertEquals(1644854405000L, reportingDataUE.getStates().get(1).getTimestamp());
+		assertEquals(1644919206000L, reportingDataUE.getStates().get(2).getTimestamp());
+
+
 		// Check the reporting data is correctly translated in the output
 		/* à implémenter ? *//*
 		"Affectée, non visible enquêteur" et "Questionnaire démarré"*/
@@ -62,10 +69,10 @@ void controlHeaderTest() {
 	@Test
 	void convertDateTest() {
 		CSVReportingDataParser csvReportingDataParser = new CSVReportingDataParser(fileUtilsInterface);
-		assertEquals(1645007098, csvReportingDataParser.convertToTimestamp("16/02/2022 11:24:58"));
-		assertEquals(1566544132, csvReportingDataParser.convertToTimestamp("23/08/2019 09:08:52"));
-		assertEquals(1111111111, csvReportingDataParser.convertToTimestamp("18/03/2005 02:58:31"));
-		assertEquals(1, csvReportingDataParser.convertToTimestamp("01/01/1970 01:00:01"));
+		assertEquals(1645007098000L, csvReportingDataParser.convertToTimestamp("16/02/2022 11:24:58"));
+		assertEquals(1566544132000L, csvReportingDataParser.convertToTimestamp("23/08/2019 09:08:52"));
+		assertEquals(1111111111000L, csvReportingDataParser.convertToTimestamp("18/03/2005 02:58:31"));
+		assertEquals(1000L, csvReportingDataParser.convertToTimestamp("01/01/1970 01:00:01"));
 
 	}
 

kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java

@@ -4,14 +4,18 @@
 
 import org.junit.jupiter.api.Test;
 
+import java.text.SimpleDateFormat;
+
 class DateUtilsTest {
 
 	@Test
 	void convertDateTest() {
-		assertEquals(1645007098, DateUtils.convertToTimestamp("16/02/2022 11:24:58"));
-		assertEquals(1566544132, DateUtils.convertToTimestamp("23/08/2019 09:08:52"));
-		assertEquals(1111111111, DateUtils.convertToTimestamp("18/03/2005 02:58:31"));
-		assertEquals(1, DateUtils.convertToTimestamp("01/01/1970 01:00:01"));
+		assertEquals(1645007098000L, DateUtils.convertToTimestamp("16/02/2022 11:24:58", new SimpleDateFormat("dd/MM" +
+				"/yyyy HH:mm:ss")));
+		assertEquals(1566544132000L, DateUtils.convertToTimestamp("23/08/2019 09:08:52", new SimpleDateFormat("dd/MM" +
+				"/yyyy HH:mm:ss")));
+		assertEquals(1111111111000L, DateUtils.convertToTimestamp("18/03/2005 02:58:31", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss")));
+		assertEquals(1000L, DateUtils.convertToTimestamp("01/01/1970 01:00:01", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss")));
 
 	}
 

kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv

@@ -2,5 +2,5 @@ statut,dateInfo,idUe,idContact,nom,prenom,adresse,numeroDeLot
 INITLA,14/02/2022 14:35:34,L0000005,XXXXXXX,NAME,FirstName,13  rue du Général hulot Insee Grand Est  54042 - Nancy cedex,99
 INITLA,14/02/2022 14:35:34,L0000006,XXXXXXX,NAME,FirstName,13  rue du Général hulot Insee Grand Est  54042 - Nancy cedex,99
 INITLA,14/02/2022 14:35:34,L0000169,XXXXXXX,NAME,FirstName,13  rue du Général hulot   54042 - Nancy cedex,2
-VALINT,15/02/2022 11:00:06,L0000169,XXXXXXX,NAME,FirstName,1  rue du moulin - Paris,1
-PARTIELINT,14/02/2022 17:00:05,L0000169,XXXXXXX,NAME,FirstName,1  rue du pont - Paris,1
+VALINT,"15/02/2022 11:00:06",L0000169,XXXXXXX,NAME,FirstName,1  rue du moulin - Paris,1
+PARTIELINT,"14/02/2022 17:00:05",L0000169,XXXXXXX,NAME,FirstName,1  rue du pont - Paris,1

pom.xml

@@ -11,22 +11,22 @@
 	<parent>
 		<groupId>org.springframework.boot</groupId>
 		<artifactId>spring-boot-starter-parent</artifactId>
-		<version>3.3.3</version>
+		<version>3.3.4</version>
 		<relativePath /> <!-- lookup parent from repository -->
 	</parent>
 	<properties>
 		<java.version>21</java.version>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-		<cucumber.version>7.18.1</cucumber.version>
+		<cucumber.version>7.19.0</cucumber.version>
 		<!-- Proprietes sonar -->
 		<jacoco.version>0.8.12</jacoco.version>
 		<sonar.core.codeCoveragePlugin>jacoco</sonar.core.codeCoveragePlugin>
 		<sonar.language>java</sonar.language>
 		<sonar.sourceEncoding>UTF-8</sonar.sourceEncoding>
 		<skipSurefireReport>true</skipSurefireReport>
 		<!-- Pi Test-->
-		<pitest.version>1.16.3</pitest.version>
+		<pitest.version>1.17.0</pitest.version>
 		<pitest.junit.version>1.2.1</pitest.junit.version>
 
 	</properties>