diff --git a/kraftwerk-core/pom.xml b/kraftwerk-core/pom.xml
index d4faa016..d67138a5 100644
--- a/kraftwerk-core/pom.xml
+++ b/kraftwerk-core/pom.xml
@@ -16,7 +16,7 @@
1.5.0
- 1.0.0
+ 1.1.0
diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java
index 18d25344..3eee6111 100644
--- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java
+++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java
@@ -57,6 +57,8 @@ private Constants() {}
public static final String MULTIMODE_DATASET_NAME = "MULTIMODE";
public static final String REPORTING_DATA_GROUP_NAME = "REPORTINGDATA";
public static final String REPORTING_DATA_INTERVIEWER_ID_NULL_PLACEHOLDER = "NON_AFFECTE_";
+ public static final String REPORTING_DATA_INPUT_DATE_FORMAT = "dd/MM/yyyy HH:mm:ss";
+ public static final String REPORTING_DATA_OUTPUT_DATE_FORMAT = "yyyy-MM-dd-hh-mm-ss";
public static final String END_LINE = "\n";
public static final String OUTPUT_FOLDER_DATETIME_PATTERN = "yyyy_MM_dd_HH_mm_ss";
public static final String ERRORS_FILE_NAME = "errors.txt";
diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java
index 5da311e8..4b99cfb2 100644
--- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java
+++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java
@@ -1,26 +1,25 @@
package fr.insee.kraftwerk.core.extradata.reportingdata;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.file.Path;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
-
import com.opencsv.CSVParser;
import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvValidationException;
-
+import fr.insee.kraftwerk.core.Constants;
import fr.insee.kraftwerk.core.exceptions.NullException;
import fr.insee.kraftwerk.core.rawdata.SurveyRawData;
import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface;
import lombok.extern.log4j.Log4j2;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+
@Log4j2
public class CSVReportingDataParser extends ReportingDataParser {
@@ -67,20 +66,19 @@ public void parseReportingData(ReportingData reportingData, SurveyRawData data,
}
}
- public long convertToTimestamp(String rowTimestamp) {
- SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
- dateFormat.setTimeZone(TimeZone.getTimeZone("CET"));
- Date parsedDate = null;
+ public long convertToTimestamp(String dateString) {
+ DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern(Constants.REPORTING_DATA_INPUT_DATE_FORMAT);
+ LocalDateTime parsedDate = null;
try {
- parsedDate = dateFormat.parse(rowTimestamp);
- } catch (ParseException e1) {
- log.error("Parsing error : {}", e1.getMessage());
+ parsedDate = LocalDateTime.parse(dateString, dateFormat);
+ } catch (DateTimeParseException e) {
+ log.error("Parsing error : {}", e.getMessage());
}
if (parsedDate == null) {
log.error("Parsing error : the parsed date is null");
return 0L;
}
- return TimeUnit.MILLISECONDS.toSeconds(parsedDate.getTime());
+ return parsedDate.atZone(ZoneId.of("CET")).toInstant().toEpochMilli();
}
/**
diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java
index e2791196..2898b1c9 100644
--- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java
+++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java
@@ -1,5 +1,8 @@
package fr.insee.kraftwerk.core.extradata.reportingdata;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.util.Date;
import lombok.Getter;
@@ -22,8 +25,8 @@ public ContactAttempt(String status, long timestamp) {
this.timestamp = timestamp;
}
- public Date getDate() {
- return new Date(timestamp);
+ public LocalDateTime getDate() {
+ return LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.of("CET"));
}
diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java
index 524b793b..d6767660 100644
--- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java
+++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java
@@ -7,11 +7,13 @@
import fr.insee.bpm.metadata.model.VariableType;
import fr.insee.kraftwerk.core.rawdata.QuestionnaireData;
import fr.insee.kraftwerk.core.rawdata.SurveyRawData;
-import fr.insee.kraftwerk.core.utils.DateUtils;
import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface;
import lombok.extern.log4j.Log4j2;
-import java.sql.Date;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
@@ -19,6 +21,8 @@
public abstract class ReportingDataParser {
public static final String DATE_SUFFIX = "_DATE";
+ private final DateTimeFormatter reportingDataOutputDateFormat =
+ DateTimeFormatter.ofPattern(Constants.REPORTING_DATA_OUTPUT_DATE_FORMAT);
Group reportingGroup;
private int maxStates = 0;
private int maxAttempts = 0;
@@ -221,7 +225,7 @@ private void addReportingDataUEToQuestionnaire(SurveyRawData surveyRawData, Repo
}
if(reportingDataUE.getSurveyValidationDateTimeStamp() != null){
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME).getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()).putValue(Constants.REPORTING_DATA_SURVEY_VALIDATION_NAME,
- DateUtils.formatLongToString(reportingDataUE.getSurveyValidationDateTimeStamp()));
+ reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getSurveyValidationDateTimeStamp()),ZoneId.of("CET"))));
}
}
@@ -234,12 +238,12 @@ private void addContactAttempts(ReportingDataUE reportingDataUE, QuestionnaireDa
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
.putValue(Constants.OUTCOME_ATTEMPT_SUFFIX_NAME + "_" + k + DATE_SUFFIX,
- DateUtils.formatDateToString(reportingDataUE.getContactAttempts().get(k - 1).getDate()));
+ reportingDataOutputDateFormat.format(reportingDataUE.getContactAttempts().get(k - 1).getDate()));
}
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
.putValue(Constants.LAST_ATTEMPT_DATE,
- DateUtils.formatDateToString(getLastContactAttempt(reportingDataUE).getDate()));
+ reportingDataOutputDateFormat.format(getLastContactAttempt(reportingDataUE).getDate()));
}
private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireData questionnaire) {
@@ -251,7 +255,7 @@ private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireDat
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
.putValue(Constants.OUTCOME_DATE,
- DateUtils.formatDateToString(new Date(contactOutcome.getDateEndContact())));
+ reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(contactOutcome.getDateEndContact()),ZoneId.of("CET"))));
}
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
@@ -267,8 +271,8 @@ private void addStates(ReportingDataUE reportingDataUE, QuestionnaireData questi
StateType.getStateType((reportingDataUE.getStates().get(k - 1)).getStateType()));
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
- .putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX, DateUtils
- .formatDateToString(new Date((reportingDataUE.getStates().get(k - 1)).getTimestamp())));
+ .putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX,
+ reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getStates().get(k - 1).getTimestamp()),ZoneId.of("CET"))));
}
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java
index bbffd981..a1c70119 100644
--- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java
+++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java
@@ -31,7 +31,8 @@ public enum StateType {
STATE22("VALPAP", "Questionnaire validé sur papier"),
STATE23("VALINT", "Questionnaire validé sur internet"),
STATE24("REFUSAL", "Refus de répondre"),
- STATE25("RELANCE", "RELANCE");
+ STATE25("RELANCE", "RELANCE"),
+ STATE26("FOLLOWUP", "RELANCE");
private final String key;
private final String value;
diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java
index 65235b60..ee6840b4 100644
--- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java
+++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java
@@ -1,5 +1,6 @@
package fr.insee.kraftwerk.core.utils;
+import fr.insee.kraftwerk.core.Constants;
import lombok.extern.log4j.Log4j2;
import java.sql.Timestamp;
@@ -22,8 +23,7 @@ public static String getCurrentTimeStamp() {
return sdf.format(timestamp);
}
- public static long convertToTimestamp(String rowTimestamp) {
- SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
+ public static long convertToTimestamp(String rowTimestamp, SimpleDateFormat dateFormat) {
dateFormat.setTimeZone(TimeZone.getTimeZone("CET"));
Date parsedDate;
try {
@@ -32,15 +32,6 @@ public static long convertToTimestamp(String rowTimestamp) {
log.error("Parsing error : {}", e1.getMessage());
return 0L;
}
- return TimeUnit.MILLISECONDS.toSeconds(parsedDate.getTime());
- }
-
- public static String formatLongToString(long l) {
- return formatDateToString(new Date(l));
- }
-
- public static String formatDateToString(Date date) {
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
- return sdf.format(date);
+ return parsedDate.getTime();
}
}
diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java
index 34c91056..1584c285 100644
--- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java
+++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java
@@ -42,6 +42,13 @@ void parseReportingDataTest() throws NullException {
assertEquals("INITLA", reportingDataUE.getStates().get(0).getStateType());
assertEquals("PARTIELINT", reportingDataUE.getStates().get(1).getStateType());
assertEquals("VALINT", reportingDataUE.getStates().get(2).getStateType());
+
+ // Check the reporting data date is correctly parsed
+ assertEquals(1644845734000L, reportingDataUE.getStates().get(0).getTimestamp());
+ assertEquals(1644854405000L, reportingDataUE.getStates().get(1).getTimestamp());
+ assertEquals(1644919206000L, reportingDataUE.getStates().get(2).getTimestamp());
+
+
// Check the reporting data is correctly translated in the output
/* à implémenter ? *//*
"Affectée, non visible enquêteur" et "Questionnaire démarré"*/
@@ -62,10 +69,10 @@ void controlHeaderTest() {
@Test
void convertDateTest() {
CSVReportingDataParser csvReportingDataParser = new CSVReportingDataParser(fileUtilsInterface);
- assertEquals(1645007098, csvReportingDataParser.convertToTimestamp("16/02/2022 11:24:58"));
- assertEquals(1566544132, csvReportingDataParser.convertToTimestamp("23/08/2019 09:08:52"));
- assertEquals(1111111111, csvReportingDataParser.convertToTimestamp("18/03/2005 02:58:31"));
- assertEquals(1, csvReportingDataParser.convertToTimestamp("01/01/1970 01:00:01"));
+ assertEquals(1645007098000L, csvReportingDataParser.convertToTimestamp("16/02/2022 11:24:58"));
+ assertEquals(1566544132000L, csvReportingDataParser.convertToTimestamp("23/08/2019 09:08:52"));
+ assertEquals(1111111111000L, csvReportingDataParser.convertToTimestamp("18/03/2005 02:58:31"));
+ assertEquals(1000L, csvReportingDataParser.convertToTimestamp("01/01/1970 01:00:01"));
}
diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java
index 3ef3f063..f2b43891 100644
--- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java
+++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java
@@ -4,14 +4,18 @@
import org.junit.jupiter.api.Test;
+import java.text.SimpleDateFormat;
+
class DateUtilsTest {
@Test
void convertDateTest() {
- assertEquals(1645007098, DateUtils.convertToTimestamp("16/02/2022 11:24:58"));
- assertEquals(1566544132, DateUtils.convertToTimestamp("23/08/2019 09:08:52"));
- assertEquals(1111111111, DateUtils.convertToTimestamp("18/03/2005 02:58:31"));
- assertEquals(1, DateUtils.convertToTimestamp("01/01/1970 01:00:01"));
+ assertEquals(1645007098000L, DateUtils.convertToTimestamp("16/02/2022 11:24:58", new SimpleDateFormat("dd/MM" +
+ "/yyyy HH:mm:ss")));
+ assertEquals(1566544132000L, DateUtils.convertToTimestamp("23/08/2019 09:08:52", new SimpleDateFormat("dd/MM" +
+ "/yyyy HH:mm:ss")));
+ assertEquals(1111111111000L, DateUtils.convertToTimestamp("18/03/2005 02:58:31", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss")));
+ assertEquals(1000L, DateUtils.convertToTimestamp("01/01/1970 01:00:01", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss")));
}
diff --git a/kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv b/kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv
index 716d3999..220a9256 100644
--- a/kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv
+++ b/kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv
@@ -2,5 +2,5 @@ statut,dateInfo,idUe,idContact,nom,prenom,adresse,numeroDeLot
INITLA,14/02/2022 14:35:34,L0000005,XXXXXXX,NAME,FirstName,13 rue du Général hulot Insee Grand Est 54042 - Nancy cedex,99
INITLA,14/02/2022 14:35:34,L0000006,XXXXXXX,NAME,FirstName,13 rue du Général hulot Insee Grand Est 54042 - Nancy cedex,99
INITLA,14/02/2022 14:35:34,L0000169,XXXXXXX,NAME,FirstName,13 rue du Général hulot 54042 - Nancy cedex,2
-VALINT,15/02/2022 11:00:06,L0000169,XXXXXXX,NAME,FirstName,1 rue du moulin - Paris,1
-PARTIELINT,14/02/2022 17:00:05,L0000169,XXXXXXX,NAME,FirstName,1 rue du pont - Paris,1
+VALINT,"15/02/2022 11:00:06",L0000169,XXXXXXX,NAME,FirstName,1 rue du moulin - Paris,1
+PARTIELINT,"14/02/2022 17:00:05",L0000169,XXXXXXX,NAME,FirstName,1 rue du pont - Paris,1
diff --git a/pom.xml b/pom.xml
index e685ef66..ebf7141b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -11,14 +11,14 @@
org.springframework.boot
spring-boot-starter-parent
- 3.3.3
+ 3.3.4
21
UTF-8
UTF-8
- 7.18.1
+ 7.19.0
0.8.12
jacoco
@@ -26,7 +26,7 @@
UTF-8
true
- 1.16.3
+ 1.17.0
1.2.1