From b1124a9d3fc2437528ec20ca314bc40c2e548c67 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 10:04:23 +0000 Subject: [PATCH 1/9] Update dependency org.duckdb:duckdb_jdbc to v1.1.0 --- kraftwerk-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kraftwerk-core/pom.xml b/kraftwerk-core/pom.xml index 4698a4aa..75fb1f1f 100644 --- a/kraftwerk-core/pom.xml +++ b/kraftwerk-core/pom.xml @@ -16,7 +16,7 @@ 1.5.0 - 1.0.0 + 1.1.0 From e20530e85ded5313a1341ee3314a418842601f29 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 10:39:46 +0000 Subject: [PATCH 2/9] Update dependency org.pitest:pitest-maven to v1.17.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2825c2d1..3550d983 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ UTF-8 true - 1.16.3 + 1.17.0 1.2.1 From 0a4a84b5f54b343eab3260d3a811b8463ce60bd7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 19 Sep 2024 13:39:24 +0000 Subject: [PATCH 3/9] Update dependency org.springframework.boot:spring-boot-starter-parent to v3.3.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2825c2d1..26667def 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.springframework.boot spring-boot-starter-parent - 3.3.3 + 3.3.4 From 72b0b9cd50867138efd782a57fcb72f61527f2f1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 19 Sep 2024 17:11:58 +0000 Subject: [PATCH 4/9] Update cucumber.version to v7.19.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2825c2d1..e5f403e1 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 21 UTF-8 UTF-8 - 7.18.1 + 7.19.0 0.8.12 jacoco From 919ef6ee99086e46eacb574706ecea12ef2506fe Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Mon, 23 Sep 2024 14:30:44 +0200 Subject: [PATCH 5/9] Fixed wrong xml reporting data FOLLOWUP + csv date format --- .../java/fr/insee/kraftwerk/core/Constants.java | 2 ++ .../reportingdata/CSVReportingDataParser.java | 3 ++- .../reportingdata/ReportingDataParser.java | 16 +++++++++------- .../core/extradata/reportingdata/StateType.java | 3 ++- .../fr/insee/kraftwerk/core/utils/DateUtils.java | 13 ++----------- .../extradata/CSVReportingDataParserTest.java | 7 +++++++ .../kraftwerk/core/utils/DateUtilsTest.java | 10 ++++++---- .../unit_tests/reportingdata/reportingdata.csv | 4 ++-- 8 files changed, 32 insertions(+), 26 deletions(-) diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java index 18d25344..3eee6111 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java @@ -57,6 +57,8 @@ private Constants() {} public static final String MULTIMODE_DATASET_NAME = "MULTIMODE"; public static final String REPORTING_DATA_GROUP_NAME = "REPORTINGDATA"; public static final String REPORTING_DATA_INTERVIEWER_ID_NULL_PLACEHOLDER = "NON_AFFECTE_"; + public static final String REPORTING_DATA_INPUT_DATE_FORMAT = "dd/MM/yyyy HH:mm:ss"; + public static final String REPORTING_DATA_OUTPUT_DATE_FORMAT = "yyyy-MM-dd-hh-mm-ss"; public static final String END_LINE = "\n"; public static final String OUTPUT_FOLDER_DATETIME_PATTERN = "yyyy_MM_dd_HH_mm_ss"; public static final String ERRORS_FILE_NAME = "errors.txt"; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java index 5da311e8..086f9575 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java @@ -16,6 +16,7 @@ import com.opencsv.CSVReaderBuilder; import com.opencsv.exceptions.CsvValidationException; +import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.NullException; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; @@ -68,7 +69,7 @@ public void parseReportingData(ReportingData reportingData, SurveyRawData data, } public long convertToTimestamp(String rowTimestamp) { - SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); + SimpleDateFormat dateFormat = new SimpleDateFormat(Constants.REPORTING_DATA_INPUT_DATE_FORMAT); dateFormat.setTimeZone(TimeZone.getTimeZone("CET")); Date parsedDate = null; try { diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java index 524b793b..1bdf5387 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java @@ -7,11 +7,11 @@ import fr.insee.bpm.metadata.model.VariableType; import fr.insee.kraftwerk.core.rawdata.QuestionnaireData; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; -import fr.insee.kraftwerk.core.utils.DateUtils; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; import java.sql.Date; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; @@ -19,6 +19,8 @@ public abstract class ReportingDataParser { public static final String DATE_SUFFIX = "_DATE"; + private final SimpleDateFormat reportingDataOutputDateFormat = + new SimpleDateFormat(Constants.REPORTING_DATA_OUTPUT_DATE_FORMAT); Group reportingGroup; private int maxStates = 0; private int maxAttempts = 0; @@ -221,7 +223,7 @@ private void addReportingDataUEToQuestionnaire(SurveyRawData surveyRawData, Repo } if(reportingDataUE.getSurveyValidationDateTimeStamp() != null){ questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME).getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()).putValue(Constants.REPORTING_DATA_SURVEY_VALIDATION_NAME, - DateUtils.formatLongToString(reportingDataUE.getSurveyValidationDateTimeStamp())); + reportingDataOutputDateFormat.format(new java.util.Date(reportingDataUE.getSurveyValidationDateTimeStamp()))); } } @@ -234,12 +236,12 @@ private void addContactAttempts(ReportingDataUE reportingDataUE, QuestionnaireDa questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) .putValue(Constants.OUTCOME_ATTEMPT_SUFFIX_NAME + "_" + k + DATE_SUFFIX, - DateUtils.formatDateToString(reportingDataUE.getContactAttempts().get(k - 1).getDate())); + reportingDataOutputDateFormat.format(reportingDataUE.getContactAttempts().get(k - 1).getDate())); } questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) .putValue(Constants.LAST_ATTEMPT_DATE, - DateUtils.formatDateToString(getLastContactAttempt(reportingDataUE).getDate())); + reportingDataOutputDateFormat.format(getLastContactAttempt(reportingDataUE).getDate())); } private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireData questionnaire) { @@ -251,7 +253,7 @@ private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireDat questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) .putValue(Constants.OUTCOME_DATE, - DateUtils.formatDateToString(new Date(contactOutcome.getDateEndContact()))); + reportingDataOutputDateFormat.format(new Date(contactOutcome.getDateEndContact()))); } questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) @@ -267,8 +269,8 @@ private void addStates(ReportingDataUE reportingDataUE, QuestionnaireData questi StateType.getStateType((reportingDataUE.getStates().get(k - 1)).getStateType())); questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) - .putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX, DateUtils - .formatDateToString(new Date((reportingDataUE.getStates().get(k - 1)).getTimestamp()))); + .putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX, + reportingDataOutputDateFormat.format(new Date((reportingDataUE.getStates().get(k - 1)).getTimestamp()))); } questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java index bbffd981..a1c70119 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java @@ -31,7 +31,8 @@ public enum StateType { STATE22("VALPAP", "Questionnaire validé sur papier"), STATE23("VALINT", "Questionnaire validé sur internet"), STATE24("REFUSAL", "Refus de répondre"), - STATE25("RELANCE", "RELANCE"); + STATE25("RELANCE", "RELANCE"), + STATE26("FOLLOWUP", "RELANCE"); private final String key; private final String value; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java index 65235b60..95bdea3a 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java @@ -1,5 +1,6 @@ package fr.insee.kraftwerk.core.utils; +import fr.insee.kraftwerk.core.Constants; import lombok.extern.log4j.Log4j2; import java.sql.Timestamp; @@ -22,8 +23,7 @@ public static String getCurrentTimeStamp() { return sdf.format(timestamp); } - public static long convertToTimestamp(String rowTimestamp) { - SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); + public static long convertToTimestamp(String rowTimestamp, SimpleDateFormat dateFormat) { dateFormat.setTimeZone(TimeZone.getTimeZone("CET")); Date parsedDate; try { @@ -34,13 +34,4 @@ public static long convertToTimestamp(String rowTimestamp) { } return TimeUnit.MILLISECONDS.toSeconds(parsedDate.getTime()); } - - public static String formatLongToString(long l) { - return formatDateToString(new Date(l)); - } - - public static String formatDateToString(Date date) { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); - return sdf.format(date); - } } diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java index 34c91056..e8cff45e 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java @@ -42,6 +42,13 @@ void parseReportingDataTest() throws NullException { assertEquals("INITLA", reportingDataUE.getStates().get(0).getStateType()); assertEquals("PARTIELINT", reportingDataUE.getStates().get(1).getStateType()); assertEquals("VALINT", reportingDataUE.getStates().get(2).getStateType()); + + // Check the reporting data date is correctly parsed + assertEquals(1644845734, reportingDataUE.getStates().get(0).getTimestamp()); + assertEquals(1644854405, reportingDataUE.getStates().get(1).getTimestamp()); + assertEquals(1644919206, reportingDataUE.getStates().get(2).getTimestamp()); + + // Check the reporting data is correctly translated in the output /* à implémenter ? *//* "Affectée, non visible enquêteur" et "Questionnaire démarré"*/ diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java index 3ef3f063..3a778bf9 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java @@ -4,14 +4,16 @@ import org.junit.jupiter.api.Test; +import java.text.SimpleDateFormat; + class DateUtilsTest { @Test void convertDateTest() { - assertEquals(1645007098, DateUtils.convertToTimestamp("16/02/2022 11:24:58")); - assertEquals(1566544132, DateUtils.convertToTimestamp("23/08/2019 09:08:52")); - assertEquals(1111111111, DateUtils.convertToTimestamp("18/03/2005 02:58:31")); - assertEquals(1, DateUtils.convertToTimestamp("01/01/1970 01:00:01")); + assertEquals(1645007098, DateUtils.convertToTimestamp("16/02/2022 11:24:58", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); + assertEquals(1566544132, DateUtils.convertToTimestamp("23/08/2019 09:08:52", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); + assertEquals(1111111111, DateUtils.convertToTimestamp("18/03/2005 02:58:31", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); + assertEquals(1, DateUtils.convertToTimestamp("01/01/1970 01:00:01", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); } diff --git a/kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv b/kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv index 716d3999..220a9256 100644 --- a/kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv +++ b/kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv @@ -2,5 +2,5 @@ statut,dateInfo,idUe,idContact,nom,prenom,adresse,numeroDeLot INITLA,14/02/2022 14:35:34,L0000005,XXXXXXX,NAME,FirstName,13 rue du Général hulot Insee Grand Est 54042 - Nancy cedex,99 INITLA,14/02/2022 14:35:34,L0000006,XXXXXXX,NAME,FirstName,13 rue du Général hulot Insee Grand Est 54042 - Nancy cedex,99 INITLA,14/02/2022 14:35:34,L0000169,XXXXXXX,NAME,FirstName,13 rue du Général hulot 54042 - Nancy cedex,2 -VALINT,15/02/2022 11:00:06,L0000169,XXXXXXX,NAME,FirstName,1 rue du moulin - Paris,1 -PARTIELINT,14/02/2022 17:00:05,L0000169,XXXXXXX,NAME,FirstName,1 rue du pont - Paris,1 +VALINT,"15/02/2022 11:00:06",L0000169,XXXXXXX,NAME,FirstName,1 rue du moulin - Paris,1 +PARTIELINT,"14/02/2022 17:00:05",L0000169,XXXXXXX,NAME,FirstName,1 rue du pont - Paris,1 From 6b386e46c90db8a9d99bc5616543cb5507fa41fa Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Mon, 23 Sep 2024 15:05:58 +0200 Subject: [PATCH 6/9] Fix wrong time in output CSV reporting data --- .../reportingdata/CSVReportingDataParser.java | 2 +- .../extradata/reportingdata/ReportingDataParser.java | 2 +- .../java/fr/insee/kraftwerk/core/utils/DateUtils.java | 2 +- .../fr/insee/kraftwerk/core/utils/DateUtilsTest.java | 10 ++++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java index 086f9575..353d3557 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java @@ -81,7 +81,7 @@ public long convertToTimestamp(String rowTimestamp) { log.error("Parsing error : the parsed date is null"); return 0L; } - return TimeUnit.MILLISECONDS.toSeconds(parsedDate.getTime()); + return parsedDate.getTime(); } /** diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java index 1bdf5387..18376a4f 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java @@ -10,7 +10,7 @@ import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; -import java.sql.Date; +import java.util.Date; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java index 95bdea3a..ee6840b4 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java @@ -32,6 +32,6 @@ public static long convertToTimestamp(String rowTimestamp, SimpleDateFormat date log.error("Parsing error : {}", e1.getMessage()); return 0L; } - return TimeUnit.MILLISECONDS.toSeconds(parsedDate.getTime()); + return parsedDate.getTime(); } } diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java index 3a778bf9..f2b43891 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java @@ -10,10 +10,12 @@ class DateUtilsTest { @Test void convertDateTest() { - assertEquals(1645007098, DateUtils.convertToTimestamp("16/02/2022 11:24:58", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); - assertEquals(1566544132, DateUtils.convertToTimestamp("23/08/2019 09:08:52", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); - assertEquals(1111111111, DateUtils.convertToTimestamp("18/03/2005 02:58:31", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); - assertEquals(1, DateUtils.convertToTimestamp("01/01/1970 01:00:01", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); + assertEquals(1645007098000L, DateUtils.convertToTimestamp("16/02/2022 11:24:58", new SimpleDateFormat("dd/MM" + + "/yyyy HH:mm:ss"))); + assertEquals(1566544132000L, DateUtils.convertToTimestamp("23/08/2019 09:08:52", new SimpleDateFormat("dd/MM" + + "/yyyy HH:mm:ss"))); + assertEquals(1111111111000L, DateUtils.convertToTimestamp("18/03/2005 02:58:31", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); + assertEquals(1000L, DateUtils.convertToTimestamp("01/01/1970 01:00:01", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"))); } From e981f9fa79137da0250f19bc9163ba7903745143 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Mon, 23 Sep 2024 15:24:55 +0200 Subject: [PATCH 7/9] Update CSVReportingDataParserTest.java --- .../core/extradata/CSVReportingDataParserTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java index e8cff45e..1584c285 100644 --- a/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java +++ b/kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java @@ -44,9 +44,9 @@ void parseReportingDataTest() throws NullException { assertEquals("VALINT", reportingDataUE.getStates().get(2).getStateType()); // Check the reporting data date is correctly parsed - assertEquals(1644845734, reportingDataUE.getStates().get(0).getTimestamp()); - assertEquals(1644854405, reportingDataUE.getStates().get(1).getTimestamp()); - assertEquals(1644919206, reportingDataUE.getStates().get(2).getTimestamp()); + assertEquals(1644845734000L, reportingDataUE.getStates().get(0).getTimestamp()); + assertEquals(1644854405000L, reportingDataUE.getStates().get(1).getTimestamp()); + assertEquals(1644919206000L, reportingDataUE.getStates().get(2).getTimestamp()); // Check the reporting data is correctly translated in the output @@ -69,10 +69,10 @@ void controlHeaderTest() { @Test void convertDateTest() { CSVReportingDataParser csvReportingDataParser = new CSVReportingDataParser(fileUtilsInterface); - assertEquals(1645007098, csvReportingDataParser.convertToTimestamp("16/02/2022 11:24:58")); - assertEquals(1566544132, csvReportingDataParser.convertToTimestamp("23/08/2019 09:08:52")); - assertEquals(1111111111, csvReportingDataParser.convertToTimestamp("18/03/2005 02:58:31")); - assertEquals(1, csvReportingDataParser.convertToTimestamp("01/01/1970 01:00:01")); + assertEquals(1645007098000L, csvReportingDataParser.convertToTimestamp("16/02/2022 11:24:58")); + assertEquals(1566544132000L, csvReportingDataParser.convertToTimestamp("23/08/2019 09:08:52")); + assertEquals(1111111111000L, csvReportingDataParser.convertToTimestamp("18/03/2005 02:58:31")); + assertEquals(1000L, csvReportingDataParser.convertToTimestamp("01/01/1970 01:00:01")); } From 701fc5b193c3bbea8dc058e1ef5ab8ec7c9dc6c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 25 Sep 2024 11:18:59 +0200 Subject: [PATCH 8/9] Switching from java.util.date to java.time API --- .../reportingdata/CSVReportingDataParser.java | 23 +++++++++---------- .../reportingdata/ContactAttempt.java | 7 ++++-- .../reportingdata/ReportingDataParser.java | 16 +++++++------ 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java index 353d3557..1ec5f6ca 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java @@ -5,10 +5,10 @@ import java.io.InputStreamReader; import java.nio.file.Path; import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.TimeZone; -import java.util.concurrent.TimeUnit; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import com.opencsv.CSVParser; import com.opencsv.CSVParserBuilder; @@ -68,20 +68,19 @@ public void parseReportingData(ReportingData reportingData, SurveyRawData data, } } - public long convertToTimestamp(String rowTimestamp) { - SimpleDateFormat dateFormat = new SimpleDateFormat(Constants.REPORTING_DATA_INPUT_DATE_FORMAT); - dateFormat.setTimeZone(TimeZone.getTimeZone("CET")); - Date parsedDate = null; + public long convertToTimestamp(String dateString) { + DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern(Constants.REPORTING_DATA_INPUT_DATE_FORMAT); + LocalDateTime parsedDate = null; try { - parsedDate = dateFormat.parse(rowTimestamp); - } catch (ParseException e1) { - log.error("Parsing error : {}", e1.getMessage()); + parsedDate = LocalDateTime.parse(dateString, dateFormat); + } catch (DateTimeParseException e) { + log.error("Parsing error : {}", e.getMessage()); } if (parsedDate == null) { log.error("Parsing error : the parsed date is null"); return 0L; } - return parsedDate.getTime(); + return parsedDate.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); } /** diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java index e2791196..5672202a 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java @@ -1,5 +1,8 @@ package fr.insee.kraftwerk.core.extradata.reportingdata; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.Date; import lombok.Getter; @@ -22,8 +25,8 @@ public ContactAttempt(String status, long timestamp) { this.timestamp = timestamp; } - public Date getDate() { - return new Date(timestamp); + public LocalDateTime getDate() { + return LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.systemDefault()); } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java index 18376a4f..fcfdb3ac 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java @@ -10,8 +10,10 @@ import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; -import java.util.Date; -import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; @@ -19,8 +21,8 @@ public abstract class ReportingDataParser { public static final String DATE_SUFFIX = "_DATE"; - private final SimpleDateFormat reportingDataOutputDateFormat = - new SimpleDateFormat(Constants.REPORTING_DATA_OUTPUT_DATE_FORMAT); + private final DateTimeFormatter reportingDataOutputDateFormat = + DateTimeFormatter.ofPattern(Constants.REPORTING_DATA_OUTPUT_DATE_FORMAT); Group reportingGroup; private int maxStates = 0; private int maxAttempts = 0; @@ -223,7 +225,7 @@ private void addReportingDataUEToQuestionnaire(SurveyRawData surveyRawData, Repo } if(reportingDataUE.getSurveyValidationDateTimeStamp() != null){ questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME).getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()).putValue(Constants.REPORTING_DATA_SURVEY_VALIDATION_NAME, - reportingDataOutputDateFormat.format(new java.util.Date(reportingDataUE.getSurveyValidationDateTimeStamp()))); + reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getSurveyValidationDateTimeStamp()),ZoneId.systemDefault()))); } } @@ -253,7 +255,7 @@ private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireDat questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) .putValue(Constants.OUTCOME_DATE, - reportingDataOutputDateFormat.format(new Date(contactOutcome.getDateEndContact()))); + reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(contactOutcome.getDateEndContact()),ZoneId.systemDefault()))); } questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) @@ -270,7 +272,7 @@ private void addStates(ReportingDataUE reportingDataUE, QuestionnaireData questi questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) .putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX, - reportingDataOutputDateFormat.format(new Date((reportingDataUE.getStates().get(k - 1)).getTimestamp()))); + reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getStates().get(k - 1).getTimestamp()),ZoneId.systemDefault()))); } questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) From bbb50d6a4349cf234b6d9acc95404e30b66cefc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 25 Sep 2024 11:30:43 +0200 Subject: [PATCH 9/9] Defines time zone to CET --- .../reportingdata/CSVReportingDataParser.java | 22 +++++++++---------- .../reportingdata/ContactAttempt.java | 2 +- .../reportingdata/ReportingDataParser.java | 6 ++--- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java index 1ec5f6ca..4b99cfb2 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java @@ -1,27 +1,25 @@ package fr.insee.kraftwerk.core.extradata.reportingdata; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.file.Path; -import java.text.ParseException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; - import com.opencsv.CSVParser; import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReader; import com.opencsv.CSVReaderBuilder; import com.opencsv.exceptions.CsvValidationException; - import fr.insee.kraftwerk.core.Constants; import fr.insee.kraftwerk.core.exceptions.NullException; import fr.insee.kraftwerk.core.rawdata.SurveyRawData; import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface; import lombok.extern.log4j.Log4j2; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; + @Log4j2 public class CSVReportingDataParser extends ReportingDataParser { @@ -80,7 +78,7 @@ public long convertToTimestamp(String dateString) { log.error("Parsing error : the parsed date is null"); return 0L; } - return parsedDate.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + return parsedDate.atZone(ZoneId.of("CET")).toInstant().toEpochMilli(); } /** diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java index 5672202a..2898b1c9 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ContactAttempt.java @@ -26,7 +26,7 @@ public ContactAttempt(String status, long timestamp) { } public LocalDateTime getDate() { - return LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.systemDefault()); + return LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.of("CET")); } diff --git a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java index fcfdb3ac..d6767660 100644 --- a/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java +++ b/kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java @@ -225,7 +225,7 @@ private void addReportingDataUEToQuestionnaire(SurveyRawData surveyRawData, Repo } if(reportingDataUE.getSurveyValidationDateTimeStamp() != null){ questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME).getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()).putValue(Constants.REPORTING_DATA_SURVEY_VALIDATION_NAME, - reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getSurveyValidationDateTimeStamp()),ZoneId.systemDefault()))); + reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getSurveyValidationDateTimeStamp()),ZoneId.of("CET")))); } } @@ -255,7 +255,7 @@ private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireDat questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) .putValue(Constants.OUTCOME_DATE, - reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(contactOutcome.getDateEndContact()),ZoneId.systemDefault()))); + reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(contactOutcome.getDateEndContact()),ZoneId.of("CET")))); } questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) @@ -272,7 +272,7 @@ private void addStates(ReportingDataUE reportingDataUE, QuestionnaireData questi questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()) .putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX, - reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getStates().get(k - 1).getTimestamp()),ZoneId.systemDefault()))); + reportingDataOutputDateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(reportingDataUE.getStates().get(k - 1).getTimestamp()),ZoneId.of("CET")))); } questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME) .getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())