Skip to content

Commit

Permalink
Fixed wrong xml reporting data FOLLOWUP + csv date format
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexisszmundy
alexisszmundy committed Sep 23, 2024
1 parent a0276c5 commit 919ef6e
Show file tree
Hide file tree
Showing 8 changed files with 32 additions and 26 deletions.
2 changes: 2 additions & 0 deletions kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ private Constants() {}
public static final String MULTIMODE_DATASET_NAME = "MULTIMODE";
public static final String REPORTING_DATA_GROUP_NAME = "REPORTINGDATA";
public static final String REPORTING_DATA_INTERVIEWER_ID_NULL_PLACEHOLDER = "NON_AFFECTE_";
public static final String REPORTING_DATA_INPUT_DATE_FORMAT = "dd/MM/yyyy HH:mm:ss";
public static final String REPORTING_DATA_OUTPUT_DATE_FORMAT = "yyyy-MM-dd-hh-mm-ss";
public static final String END_LINE = "\n";
public static final String OUTPUT_FOLDER_DATETIME_PATTERN = "yyyy_MM_dd_HH_mm_ss";
public static final String ERRORS_FILE_NAME = "errors.txt";
Expand Down
3 changes: 2 additions & 1 deletion ...src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/CSVReportingDataParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvValidationException;

import fr.insee.kraftwerk.core.Constants;
import fr.insee.kraftwerk.core.exceptions.NullException;
import fr.insee.kraftwerk.core.rawdata.SurveyRawData;
import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface;
Expand Down Expand Up @@ -68,7 +69,7 @@ public void parseReportingData(ReportingData reportingData, SurveyRawData data,
}

public long convertToTimestamp(String rowTimestamp) {
SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
SimpleDateFormat dateFormat = new SimpleDateFormat(Constants.REPORTING_DATA_INPUT_DATE_FORMAT);
dateFormat.setTimeZone(TimeZone.getTimeZone("CET"));
Date parsedDate = null;
try {
Expand Down
16 changes: 9 additions & 7 deletions ...re/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/ReportingDataParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,20 @@
import fr.insee.bpm.metadata.model.VariableType;
import fr.insee.kraftwerk.core.rawdata.QuestionnaireData;
import fr.insee.kraftwerk.core.rawdata.SurveyRawData;
import fr.insee.kraftwerk.core.utils.DateUtils;
import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface;
import lombok.extern.log4j.Log4j2;

import java.sql.Date;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;

@Log4j2
public abstract class ReportingDataParser {

public static final String DATE_SUFFIX = "_DATE";
private final SimpleDateFormat reportingDataOutputDateFormat =
new SimpleDateFormat(Constants.REPORTING_DATA_OUTPUT_DATE_FORMAT);
Group reportingGroup;
private int maxStates = 0;
private int maxAttempts = 0;
Expand Down Expand Up @@ -221,7 +223,7 @@ private void addReportingDataUEToQuestionnaire(SurveyRawData surveyRawData, Repo
}
if(reportingDataUE.getSurveyValidationDateTimeStamp() != null){
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME).getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier()).putValue(Constants.REPORTING_DATA_SURVEY_VALIDATION_NAME,
DateUtils.formatLongToString(reportingDataUE.getSurveyValidationDateTimeStamp()));
reportingDataOutputDateFormat.format(new java.util.Date(reportingDataUE.getSurveyValidationDateTimeStamp())));
}
}

Expand All @@ -234,12 +236,12 @@ private void addContactAttempts(ReportingDataUE reportingDataUE, QuestionnaireDa
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
.putValue(Constants.OUTCOME_ATTEMPT_SUFFIX_NAME + "_" + k + DATE_SUFFIX,
DateUtils.formatDateToString(reportingDataUE.getContactAttempts().get(k - 1).getDate()));
reportingDataOutputDateFormat.format(reportingDataUE.getContactAttempts().get(k - 1).getDate()));
}
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
.putValue(Constants.LAST_ATTEMPT_DATE,
DateUtils.formatDateToString(getLastContactAttempt(reportingDataUE).getDate()));
reportingDataOutputDateFormat.format(getLastContactAttempt(reportingDataUE).getDate()));
}

private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireData questionnaire) {
Expand All @@ -251,7 +253,7 @@ private void addContactOutcome(ReportingDataUE reportingDataUE, QuestionnaireDat
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
.putValue(Constants.OUTCOME_DATE,
DateUtils.formatDateToString(new Date(contactOutcome.getDateEndContact())));
reportingDataOutputDateFormat.format(new Date(contactOutcome.getDateEndContact())));
}
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
Expand All @@ -267,8 +269,8 @@ private void addStates(ReportingDataUE reportingDataUE, QuestionnaireData questi
StateType.getStateType((reportingDataUE.getStates().get(k - 1)).getStateType()));
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
.putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX, DateUtils
.formatDateToString(new Date((reportingDataUE.getStates().get(k - 1)).getTimestamp())));
.putValue(Constants.STATE_SUFFIX_NAME + "_" + k + DATE_SUFFIX,
reportingDataOutputDateFormat.format(new Date((reportingDataUE.getStates().get(k - 1)).getTimestamp())));
}
questionnaire.getAnswers().getSubGroup(Constants.REPORTING_DATA_GROUP_NAME)
.getInstance(Constants.REPORTING_DATA_PREFIX_NAME + reportingDataUE.getIdentifier())
Expand Down
3 changes: 2 additions & 1 deletion kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/StateType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ public enum StateType {
STATE22("VALPAP", "Questionnaire validé sur papier"),
STATE23("VALINT", "Questionnaire validé sur internet"),
STATE24("REFUSAL", "Refus de répondre"),
STATE25("RELANCE", "RELANCE");
STATE25("RELANCE", "RELANCE"),
STATE26("FOLLOWUP", "RELANCE");

private final String key;
private final String value;
Expand Down
13 changes: 2 additions & 11 deletions kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/utils/DateUtils.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package fr.insee.kraftwerk.core.utils;

import fr.insee.kraftwerk.core.Constants;
import lombok.extern.log4j.Log4j2;

import java.sql.Timestamp;
Expand All @@ -22,8 +23,7 @@ public static String getCurrentTimeStamp() {
return sdf.format(timestamp);
}

public static long convertToTimestamp(String rowTimestamp) {
SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
public static long convertToTimestamp(String rowTimestamp, SimpleDateFormat dateFormat) {
dateFormat.setTimeZone(TimeZone.getTimeZone("CET"));
Date parsedDate;
try {
Expand All @@ -34,13 +34,4 @@ public static long convertToTimestamp(String rowTimestamp) {
}
return TimeUnit.MILLISECONDS.toSeconds(parsedDate.getTime());
}

public static String formatLongToString(long l) {
return formatDateToString(new Date(l));
}

public static String formatDateToString(Date date) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
return sdf.format(date);
}
}
7 changes: 7 additions & 0 deletions ...werk-core/src/test/java/fr/insee/kraftwerk/core/extradata/CSVReportingDataParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,13 @@ void parseReportingDataTest() throws NullException {
assertEquals("INITLA", reportingDataUE.getStates().get(0).getStateType());
assertEquals("PARTIELINT", reportingDataUE.getStates().get(1).getStateType());
assertEquals("VALINT", reportingDataUE.getStates().get(2).getStateType());

// Check the reporting data date is correctly parsed
assertEquals(1644845734, reportingDataUE.getStates().get(0).getTimestamp());
assertEquals(1644854405, reportingDataUE.getStates().get(1).getTimestamp());
assertEquals(1644919206, reportingDataUE.getStates().get(2).getTimestamp());


// Check the reporting data is correctly translated in the output
/* à implémenter ? *//*
"Affectée, non visible enquêteur" et "Questionnaire démarré"*/
Expand Down
10 changes: 6 additions & 4 deletions kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/utils/DateUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,16 @@

import org.junit.jupiter.api.Test;

import java.text.SimpleDateFormat;

class DateUtilsTest {

@Test
void convertDateTest() {
assertEquals(1645007098, DateUtils.convertToTimestamp("16/02/2022 11:24:58"));
assertEquals(1566544132, DateUtils.convertToTimestamp("23/08/2019 09:08:52"));
assertEquals(1111111111, DateUtils.convertToTimestamp("18/03/2005 02:58:31"));
assertEquals(1, DateUtils.convertToTimestamp("01/01/1970 01:00:01"));
assertEquals(1645007098, DateUtils.convertToTimestamp("16/02/2022 11:24:58", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss")));
assertEquals(1566544132, DateUtils.convertToTimestamp("23/08/2019 09:08:52", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss")));
assertEquals(1111111111, DateUtils.convertToTimestamp("18/03/2005 02:58:31", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss")));
assertEquals(1, DateUtils.convertToTimestamp("01/01/1970 01:00:01", new SimpleDateFormat("dd/MM/yyyy HH:mm:ss")));

}

Expand Down
4 changes: 2 additions & 2 deletions kraftwerk-core/src/test/resources/unit_tests/reportingdata/reportingdata.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ statut,dateInfo,idUe,idContact,nom,prenom,adresse,numeroDeLot
INITLA,14/02/2022 14:35:34,L0000005,XXXXXXX,NAME,FirstName,13 rue du Général hulot Insee Grand Est 54042 - Nancy cedex,99
INITLA,14/02/2022 14:35:34,L0000006,XXXXXXX,NAME,FirstName,13 rue du Général hulot Insee Grand Est 54042 - Nancy cedex,99
INITLA,14/02/2022 14:35:34,L0000169,XXXXXXX,NAME,FirstName,13 rue du Général hulot 54042 - Nancy cedex,2
VALINT,15/02/2022 11:00:06,L0000169,XXXXXXX,NAME,FirstName,1 rue du moulin - Paris,1
PARTIELINT,14/02/2022 17:00:05,L0000169,XXXXXXX,NAME,FirstName,1 rue du pont - Paris,1
VALINT,"15/02/2022 11:00:06",L0000169,XXXXXXX,NAME,FirstName,1 rue du moulin - Paris,1
PARTIELINT,"14/02/2022 17:00:05",L0000169,XXXXXXX,NAME,FirstName,1 rue du pont - Paris,1

0 comments on commit 919ef6e

Please sign in to comment.