Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev cast #51

Merged
merged 2 commits into from
Nov 30, 2023
Merged

Dev cast #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8735f3f
Exclude unused dependency
alicela Nov 30, 2023
7dfafce
Refactor for cast Highlight
alicela Nov 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions kraftwerk-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down
6 changes: 3 additions & 3 deletions kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParaDataUE.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,12 @@ public long createLengthSessionsVariable() {
}

public String getVariableStart() {
if (getSessions().size()==0) return "0";
if (getSessions().size()==0) { return "0";}
return Long.toString(getSessions().get(0).getInitialization());
}

public String getVariableEnd() {
if (getSessions().size()==0) return "0";
if (getSessions().size()==0) { return "0";}
return Long.toString(getLastSession().getTermination());
}

Expand All @@ -89,7 +89,7 @@ public void createOrchestratorsAndSessions() {
Session session = new Session(INITIALIZATION_ONGOING);
Orchestrator orchestrator = new Orchestrator(identifier);

if (checkNoSessionToCreate(listParadataEvents)) return;
if (checkNoSessionToCreate(listParadataEvents)) {return;}
Event previousEvent = null ;

// iterate on paradata events
Expand Down
98 changes: 55 additions & 43 deletions kraftwerk-core/src/main/java/fr/insee/kraftwerk/core/extradata/paradata/ParadataParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,16 @@ public class ParadataParser {
private String timestamp = "timestamp";

private static final String NEW_VALUE = "newValue";

private List<String> inputFields = Arrays.asList("RADIO", "CHECKBOX", "INPUT", "DATEPICKER");

private List<String> inputFields = Arrays.asList("RADIO", "CHECKBOX", "INPUT", "DATEPICKER");

public void parseParadata(Paradata paradata, SurveyRawData surveyRawData) throws NullException {

log.info("Paradata parser being implemented for Survey Unit : {} !",
surveyRawData.getIdSurveyUnits().toString());
Path filePath = paradata.getFilepath();
if (filePath == null) throw new NullException("JSONFile not defined");
if (filePath == null)
throw new NullException("JSONFile not defined");

if (!filePath.toString().contentEquals("")) {

Expand Down Expand Up @@ -95,20 +95,21 @@ private void parseParadataUE(ParaDataUE paradataUE, SurveyRawData surveyRawData)
paradataUE.setEvents(events);
}

private void parseEventFromParadataUE(ParaDataUE paradataUE, VariablesMap variablesMap, String identifier, ArrayList<Event> events,
JSONArray subParadata, int j) {
private void parseEventFromParadataUE(ParaDataUE paradataUE, VariablesMap variablesMap, String identifier,
ArrayList<Event> events, JSONArray subParadata, int j) {
JSONObject collectedEvent = (JSONObject) subParadata.get(j);
if (isCollectedParadata(collectedEvent)) { //check that paradata are linked to collect (not vizualisation or readonly))
if (isCollectedParadata(collectedEvent)) { // check that paradata are linked to collect (not vizualisation or
// readonly))

Event event = new Event(identifier);
event.setIdParadataObject((String) collectedEvent.get("idParadataObject"));
event.setIdSession((String) collectedEvent.get("idSession"));
event.setTimestamp((long) collectedEvent.get(timestamp));

ParadataVariable paradataVariable = new ParadataVariable(identifier);
paradataVariable.setTimestamp((long) collectedEvent.get(timestamp));
paradataVariable.setValue(collectedEvent.get(NEW_VALUE));

if (variablesMap.getVariableNames().contains(event.getIdParadataObject())) {
paradataVariable.setVariableName(event.getIdParadataObject().toUpperCase());
// Change value -> not String dependant
Expand All @@ -121,7 +122,7 @@ private void parseEventFromParadataUE(ParaDataUE paradataUE, VariablesMap variab
if (inputFields.stream().anyMatch(event.getIdParadataObject().toUpperCase()::contains)) {
paradataVariable.setVariableName((String) collectedEvent.get("responseName"));
paradataUE.addParadataVariable(paradataVariable);
}
}
if (event.getIdParadataObject().toUpperCase().contains(Constants.FILTER_RESULT_PREFIX)) {
paradataVariable.setVariableName(event.getIdParadataObject());
paradataUE.addParadataVariable(paradataVariable);
Expand Down Expand Up @@ -178,12 +179,14 @@ private void integrateParaDataVariablesIntoUE(ParaDataUE paraDataUE, SurveyRawDa
Set<String> paradataVariables = paraDataUE.getParaDataVariables().keySet();
Variable variableDuree = new Variable(Constants.LENGTH_ORCHESTRATORS_NAME, variablesMap.getRootGroup(),
VariableType.STRING, "30");
Variable variableDureeBrute = new Variable(Constants.LENGTH_ORCHESTRATORS_NAME + Constants.PARADATA_TIMESTAMP_SUFFIX,
variablesMap.getRootGroup(), VariableType.INTEGER, "20.");
Variable variableDureeBrute = new Variable(
Constants.LENGTH_ORCHESTRATORS_NAME + Constants.PARADATA_TIMESTAMP_SUFFIX, variablesMap.getRootGroup(),
VariableType.INTEGER, "20.");
Variable variableDureeSession = new Variable(Constants.LENGTH_SESSIONS_NAME, variablesMap.getRootGroup(),
VariableType.STRING, "30");
Variable variableDureeSessionBrute = new Variable(Constants.LENGTH_SESSIONS_NAME + Constants.PARADATA_TIMESTAMP_SUFFIX,
variablesMap.getRootGroup(), VariableType.INTEGER, "20.");
Variable variableDureeSessionBrute = new Variable(
Constants.LENGTH_SESSIONS_NAME + Constants.PARADATA_TIMESTAMP_SUFFIX, variablesMap.getRootGroup(),
VariableType.INTEGER, "20.");
Variable variableStart = new Variable(Constants.START_SESSION_NAME, variablesMap.getRootGroup(),
VariableType.INTEGER, "20.");
Variable variableEnd = new Variable(Constants.FINISH_SESSION_NAME, variablesMap.getRootGroup(),
Expand All @@ -192,12 +195,14 @@ private void integrateParaDataVariablesIntoUE(ParaDataUE paraDataUE, SurveyRawDa
VariableType.INTEGER, "3.");
Variable variableNbSessions = new Variable(Constants.NUMBER_SESSIONS_NAME, variablesMap.getRootGroup(),
VariableType.INTEGER, "3.");
Variable variableDateCollecteBrute = new Variable(Constants.SURVEY_VALIDATION_DATE_NAME + Constants.PARADATA_TIMESTAMP_SUFFIX, variablesMap.getRootGroup(),
VariableType.STRING, "3.");
Variable variableDateCollecteBrute = new Variable(
Constants.SURVEY_VALIDATION_DATE_NAME + Constants.PARADATA_TIMESTAMP_SUFFIX,
variablesMap.getRootGroup(), VariableType.STRING, "3.");
Variable variableDateCollecte = new Variable(Constants.SURVEY_VALIDATION_DATE_NAME, variablesMap.getRootGroup(),
VariableType.STRING, "3.");

// Add variables to map : some variables are calculated but not used (variableDuree,variableDureeBrute, variableNbOrch)

// Add variables to map : some variables are calculated but not used
// (variableDuree,variableDureeBrute, variableNbOrch)
variablesMap.putVariable(variableDureeSession);
variablesMap.putVariable(variableDureeSessionBrute);
variablesMap.putVariable(variableNbSessions);
Expand All @@ -213,34 +218,41 @@ private void integrateParaDataVariablesIntoUE(ParaDataUE paraDataUE, SurveyRawDa
}
}

if (!paraDataUE.getOrchestrators().isEmpty()) {
long lengthOrchestrators = paraDataUE.createLengthOrchestratorsVariable();
long lengthSessions = paraDataUE.createLengthSessionsVariable();

QuestionnaireData questionnaire = surveyRawData.getQuestionnaires().stream()
.filter(questionnaireToSearch -> paraDataUE.getOrchestrators().get(0).getIdentifier()
.equals(questionnaireToSearch.getIdentifier()))
.findAny().orElse(null);
if (questionnaire != null) {
questionnaire.getAnswers().putValue(variableDuree.getName(),Constants.convertToDateFormat(lengthOrchestrators));
questionnaire.getAnswers().putValue(variableDureeBrute.getName(), Long.toString(lengthOrchestrators));
questionnaire.getAnswers().putValue(variableDureeSession.getName(),Constants.convertToDateFormat(lengthSessions));
questionnaire.getAnswers().putValue(variableDureeSessionBrute.getName(), Long.toString(lengthSessions));
questionnaire.getAnswers().putValue(variableStart.getName(), paraDataUE.getVariableStart());
questionnaire.getAnswers().putValue(variableEnd.getName(), paraDataUE.getVariableEnd());
questionnaire.getAnswers().putValue(variableNbOrch.getName(), Long.toString(paraDataUE.getOrchestrators().size()));
questionnaire.getAnswers().putValue(variableNbSessions.getName(), Long.toString(paraDataUE.getSessions().size()));
questionnaire.getAnswers().putValue(variableDateCollecte.getName(),
LocalDateTime.ofInstant(Instant.ofEpochMilli(paraDataUE.getSurveyValidationDateTimeStamp()),TimeZone.getDefault().toZoneId()).toString());
questionnaire.getAnswers().putValue(variableDateCollecteBrute.getName(), Long.toString(paraDataUE.getSurveyValidationDateTimeStamp()));
for (String variableName : paradataVariables) {
if (variableName.contentEquals("PRENOM")) {
questionnaire.getAnswers().putValue(Constants.PARADATA_VARIABLES_PREFIX + variableName,
String.valueOf(paraDataUE.getParadataVariable(variableName).size()));
}
}
if (paraDataUE.getOrchestrators().isEmpty()) return;

long lengthOrchestrators = paraDataUE.createLengthOrchestratorsVariable();
long lengthSessions = paraDataUE.createLengthSessionsVariable();

QuestionnaireData questionnaire = surveyRawData.getQuestionnaires().stream()
.filter(questionnaireToSearch -> paraDataUE.getOrchestrators().get(0).getIdentifier()
.equals(questionnaireToSearch.getIdentifier()))
.findAny().orElse(null);

if (questionnaire == null) return;

questionnaire.getAnswers().putValue(variableDuree.getName(),Constants.convertToDateFormat(lengthOrchestrators));
questionnaire.getAnswers().putValue(variableDureeBrute.getName(), Long.toString(lengthOrchestrators));
questionnaire.getAnswers().putValue(variableDureeSession.getName(),
Constants.convertToDateFormat(lengthSessions));
questionnaire.getAnswers().putValue(variableDureeSessionBrute.getName(), Long.toString(lengthSessions));
questionnaire.getAnswers().putValue(variableStart.getName(), paraDataUE.getVariableStart());
questionnaire.getAnswers().putValue(variableEnd.getName(), paraDataUE.getVariableEnd());
questionnaire.getAnswers().putValue(variableNbOrch.getName(),
Long.toString(paraDataUE.getOrchestrators().size()));
questionnaire.getAnswers().putValue(variableNbSessions.getName(),
Long.toString(paraDataUE.getSessions().size()));
questionnaire.getAnswers().putValue(variableDateCollecte.getName(),
LocalDateTime.ofInstant(Instant.ofEpochMilli(paraDataUE.getSurveyValidationDateTimeStamp()),
TimeZone.getDefault().toZoneId()).toString());
questionnaire.getAnswers().putValue(variableDateCollecteBrute.getName(),
Long.toString(paraDataUE.getSurveyValidationDateTimeStamp()));
for (String variableName : paradataVariables) {
if (variableName.contentEquals("PRENOM")) {
questionnaire.getAnswers().putValue(Constants.PARADATA_VARIABLES_PREFIX + variableName,
String.valueOf(paraDataUE.getParadataVariable(variableName).size()));
}
}

}

}
18 changes: 12 additions & 6 deletions ...src/main/java/fr/insee/kraftwerk/core/extradata/reportingdata/XMLReportingDataParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,28 +75,34 @@ public void parseReportingData(ReportingData reportingData, SurveyRawData data,
}
Elements contactAttemptsElements = surveyUnitElement.getFirstChildElement("ContactAttempts")
.getChildElements("ContactAttempt");
if (contactAttemptsElements != null)
if (contactAttemptsElements != null) {
for (int k = 0; k < contactAttemptsElements.size(); k++) {
Element contactAttemptsElement = contactAttemptsElements.get(k);
String status = contactAttemptsElement.getFirstChildElement("status").getValue().toUpperCase();
String timestamp = contactAttemptsElement.getFirstChildElement("date").getValue();
reportingDataUE.addContactAttempts(new ContactAttempt(status, Long.parseLong(timestamp)));
}
}

//Get identification
Element identificationElement = surveyUnitElement.getFirstChildElement("Identification");
reportingDataUE.setIdentification(new Identification());
if (identificationElement != null) {
if (identificationElement.getFirstChildElement(Constants.IDENTIFICATION_NAME) != null)
if (identificationElement.getFirstChildElement(Constants.IDENTIFICATION_NAME) != null) {
reportingDataUE.getIdentification().setIdentification(identificationElement.getFirstChildElement(Constants.IDENTIFICATION_NAME).getValue());
if (identificationElement.getFirstChildElement(Constants.ACCESS_NAME) != null)
}
if (identificationElement.getFirstChildElement(Constants.ACCESS_NAME) != null) {
reportingDataUE.getIdentification().setAccess(identificationElement.getFirstChildElement(Constants.ACCESS_NAME).getValue());
if (identificationElement.getFirstChildElement(Constants.SITUATION_NAME) != null)
}
if (identificationElement.getFirstChildElement(Constants.SITUATION_NAME) != null) {
reportingDataUE.getIdentification().setSituation(identificationElement.getFirstChildElement(Constants.SITUATION_NAME).getValue());
if (identificationElement.getFirstChildElement(Constants.CATEGORY_NAME) != null)
}
if (identificationElement.getFirstChildElement(Constants.CATEGORY_NAME) != null) {
reportingDataUE.getIdentification().setCategory(identificationElement.getFirstChildElement(Constants.CATEGORY_NAME).getValue());
if (identificationElement.getFirstChildElement(Constants.OCCUPANT_NAME) != null)
}
if (identificationElement.getFirstChildElement(Constants.OCCUPANT_NAME) != null) {
reportingDataUE.getIdentification().setOccupant(identificationElement.getFirstChildElement(Constants.OCCUPANT_NAME).getValue());
}
}
reportingData.addReportingDataUE(reportingDataUE);
}
Expand Down
2 changes: 1 addition & 1 deletion kraftwerk-core/src/test/java/fr/insee/kraftwerk/core/extradata/ParaDataParserTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ void whenCorrectParadata_thenCalculatedTimeIsOk() throws NullException {

private SurveyRawData addIdToTest(SurveyRawData srd, String idToAdd) {
List<String> ids = srd.getIdSurveyUnits();
if (ids == null) ids = new ArrayList<>();
if (ids == null) { ids = new ArrayList<>();}
ids.add(idToAdd);
srd.setIdSurveyUnits(ids);

Expand Down
61 changes: 30 additions & 31 deletions ...rk-functional-tests/src/test/java/cucumber/functional_tests/ReportingDataDefinitions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
package cucumber.functional_tests;

import com.opencsv.CSVReader;
import com.opencsv.exceptions.CsvException;
import com.opencsv.exceptions.CsvValidationException;
import fr.insee.kraftwerk.core.Constants;
import fr.insee.kraftwerk.core.extradata.reportingdata.ContactAttemptType;
import fr.insee.kraftwerk.core.extradata.reportingdata.StateType;
import fr.insee.kraftwerk.core.utils.CsvUtils;
import io.cucumber.java.en.Then;
import static cucumber.TestConstants.FUNCTIONAL_TESTS_INPUT_DIRECTORY;
import static cucumber.TestConstants.FUNCTIONAL_TESTS_OUTPUT_DIRECTORY;
import static cucumber.TestConstants.FUNCTIONAL_TESTS_TEMP_DIRECTORY;
import static fr.insee.kraftwerk.core.Constants.OUTCOME_ATTEMPT_SUFFIX_NAME;
import static fr.insee.kraftwerk.core.Constants.STATE_SUFFIX_NAME;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;

import java.io.File;
import java.io.IOException;
Expand All @@ -18,13 +17,15 @@
import java.util.Arrays;
import java.util.List;

import static cucumber.TestConstants.FUNCTIONAL_TESTS_INPUT_DIRECTORY;
import static cucumber.TestConstants.FUNCTIONAL_TESTS_OUTPUT_DIRECTORY;
import static cucumber.TestConstants.FUNCTIONAL_TESTS_TEMP_DIRECTORY;
import static fr.insee.kraftwerk.core.Constants.OUTCOME_ATTEMPT_SUFFIX_NAME;
import static fr.insee.kraftwerk.core.Constants.STATE_SUFFIX_NAME;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;
import com.opencsv.CSVReader;
import com.opencsv.exceptions.CsvException;
import com.opencsv.exceptions.CsvValidationException;

import fr.insee.kraftwerk.core.Constants;
import fr.insee.kraftwerk.core.extradata.reportingdata.ContactAttemptType;
import fr.insee.kraftwerk.core.extradata.reportingdata.StateType;
import fr.insee.kraftwerk.core.utils.CsvUtils;
import io.cucumber.java.en.Then;


// These definitions are used in do_we_export_contact_attempts feature
Expand Down Expand Up @@ -214,7 +215,7 @@ public void check_contact_attempt_date_format(String fileName, String directory,
//Date check
SimpleDateFormat sdf = new SimpleDateFormat(expectedDateFormat);
for (String[] line : content) {
int i = 0;
int i = 0;
// For each field
for (String element : line) {
String fieldName = header[i];
Expand Down Expand Up @@ -263,13 +264,7 @@ public void check_outcome_spotting_result(String surveyUnitId, String fileName,
assertThat(header).contains("OUTCOME_SPOTTING");

// Fetch concerned survey unit line from file
String[] concernedLine = null;
for (String[] line : content) {
if (line[0].equals(surveyUnitId)) {
concernedLine = line;
break;
}
}
String[] concernedLine = fetchConcernedSurveyUnitLineFromFile(surveyUnitId, content);

// Survey unit existence assertion
assertThat(concernedLine).isNotNull();
Expand All @@ -287,6 +282,17 @@ public void check_outcome_spotting_result(String surveyUnitId, String fileName,
assertThat(outcomeSpottingContent).isEqualTo(expectedOutcomeSpotting);
}

public String[] fetchConcernedSurveyUnitLineFromFile(String surveyUnitId, List<String[]> content) {
String[] concernedLine = null;
for (String[] line : content) {
if (line[0].equals(surveyUnitId)) {
concernedLine = line;
break;
}
}
return concernedLine;
}

@Then("For SurveyUnit {string} in a file named {string} in directory {string} we should have {string} in the identification field")
public void check_identification(String surveyUnitId, String fileName, String directory, String expectedValue) throws IOException, CsvException {
CSVReader csvReader = CsvUtils.getReader(
Expand All @@ -303,14 +309,7 @@ public void check_identification(String surveyUnitId, String fileName, String di
// OUTCOME_SPOTTING field existence assertion
assertThat(header).contains(Constants.IDENTIFICATION_NAME);

// Fetch concerned survey unit line from file
String[] concernedLine = null;
for (String[] line : content) {
if (line[0].equals(surveyUnitId)) {
concernedLine = line;
break;
}
}
String[] concernedLine = fetchConcernedSurveyUnitLineFromFile(surveyUnitId, content);

// Survey unit existence assertion
assertThat(concernedLine).isNotNull();
Expand Down
Loading