Skip to content

Commit

Permalink
#459 | Validate Entity UUIDS to avoid failures without error info dur…
Browse files Browse the repository at this point in the history 
…ing LongitudinalExportV2 job processing
  • Loading branch information
@himeshr
himeshr committed Jan 5, 2023
1 parent 87c6563 commit 476643f
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 37 deletions.
3 changes: 1 addition & 2 deletions avni-server-api/src/main/java/org/avni/messaging/service/MessagingService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,7 @@ public MessageRequest sendMessage(MessageRequest messageRequest) {
logger.debug(String.format("Sent message for %d", messageRequest.getId()));
} catch (PhoneNumberNotAvailableException p) {
logger.warn("Phone number not available for receiver: " + messageRequest.getMessageReceiver().getReceiverId());
}
catch (Exception e) {
} catch (Exception e) {
logger.error("Could not send message for message request id: " + messageRequest.getId(), e);
bugsnag.notify(e);
}
Expand Down
47 changes: 26 additions & 21 deletions avni-server-api/src/main/java/org/avni/server/exporter/v2/ExportV2CSVFieldExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class ExportV2CSVFieldExtractor implements FieldExtractor<ItemRow>, FlatF

private static final String selectedAnswerFieldValue = "1";
private static final String unSelectedAnswerFieldValue = "0";
public static final String EMPTY_STRING = "";
private static final String EMPTY_STRING = "";
private final ExportJobParametersRepository exportJobParametersRepository;
private final ObjectMapper objectMapper;
private final EncounterRepository encounterRepository;
Expand Down Expand Up @@ -98,34 +98,43 @@ public void init() {
String timezone = exportJobParameters.getTimezone();
exportOutput = objectMapper.convertValue(exportJobParameters.getReportFormat(), new TypeReference<ExportOutput>() {
});
initializeDataMaps(timezone);
}

private void initializeDataMaps(String timezone) {
String subjectTypeUUID = exportOutput.getUuid();
SubjectType individualType = subjectTypeRepository.findByUuid(subjectTypeUUID);
this.registrationMap = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(subjectTypeUUID, null, null, FormType.IndividualProfile), exportOutput);
this.headers.append(headerCreator.addRegistrationHeaders(subjectTypeRepository.findByUuid(subjectTypeUUID), this.registrationMap, this.addressLevelTypes, exportOutput.getFields()));
this.headers.append(headerCreator.addRegistrationHeaders(individualType, this.registrationMap, this.addressLevelTypes, exportOutput.getFields()));
exportOutput.getEncounters().forEach(e -> populateGeneralEncounterMap(subjectTypeUUID, e, e.getUuid(), this.encounterMap, this.encounterCancelMap, timezone));
exportOutput.getGroups().forEach(g -> {
String groupSubjectTypeUUID = g.getUuid();
SubjectType groupSubjectType = subjectTypeRepository.findByUuid(groupSubjectTypeUUID);
LinkedHashMap<String, FormElement> applicableGroupsFields = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(groupSubjectTypeUUID, null, null, FormType.IndividualProfile), g);
this.groupsMap.put(g.getUuid(), applicableGroupsFields);
this.headers.append(",")
.append(headerCreator.addRegistrationHeaders(subjectTypeRepository.findByUuid(groupSubjectTypeUUID), applicableGroupsFields, this.addressLevelTypes, g.getFields()));
.append(headerCreator.addRegistrationHeaders(groupSubjectType, applicableGroupsFields, this.addressLevelTypes, g.getFields()));
g.getEncounters().forEach(ge -> populateGeneralEncounterMap(groupSubjectTypeUUID, ge, ge.getUuid(), this.groupEncounterMap, this.groupEncounterCancelMap, timezone));
});
exportOutput.getPrograms().forEach(p -> {
LinkedHashMap<String, FormElement> applicableEnrolmentFields = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(subjectTypeUUID, p.getUuid(), null, FormType.ProgramEnrolment), p);
LinkedHashMap<String, FormElement> applicableExitFields = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(subjectTypeUUID, p.getUuid(), null, FormType.ProgramExit), p);
this.enrolmentMap.put(p.getUuid(), applicableEnrolmentFields);
this.exitEnrolmentMap.put(p.getUuid(), applicableExitFields);
String programUUID = p.getUuid();
Program program = programRepository.findByUuid(programUUID);
LinkedHashMap<String, FormElement> applicableEnrolmentFields = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(subjectTypeUUID, programUUID, null, FormType.ProgramEnrolment), p);
LinkedHashMap<String, FormElement> applicableExitFields = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(subjectTypeUUID, programUUID, null, FormType.ProgramExit), p);
this.enrolmentMap.put(programUUID, applicableEnrolmentFields);
this.exitEnrolmentMap.put(programUUID, applicableExitFields);
this.headers.append(",")
.append(headerCreator.addEnrolmentHeaders(applicableEnrolmentFields, applicableExitFields, programRepository.findByUuid(p.getUuid()).getName(), p.getFields()));
.append(headerCreator.addEnrolmentHeaders(applicableEnrolmentFields, applicableExitFields, programRepository.findByUuid(programUUID).getName(), p.getFields()));
p.getEncounters().forEach(pe -> {
LinkedHashMap<String, FormElement> applicableProgramEncounterFields = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(subjectTypeUUID, p.getUuid(), pe.getUuid(), FormType.ProgramEncounter), pe);
LinkedHashMap<String, FormElement> applicableProgramCancelFields = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(subjectTypeUUID, p.getUuid(), pe.getUuid(), FormType.ProgramEncounterCancellation), pe);
this.programEncounterMap.put(pe.getUuid(), applicableProgramEncounterFields);
this.programEncounterCancelMap.put(pe.getUuid(), applicableProgramCancelFields);
String programEncounterTypeUUID = pe.getUuid();
EncounterType encounterType = encounterTypeRepository.findByUuid(programEncounterTypeUUID);
LinkedHashMap<String, FormElement> applicableProgramEncounterFields = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(subjectTypeUUID, programUUID, programEncounterTypeUUID, FormType.ProgramEncounter), pe);
LinkedHashMap<String, FormElement> applicableProgramCancelFields = getApplicableFields(formMappingService.getAllFormElementsAndDecisionMap(subjectTypeUUID, programUUID, programEncounterTypeUUID, FormType.ProgramEncounterCancellation), pe);
this.programEncounterMap.put(programEncounterTypeUUID, applicableProgramEncounterFields);
this.programEncounterCancelMap.put(programEncounterTypeUUID, applicableProgramCancelFields);
ExportFilters.DateFilter dateFilter = pe.getFilters().getDate();
Long maxProgramEncounterCount = programEncounterRepository.getMaxProgramEncounterCount(pe.getUuid(), getCalendarTime(dateFilter.getFrom(), timezone), getCalendarTime(dateFilter.getTo(), timezone));
Long maxProgramEncounterCount = programEncounterRepository.getMaxProgramEncounterCount(programEncounterTypeUUID, getCalendarTime(dateFilter.getFrom(), timezone), getCalendarTime(dateFilter.getTo(), timezone));
maxProgramEncounterCount = maxProgramEncounterCount == null ? 1l : maxProgramEncounterCount;
EncounterType encounterType = encounterTypeRepository.findByUuid(pe.getUuid());
pe.setMaxCount(maxProgramEncounterCount);
this.headers.append(",")
.append(headerCreator.addEncounterHeaders(maxProgramEncounterCount, applicableProgramEncounterFields, applicableProgramCancelFields, encounterType.getName(), pe.getFields()));
Expand Down Expand Up @@ -170,10 +179,6 @@ public ExportOutput getExportOutput() {
return exportOutput;
}

public void setExportOutput(ExportOutput exportOutput) {
this.exportOutput = exportOutput;
}

@Override
public Object[] extract(ItemRow individual) {
return createRow(individual);
Expand Down Expand Up @@ -238,7 +243,7 @@ private Object[] createRow(ItemRow itemRow) {
return columnsData.toArray();
}

public void addRegistrationColumns(List<Object> columnsData, Individual individual, Map<String, FormElement> registrationMap, List<String> fields) {
private void addRegistrationColumns(List<Object> columnsData, Individual individual, Map<String, FormElement> registrationMap, List<String> fields) {
addStaticRegistrationColumns(columnsData, individual, HeaderCreator.getRegistrationDataMap(), fields);
addAddressLevels(columnsData, individual.getAddressLevel());
if (individual.getSubjectType().isGroup()) {
Expand All @@ -254,15 +259,15 @@ private void addStaticRegistrationColumns(List<Object> columnsData, Individual i
.forEach(key -> columnsData.add(registrationDataMap.get(key).getValueFunction().apply(individual)));
}

public void addEnrolmentColumns(List<Object> columnsData, ProgramEnrolment programEnrolment, Map<String, FormElement> enrolmentMap,
private void addEnrolmentColumns(List<Object> columnsData, ProgramEnrolment programEnrolment, Map<String, FormElement> enrolmentMap,
Map<String, FormElement> exitEnrolmentMap,
ExportOutput.ExportNestedOutput program) {
addStaticEnrolmentColumns(program, columnsData, programEnrolment, HeaderCreator.getEnrolmentDataMap());
columnsData.addAll(getObs(programEnrolment.getObservations(), enrolmentMap));
columnsData.addAll(getObs(programEnrolment.getObservations(), exitEnrolmentMap));
}

public <T extends AbstractEncounter> void addEncounterColumns(Long maxVisitCount, List<Object> columnsData, List<T> encounters,
private <T extends AbstractEncounter> void addEncounterColumns(Long maxVisitCount, List<Object> columnsData, List<T> encounters,
Map<String, FormElement> map, Map<String, FormElement> cancelMap, ExportEntityType encounterEntityType) {
AtomicInteger counter = new AtomicInteger(0);
encounters.forEach(encounter -> {
Expand Down
19 changes: 13 additions & 6 deletions avni-server-api/src/main/java/org/avni/server/exporter/v2/ExportV2ValidationHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@ public class ExportV2ValidationHelper implements LongitudinalExportRequestFieldN
private final static Pattern UUID_REGEX_PATTERN =
Pattern.compile("^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89aAbB][a-f0-9]{3}-[a-f0-9]{12}$");

public static final String INDIVIDUAL = "Individual";
public static final String ENCOUNTER = "Encounter";
public static final String GROUP_SUBJECT = "Group Subject";
public static final String GROUP_SUBJECT_ENCOUNTER = "Group Subject Encounter";
public static final String PROGRAM_ENROLMENT = "Program Enrolment";
public static final String PROGRAM_ENCOUNTER = "Program Encounter";

private static List<String> validRegistrationFields = Arrays.asList(
ID,
UUID,
Expand Down Expand Up @@ -91,26 +98,26 @@ public List<String> validate(ExportOutput exportOutput) {
if(validateIfDateFilterIsNotSpecified(exportOutput)) {
errorList.add("Individual Registration Date isn't specified");
}
validateRegistrationHeaders(errorList, "Individual", exportOutput.getFields());
validateRegistrationHeaders(errorList, INDIVIDUAL, exportOutput.getFields());

exportOutput.getEncounters().forEach(enc -> {
validateEncounterHeaders(errorList, "Encounter", enc.getFields());
validateEncounterHeaders(errorList, ENCOUNTER, enc.getFields());
});

exportOutput.getPrograms().forEach(enr -> {
validateEnrolmentHeaders(errorList, "Program Enrolments", enr.getFields());
validateEnrolmentHeaders(errorList, PROGRAM_ENROLMENT, enr.getFields());
});

exportOutput.getPrograms().stream().flatMap(enr -> enr.getEncounters().stream()).forEach(enc -> {
validateEncounterHeaders(errorList, "Program Encounter", enc.getFields());
validateEncounterHeaders(errorList, PROGRAM_ENCOUNTER, enc.getFields());
});

exportOutput.getGroups().forEach(grp -> {
validateRegistrationHeaders(errorList, "Group Subject", grp.getFields());
validateRegistrationHeaders(errorList, GROUP_SUBJECT, grp.getFields());
});

exportOutput.getGroups().stream().flatMap(grp -> grp.getEncounters().stream()).forEach(enc -> {
validateEncounterHeaders(errorList, "Group Subject Encounter", enc.getFields());
validateEncounterHeaders(errorList, GROUP_SUBJECT_ENCOUNTER, enc.getFields());
});

return errorList;
Expand Down
88 changes: 80 additions & 8 deletions avni-server-api/src/main/java/org/avni/server/web/ExportController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,14 @@

import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.commons.io.IOUtils;
import org.avni.server.dao.EncounterTypeRepository;
import org.avni.server.dao.JobStatus;
import org.avni.server.dao.ProgramRepository;
import org.avni.server.dao.SubjectTypeRepository;
import org.avni.server.domain.CHSBaseEntity;
import org.avni.server.domain.EncounterType;
import org.avni.server.domain.Program;
import org.avni.server.domain.SubjectType;
import org.avni.server.exporter.ExportJobService;
import org.avni.server.service.ExportS3Service;
import org.avni.server.util.ObjectMapperSingleton;
Expand All @@ -22,18 +29,27 @@

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

import static org.avni.server.exporter.v2.ExportV2ValidationHelper.*;

@RestController
public class ExportController {


private ExportJobService exportJobService;
private ExportS3Service exportS3Service;
private final ExportJobService exportJobService;
private final ExportS3Service exportS3Service;
private final SubjectTypeRepository subjectTypeRepository;
private final ProgramRepository programRepository;
private final EncounterTypeRepository encounterTypeRepository;

@Autowired
public ExportController(ExportJobService exportJobService, ExportS3Service exportS3Service) {
public ExportController(ExportJobService exportJobService, ExportS3Service exportS3Service, SubjectTypeRepository subjectTypeRepository, ProgramRepository programRepository, EncounterTypeRepository encounterTypeRepository) {
this.exportJobService = exportJobService;
this.exportS3Service = exportS3Service;
this.subjectTypeRepository = subjectTypeRepository;
this.programRepository = programRepository;
this.encounterTypeRepository = encounterTypeRepository;
}

@RequestMapping(value = "/export", method = RequestMethod.POST)
Expand All @@ -45,16 +61,24 @@ public ResponseEntity<?> getVisitData(@RequestBody ExportJobRequest exportJobReq
@RequestMapping(value = "/export/v2", method = RequestMethod.POST)
@PreAuthorize(value = "hasAnyAuthority('organisation_admin')")
public ResponseEntity<?> getVisitDataV2(@RequestBody ExportV2JobRequest exportJobRequest) {
ResponseEntity<?> validationErrorResponseEntity = validateHeader(exportJobRequest);
ExportOutput exportOutput = getExportOutput(exportJobRequest);
ResponseEntity<?> validationErrorResponseEntity = validateHeader(exportOutput);
if(validationErrorResponseEntity != null) {
return validationErrorResponseEntity;
}
ResponseEntity<?> uuidValidationErrorResponseEntity = validateEntityUUIDs(exportOutput);
if(uuidValidationErrorResponseEntity != null) {
return uuidValidationErrorResponseEntity;
}
return exportJobService.runExportV2Job(exportJobRequest);
}

private ResponseEntity<?> validateHeader(ExportV2JobRequest exportJobRequest) {
return ObjectMapperSingleton.getObjectMapper().convertValue(exportJobRequest.getIndividual(), new TypeReference<ExportOutput>() {})
.validate();
private ExportOutput getExportOutput(ExportV2JobRequest exportJobRequest) {
return ObjectMapperSingleton.getObjectMapper().convertValue(exportJobRequest.getIndividual(), new TypeReference<ExportOutput>() {});
}

private ResponseEntity<?> validateHeader(ExportOutput exportOutput) {
return exportOutput.validate();
}

@RequestMapping(value = "/export/status", method = RequestMethod.GET)
Expand Down Expand Up @@ -83,4 +107,52 @@ private HttpHeaders getHttpHeaders(String filename) {
return header;
}

private ResponseEntity<List<String>> validateEntityUUIDs(ExportOutput exportOutput) {
List<String> errorList = new ArrayList<>();
String subjectTypeUUID = exportOutput.getUuid();
SubjectType individualType = subjectTypeRepository.findByUuid(subjectTypeUUID);
validateEntity(individualType, INDIVIDUAL, subjectTypeUUID, errorList);

exportOutput.getEncounters().forEach(e -> {
String programEncounterTypeUUID = e.getUuid();
EncounterType encounterType = encounterTypeRepository.findByUuid(programEncounterTypeUUID);
validateEntity(encounterType, ENCOUNTER, programEncounterTypeUUID, errorList);
});

exportOutput.getGroups().forEach(g -> {
String groupSubjectTypeUUID = g.getUuid();
SubjectType groupSubjectType = subjectTypeRepository.findByUuid(groupSubjectTypeUUID);
validateEntity(groupSubjectType, GROUP_SUBJECT, groupSubjectTypeUUID, errorList);

g.getEncounters().forEach(ge -> {
String programEncounterTypeUUID = ge.getUuid();
EncounterType encounterType = encounterTypeRepository.findByUuid(programEncounterTypeUUID);
validateEntity(encounterType, GROUP_SUBJECT_ENCOUNTER, programEncounterTypeUUID, errorList);
});
});

exportOutput.getPrograms().forEach(p -> {
String programUUID = p.getUuid();
Program program = programRepository.findByUuid(programUUID);
validateEntity(program, PROGRAM_ENROLMENT, programUUID, errorList);

p.getEncounters().forEach(pe -> {
String programEncounterTypeUUID = pe.getUuid();
EncounterType encounterType = encounterTypeRepository.findByUuid(programEncounterTypeUUID);
validateEntity(encounterType, PROGRAM_ENCOUNTER, programEncounterTypeUUID, errorList);
});
});

if(!errorList.isEmpty()) {
return ResponseEntity.badRequest().body(errorList);
}
return null;
}

private void validateEntity(CHSBaseEntity entity, String entityType, String entityUUID, List<String> errorList) {
if(entity == null) {
errorList.add(String.format("Invalid UUID %s specified for %s type field", entityUUID, entityType));
}
}

}

0 comments on commit 476643f

Please sign in to comment.