Skip to content

Commit

Permalink
Datadog logging support (#57)
Browse files Browse the repository at this point in the history 
Signed-off-by: Angelica Ochoa <[email protected]>
  • Loading branch information
@ao508
ao508 authored Jul 18, 2024
1 parent c490510 commit b3ec946
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 4 deletions.
1 change: 1 addition & 0 deletions src/main/java/org/mskcc/smile/service/ValidRequestChecker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,5 @@ Map<String, Object> generatePromotedRequestValidationMap(String requestJson) thr
JsonProcessingException, IOException;
Map<String, Object> generatePromotedSampleValidationMap(Map<String, Object> sampleMap)
throws JsonMappingException, JsonProcessingException;
String generateValidationReport(String originalJson, String filteredJson) throws JsonProcessingException;
}
14 changes: 10 additions & 4 deletions src/main/java/org/mskcc/smile/service/impl/RequestFilterMsgHandlingServiceIml.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,29 +78,35 @@ public void run() {
String filteredRequestJson = validRequestChecker.getFilteredValidRequestJson(
requestJson);
Boolean passCheck = (filteredRequestJson != null);

if (validRequestChecker.isCmo(requestJson)) {
LOG.info("Handling CMO-specific sanity checking...");
if (passCheck) {
LOG.info("Sanity check passed, publishing to: " + CMO_LABEL_GENERATOR_TOPIC);
LOG.info("Request'" + requestId + "' passed sanity check, publishing to: "
+ CMO_LABEL_GENERATOR_TOPIC);
// even if sanity check passed there might still be information worth
// reporting from the sample-level validation reports
messagingGateway.publish(requestId,
CMO_LABEL_GENERATOR_TOPIC,
filteredRequestJson);
} else {
LOG.error("Sanity check failed on request: " + requestId);
}

} else {
LOG.info("Handling non-CMO request...");
if (passCheck) {
LOG.info("Sanity check passed, publishing to: " + IGO_NEW_REQUEST_TOPIC);
LOG.info("Request '" + requestId + "' passed sanity check, publishing to: "
+ IGO_NEW_REQUEST_TOPIC);
messagingGateway.publish(requestId,
IGO_NEW_REQUEST_TOPIC,
filteredRequestJson);
} else {
LOG.error("Sanity check failed on request: " + requestId);
}
}
// data dog log message
String ddogLogMessage = validRequestChecker.generateValidationReport(
requestJson, filteredRequestJson);
LOG.info(ddogLogMessage);
}
if (interrupted && requestFilterQueue.isEmpty()) {
break;
Expand Down
66 changes: 66 additions & 0 deletions src/main/java/org/mskcc/smile/service/impl/ValidRequestCheckerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ public class ValidRequestCheckerImpl implements ValidRequestChecker {
* - If the number of valid samples is less than the total number of samples that came
* with the request then the request is still considered valid but the request is
* logged by the request status logger to keep note of the invalid samples.
* @param requestJson
* @return String
* @throws IOException
*/
@Override
Expand Down Expand Up @@ -620,4 +622,68 @@ private Boolean requestHasSamples(String requestJson) throws JsonProcessingExcep
private Boolean isBlank(String value) {
return (Strings.isBlank(value) || value.equals("null"));
}

@Override
public String generateValidationReport(String originalJson, String filteredJson)
throws JsonProcessingException {
StringBuilder builder = new StringBuilder();
String requestId = getRequestId(originalJson);
Map<String, Object> filteredJsonMap = mapper.readValue(filteredJson, Map.class);
// keeps track if there's anything to report or not. if still true after all checks
// then return null
Boolean allValid = Boolean.TRUE;

// if request-level status is missing from the filtered json then
// a critical error likely occurred, in which case the original json
// would be more helpful to have as a reference when debugging the error
if (!filteredJsonMap.containsKey("status")) {
allValid = Boolean.FALSE;
builder.append("Request JSON missing validation report ('status') post-validation:");
builder.append("\nOriginal JSON contents:\n")
.append(originalJson).append("\nFiltered JSON contents:\n")
.append(filteredJson);
} else {
Map<String, Object> statusMap = (Map<String, Object>) filteredJsonMap.get("status");
Map<String, Object> validationReport =
mapper.convertValue(statusMap.get("validationReport"), Map.class);

// if request validation report is not empty then log for ddog
if (!validationReport.isEmpty()) {
allValid = Boolean.FALSE;
builder.append("Request-level status and validation report for request '")
.append(requestId)
.append("': ")
.append(mapper.writeValueAsString(statusMap));
}
// check validation status for each sample individually as well and
// add contents to report for ddog
Object[] sampleList = mapper.convertValue(filteredJsonMap.get("samples"),
Object[].class);
for (Object s : sampleList) {
Map<String, Object> sampleMap = mapper.convertValue(s, Map.class);
Map<String, Object> sampleStatusMap = mapper.convertValue(sampleMap.get("status"), Map.class);
Map<String, String> sampleValidationReport =
mapper.convertValue(sampleStatusMap.get("validationReport"), Map.class);
try {
String sampleId = ObjectUtils.firstNonNull(
sampleMap.get("igoId"), sampleMap.get("primaryId")).toString();
if (!sampleValidationReport.isEmpty()) {
allValid = Boolean.FALSE;
builder.append("\nValidation report for sample '")
.append(sampleId)
.append("': ")
.append(mapper.writeValueAsString(sampleStatusMap));
}
} catch (NullPointerException e) {
builder.append("\nNo known identifiers in current sample data: ")
.append(mapper.writeValueAsString(sampleMap))
.append(", Validation report for unknown sample: ")
.append(mapper.writeValueAsString(sampleStatusMap));
}
}
}
// if allValid is still true then there wasn't anything to report at the request
// or sample level.. return null
return allValid ? null : builder.toString();
}
}
57 changes: 57 additions & 0 deletions src/test/java/org/mskcc/smile/ValidRequestCheckerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
Expand Down Expand Up @@ -142,6 +143,10 @@ public void testRequestJsonWithNoValidSamples() throws Exception {
Map<String, Object> sampleStatus = mapper.convertValue(sampleMap.get("status"), Map.class);
Assert.assertFalse((Boolean) sampleStatus.get("validationStatus"));
}
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson.getJsonString(),
modifiedRequestJson);
Assert.assertFalse(StringUtils.isBlank(ddogValidationReport));
}

/**
Expand All @@ -155,6 +160,10 @@ public void testRequestJsonWithTwoInvalidSamples() throws Exception {
String modifiedRequestJson = validRequestChecker
.getFilteredValidRequestJson(requestJson.getJsonString());
Assert.assertNotSame(modifiedRequestJson, requestJson.getJsonString());
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson.getJsonString(),
modifiedRequestJson);
Assert.assertFalse(StringUtils.isBlank(ddogValidationReport));
}

/**
Expand All @@ -168,6 +177,10 @@ public void testRequestJsonWithMissingFieldsUsedForSTA() throws Exception {
String modifiedRequestJson = validRequestChecker
.getFilteredValidRequestJson(requestJson.getJsonString());
Assert.assertNotNull(modifiedRequestJson);
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson.getJsonString(),
modifiedRequestJson);
Assert.assertTrue(StringUtils.isBlank(ddogValidationReport));
}

/**
Expand All @@ -181,6 +194,10 @@ public void testRequestJsonWithMissingFieldsUsedForNAA() throws Exception {
String modifiedRequestJson = validRequestChecker
.getFilteredValidRequestJson(requestJson.getJsonString());
Assert.assertNotNull(modifiedRequestJson);
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson.getJsonString(),
modifiedRequestJson);
Assert.assertTrue(StringUtils.isBlank(ddogValidationReport));
}

@Test
Expand Down Expand Up @@ -212,6 +229,10 @@ public void testValidPromotedRequest() throws Exception {
Map<String, Object> requestStatus =
mapper.convertValue(promotedRequestJsonMap.get("status"), Map.class);
Assert.assertTrue((Boolean) requestStatus.get("validationStatus"));
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson,
mapper.writeValueAsString(promotedRequestJsonMap));
Assert.assertTrue(StringUtils.isBlank(ddogValidationReport));
}

@Test
Expand All @@ -223,6 +244,10 @@ public void testValidPromotedRequestFailedSample() throws Exception {
Map<String, Object> requestStatus =
mapper.convertValue(promotedRequestJsonMap.get("status"), Map.class);
Assert.assertFalse((Boolean) requestStatus.get("validationStatus"));
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson,
mapper.writeValueAsString(promotedRequestJsonMap));
Assert.assertFalse(StringUtils.isBlank(ddogValidationReport));
}

@Test
Expand All @@ -234,6 +259,10 @@ public void testValidPromotedRequestUniversalSchema() throws Exception {
Map<String, Object> requestStatus =
mapper.convertValue(promotedRequestJsonMap.get("status"), Map.class);
Assert.assertTrue((Boolean) requestStatus.get("validationStatus"));
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson,
mapper.writeValueAsString(promotedRequestJsonMap));
Assert.assertTrue(StringUtils.isBlank(ddogValidationReport));
}

@Test
Expand All @@ -245,6 +274,10 @@ public void testValidPromotedRequestMissingProjectId() throws Exception {
Map<String, Object> requestStatus =
mapper.convertValue(promotedRequestJsonMap.get("status"), Map.class);
Assert.assertTrue((Boolean) requestStatus.get("validationStatus"));
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson,
mapper.writeValueAsString(promotedRequestJsonMap));
Assert.assertTrue(StringUtils.isBlank(ddogValidationReport));
}

@Test
Expand All @@ -256,6 +289,10 @@ public void testValidPromotedRequestMissingRequestId() throws Exception {
Map<String, Object> requestStatus =
mapper.convertValue(promotedRequestJsonMap.get("status"), Map.class);
Assert.assertFalse((Boolean) requestStatus.get("validationStatus"));
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson,
mapper.writeValueAsString(promotedRequestJsonMap));
Assert.assertFalse(StringUtils.isBlank(ddogValidationReport));
}

@Test
Expand All @@ -268,6 +305,10 @@ public void testValidPromotedSampleWithNormPatientId() throws Exception {
Map<String, Object> requestStatus =
mapper.convertValue(promotedRequestJsonMap.get("status"), Map.class);
Assert.assertTrue((Boolean) requestStatus.get("validationStatus"));
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson,
mapper.writeValueAsString(promotedRequestJsonMap));
Assert.assertTrue(StringUtils.isBlank(ddogValidationReport));
}

/**
Expand All @@ -282,6 +323,10 @@ public void testRequestJsonWithSamplesMissingFastqs() throws Exception {
String modifiedRequestJson = validRequestChecker
.getFilteredValidRequestJson(requestJson.getJsonString());
Assert.assertNotNull(modifiedRequestJson);
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson.getJsonString(),
modifiedRequestJson);
Assert.assertFalse(StringUtils.isBlank(ddogValidationReport));
}

/**
Expand All @@ -306,6 +351,10 @@ public void testRequestJsonWithAllSamplesMissingFastqs() throws Exception {
Map<String, Object> sampleStatus = mapper.convertValue(sampleMap.get("status"), Map.class);
Assert.assertFalse((Boolean) sampleStatus.get("validationStatus"));
}
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson.getJsonString(),
modifiedRequestJson);
Assert.assertFalse(StringUtils.isBlank(ddogValidationReport));
}

/**
Expand Down Expand Up @@ -334,6 +383,10 @@ public void testRequestSamplesMissingPatientIds() throws Exception {
Object[] failedSamplesList = mapper.convertValue(validationReport.get("samples"),
Object[].class);
Assert.assertTrue(failedSamplesList.length == 3);
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson.getJsonString(),
modifiedRequestJson);
Assert.assertFalse(StringUtils.isBlank(ddogValidationReport));
}

/**
Expand All @@ -359,5 +412,9 @@ public void testRequestJsonWithSamplesHavingFalseIgoComplete() throws Exception
Assert.assertFalse((Boolean) sampleStatus.get("validationStatus"));
}
}
String ddogValidationReport =
validRequestChecker.generateValidationReport(requestJson.getJsonString(),
modifiedRequestJson);
Assert.assertFalse(StringUtils.isBlank(ddogValidationReport));
}
}

0 comments on commit b3ec946

Please sign in to comment.