Skip to content

Commit

Permalink
Merge pull request #72 from ao508/ddog-logging
Browse files Browse the repository at this point in the history 
Additional logging for datadaog
  • Loading branch information
@ao508
ao508 authored Jul 20, 2024
2 parents 9a38f8f + 73b86aa commit 9cfadbd
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 7 deletions.
2 changes: 2 additions & 0 deletions src/main/java/org/mskcc/smile/service/CmoLabelGeneratorService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,6 @@ Status generateSampleStatus(SampleMetadata sampleMetadata,
Boolean igoSampleRequiresLabelUpdate(String newCmoLabel, String existingCmoLabel);
String resolveSampleTypeAbbreviation(String specimenTypeValue, String sampleOriginValue,
String cmoSampleClassValue);
String generateValidationReport(String originalJson, String filteredJson)
throws JsonProcessingException;
}
103 changes: 100 additions & 3 deletions src/main/java/org/mskcc/smile/service/impl/CmoLabelGeneratorServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.logging.log4j.util.Strings;
import org.mskcc.smile.commons.enums.CmoSampleClass;
import org.mskcc.smile.commons.enums.NucleicAcid;
import org.mskcc.smile.commons.enums.SampleOrigin;
Expand Down Expand Up @@ -322,7 +324,7 @@ private String resolveNucleicAcidAbbreviation(String sampleTypeString,
return "d";
}
} catch (Exception e) {
LOG.debug("Could not resolve sample type acid from 'sampleType' - using default 'd'");
LOG.warn("Could not resolve sample type acid from 'sampleType' - using default 'd'");
}
// if nucleic acid abbreviation is still unknown then attempt to resolve from
// sample metadata --> cmo sample id fields --> naToExtract
Expand All @@ -341,7 +343,7 @@ private String resolveNucleicAcidAbbreviation(String sampleTypeString,
}
}
} catch (Exception e) {
LOG.debug("Could not resolve nucleic acid from 'naToExtract' - using default 'd'");
LOG.warn("Could not resolve nucleic acid from 'naToExtract' - using default 'd'");
return "d";
}

Expand Down Expand Up @@ -383,7 +385,7 @@ public String resolveSampleTypeAbbreviation(String specimenTypeValue, String sam
}
}
} catch (Exception e) {
LOG.debug("Could not resolve specimen type acid from 'specimenType': "
LOG.warn("Could not resolve specimen type acid from 'specimenType': "
+ specimenTypeValue);
}

Expand Down Expand Up @@ -564,4 +566,99 @@ private Boolean isCmoCelllineSample(IgoSampleManifest sample) {
return isCmoCelllineSample(sample.getSpecimenType(), sample.getCmoSampleIdFields());
}

@Override
public String generateValidationReport(String originalJson, String filteredJson)
throws JsonProcessingException {
StringBuilder builder = new StringBuilder();
String requestId = getRequestId(originalJson);
Map<String, Object> filteredJsonMap = mapper.readValue(filteredJson, Map.class);
// keeps track if there's anything to report or not. if still true after all checks
// then return null
Boolean allValid = Boolean.TRUE;

// if request-level status is missing from the filtered json then
// a critical error likely occurred, in which case the original json
// would be more helpful to have as a reference when debugging the error
if (!filteredJsonMap.containsKey("status")) {
allValid = Boolean.FALSE;
builder.append("Request JSON missing validation report ('status') post-validation:");
builder.append("\nOriginal JSON contents:\n")
.append(originalJson).append("\nFiltered JSON contents:\n")
.append(filteredJson);
} else {
Map<String, Object> statusMap = (Map<String, Object>) filteredJsonMap.get("status");
Map<String, Object> validationReport =
mapper.convertValue(statusMap.get("validationReport"), Map.class);

// if request validation report is not empty then log for ddog
if (!validationReport.isEmpty()) {
allValid = Boolean.FALSE;
builder.append("Request-level status and validation report for request '")
.append(requestId)
.append("': ")
.append(mapper.writeValueAsString(statusMap));
}
// check validation status for each sample individually as well and
// add contents to report for ddog
Object[] sampleList = mapper.convertValue(filteredJsonMap.get("samples"),
Object[].class);
for (Object s : sampleList) {
Map<String, Object> sampleMap = mapper.convertValue(s, Map.class);
Map<String, Object> sampleStatusMap = mapper.convertValue(sampleMap.get("status"), Map.class);
Map<String, String> sampleValidationReport =
mapper.convertValue(sampleStatusMap.get("validationReport"), Map.class);
try {
String sampleId = ObjectUtils.firstNonNull(
sampleMap.get("igoId"), sampleMap.get("primaryId")).toString();
if (!sampleValidationReport.isEmpty()) {
allValid = Boolean.FALSE;
builder.append("\nValidation report for sample '")
.append(sampleId)
.append("': ")
.append(mapper.writeValueAsString(sampleStatusMap));
}
} catch (NullPointerException e) {
builder.append("\nNo known identifiers in current sample data: ")
.append(mapper.writeValueAsString(sampleMap))
.append(", Validation report for unknown sample: ")
.append(mapper.writeValueAsString(sampleStatusMap));
}
}
}
// if allValid is still true then there wasn't anything to report at the request
// or sample level.. return null
return allValid ? null : builder.toString();
}

private String getRequestId(String json) throws JsonProcessingException {
if (isBlank(json)) {
return null;
}
Map<String, Object> jsonMap = mapper.readValue(json, Map.class);
return getRequestId(jsonMap);
}

private String getRequestId(Map<String, Object> jsonMap) throws JsonProcessingException {
if (jsonMap.containsKey("requestId")) {
return jsonMap.get("requestId").toString();
}
if (jsonMap.containsKey("igoRequestId")) {
return jsonMap.get("igoRequestId").toString();
}
if (jsonMap.containsKey("additionalProperties")) {
Map<String, String> additionalProperties = mapper.convertValue(
jsonMap.get("additionalProperties"), Map.class);
if (additionalProperties.containsKey("requestId")) {
return additionalProperties.get("requestId");
}
if (additionalProperties.containsKey("igoRequestId")) {
return additionalProperties.get("igoRequestId");
}
}
return null;
}

private Boolean isBlank(String value) {
return (Strings.isBlank(value) || value.equals("null"));
}
}
23 changes: 19 additions & 4 deletions src/main/java/org/mskcc/smile/service/impl/LabelGenMessageHandlingServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,8 @@ public void run() {
try {
String requestJson = labelGeneratorQueue.poll(100, TimeUnit.MILLISECONDS);
if (requestJson != null) {
LOG.info("Extracting samples from request received...");
String requestId = getRequestIdFromRequestJson(requestJson);
LOG.info("Extracting samples from request received: " + requestId);
List<Object> samples = getSamplesFromRequestJson(requestJson);

// get existing samples for all patients in the request
Expand Down Expand Up @@ -259,12 +259,19 @@ public void run() {
// and add updated request json to publisher queue
Map<String, Object> requestJsonMap = mapper.readValue(requestJson, Map.class);
requestJsonMap.put("samples", updatedSamples);
String updatedRequestJson = mapper.writeValueAsString(requestJsonMap);
// data dog log message
String ddogLogMessage = cmoLabelGeneratorService.generateValidationReport(
requestJson, updatedRequestJson);
if (ddogLogMessage != null) {
LOG.info(ddogLogMessage);
}
switch (igoRequestDest) {
case NEW_REQUEST_DEST:
igoNewRequestQueue.add(mapper.writeValueAsString(requestJsonMap));
igoNewRequestQueue.add(updatedRequestJson);
break;
case PROMOTED_REQUEST_DEST:
igoPromotedRequestQueue.add(mapper.writeValueAsString(requestJsonMap));
igoPromotedRequestQueue.add(updatedRequestJson);
break;
default:
break;
Expand Down Expand Up @@ -308,6 +315,7 @@ public void run() {
try {
SampleMetadata sample = cmoSampleLabelUpdateQueue.poll(100, TimeUnit.MILLISECONDS);
if (sample != null) {
String origSampleJson = mapper.writeValueAsString(sample);
List<SampleMetadata> existingSamples =
getExistingPatientSamples(sample.getCmoPatientId());
// Case when sample update json doesn't have status
Expand Down Expand Up @@ -335,8 +343,15 @@ public void run() {
sample.getPrimaryId(), existingSamples, newCmoSampleLabel);
sample.setCmoSampleName(resolvedCmoSampleLabel);
}
String updatedSampleJson = mapper.writeValueAsString(sample);
// data dog log message
String ddogLogMessage = cmoLabelGeneratorService.generateValidationReport(
origSampleJson, updatedSampleJson);
if (ddogLogMessage != null) {
LOG.info(ddogLogMessage);
}
LOG.info("Publishing sample to IGO_SAMPLE_UPDATE_TOPIC");
messagingGateway.publish(IGO_SAMPLE_UPDATE_TOPIC, mapper.writeValueAsString(sample));
messagingGateway.publish(IGO_SAMPLE_UPDATE_TOPIC, updatedSampleJson);
}
if (interrupted && cmoSampleLabelUpdateQueue.isEmpty()) {
break;
Expand Down

0 comments on commit 9cfadbd

Please sign in to comment.