From 7637694fdaa7bbd8e0f0836e9c4943fd63acb4d4 Mon Sep 17 00:00:00 2001 From: julianxcarter Date: Fri, 8 Jul 2022 11:30:11 -0400 Subject: [PATCH 1/7] CDS 2022 value set updates --- src/extractors/CSVCancerDiseaseStatusExtractor.js | 2 +- src/helpers/lookups/diseaseStatusLookup.js | 9 +++++++-- .../fixtures/csv-cancer-disease-status-bundle.json | 10 +++++----- .../csv-cancer-disease-status-module-response.json | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/extractors/CSVCancerDiseaseStatusExtractor.js b/src/extractors/CSVCancerDiseaseStatusExtractor.js index 57fbbd0b..b5ccaeb5 100644 --- a/src/extractors/CSVCancerDiseaseStatusExtractor.js +++ b/src/extractors/CSVCancerDiseaseStatusExtractor.js @@ -36,7 +36,7 @@ class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor { status: observationStatus || 'final', value: { code: diseaseStatusCode, - system: 'http://snomed.info/sct', + system: diseaseStatusCode.includes('USCRS') ? 'http://hl7.org/fhir/us/mcode/CodeSystem/snomed-requested-cs' : 'http://snomed.info/sct', display: diseaseStatusText || getDiseaseStatusDisplay(diseaseStatusCode, this.implementation), }, subject: { diff --git a/src/helpers/lookups/diseaseStatusLookup.js b/src/helpers/lookups/diseaseStatusLookup.js index 1dc58dc7..795113b2 100644 --- a/src/helpers/lookups/diseaseStatusLookup.js +++ b/src/helpers/lookups/diseaseStatusLookup.js @@ -1,12 +1,17 @@ const { createInvertedLookup, createLowercaseLookup } = require('../lookupUtils'); -// Code mapping is based on current values at https://www.hl7.org/fhir/us/mcode/2021May/ValueSet-mcode-condition-status-trend-vs.html +// Code mapping is based on current values at https://hl7.org/fhir/us/mcode/ValueSet-mcode-condition-status-trend-vs.html +// along with legacy codes included at https://www.hl7.org/fhir/us/mcode/2021May/ValueSet-mcode-condition-status-trend-vs.html const mcodeDiseaseStatusTextToCodeLookup = { - 'No abnormality detected (finding)': '281900007', + 'No abnormality detected (finding)': '281900007', // No longer in the Vs, included for backwards compatibility 'Patient condition improved (finding)': '268910001', 'Patient\'s condition stable (finding)': '359746009', 'Patient\'s condition worsened (finding)': '271299001', 'Patient condition undetermined (finding)': '709137006', + // TODO: These are placeholder codes representing codes that are requested additions to the SNOMED vocabulary + // They will likely need to be updated in future versions of mCODE + 'Cancer in complete remission(finding)': 'USCRS-352236', + 'Cancer in partial remission (finding)': 'USCRS-352237' }; const mcodeDiseaseStatusCodeToTextLookup = createInvertedLookup(mcodeDiseaseStatusTextToCodeLookup); diff --git a/test/extractors/fixtures/csv-cancer-disease-status-bundle.json b/test/extractors/fixtures/csv-cancer-disease-status-bundle.json index e23b9cbe..b7bff8a8 100644 --- a/test/extractors/fixtures/csv-cancer-disease-status-bundle.json +++ b/test/extractors/fixtures/csv-cancer-disease-status-bundle.json @@ -3,10 +3,10 @@ "type": "collection", "entry": [ { - "fullUrl": "urn:uuid:4b9e9b5a8db529782cd9e89b68c6a3fac408d2195f025691a3f2850cab18057f", + "fullUrl": "urn:uuid:e8293a0d18fb20d6b1749009032a26f2d34d8418369c87c8345f6c988fa0fc33", "resource": { "resourceType": "Observation", - "id": "4b9e9b5a8db529782cd9e89b68c6a3fac408d2195f025691a3f2850cab18057f", + "id": "e8293a0d18fb20d6b1749009032a26f2d34d8418369c87c8345f6c988fa0fc33", "meta": { "profile": [ "http://hl7.org/fhir/us/mcode/StructureDefinition/mcode-cancer-disease-status" @@ -47,9 +47,9 @@ "valueCodeableConcept": { "coding": [ { - "system": "http://snomed.info/sct", - "code": "268910001", - "display": "Patient condition improved (finding)" + "system": "http://hl7.org/fhir/us/mcode/CodeSystem/snomed-requested-cs", + "code": "USCRS-352236", + "display": "Cancer in complete remission(finding)" } ] }, diff --git a/test/extractors/fixtures/csv-cancer-disease-status-module-response.json b/test/extractors/fixtures/csv-cancer-disease-status-module-response.json index 497744d0..82231bca 100644 --- a/test/extractors/fixtures/csv-cancer-disease-status-module-response.json +++ b/test/extractors/fixtures/csv-cancer-disease-status-module-response.json @@ -2,7 +2,7 @@ { "mrn": "mrn-1", "conditionid": "cond-1", - "diseasestatuscode": "268910001", + "diseasestatuscode": "USCRS-352236", "dateofobservation": "2019-12-02", "evidence": "363679005|252416005", "observationstatus": "amended" From d3407265b6acfcf1e3853d8682932012b951a589 Mon Sep 17 00:00:00 2001 From: julianxcarter Date: Fri, 8 Jul 2022 11:40:48 -0400 Subject: [PATCH 2/7] Lint fixes, unit test addition --- src/helpers/lookups/diseaseStatusLookup.js | 2 +- test/helpers/diseaseStatusUtils.test.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/helpers/lookups/diseaseStatusLookup.js b/src/helpers/lookups/diseaseStatusLookup.js index 795113b2..c00c4d93 100644 --- a/src/helpers/lookups/diseaseStatusLookup.js +++ b/src/helpers/lookups/diseaseStatusLookup.js @@ -11,7 +11,7 @@ const mcodeDiseaseStatusTextToCodeLookup = { // TODO: These are placeholder codes representing codes that are requested additions to the SNOMED vocabulary // They will likely need to be updated in future versions of mCODE 'Cancer in complete remission(finding)': 'USCRS-352236', - 'Cancer in partial remission (finding)': 'USCRS-352237' + 'Cancer in partial remission (finding)': 'USCRS-352237', }; const mcodeDiseaseStatusCodeToTextLookup = createInvertedLookup(mcodeDiseaseStatusTextToCodeLookup); diff --git a/test/helpers/diseaseStatusUtils.test.js b/test/helpers/diseaseStatusUtils.test.js index fd6bdb31..13d1a054 100644 --- a/test/helpers/diseaseStatusUtils.test.js +++ b/test/helpers/diseaseStatusUtils.test.js @@ -13,6 +13,8 @@ const mcodeDiseaseStatusTextToCodeLookup = { 'Patient\'s condition stable (finding)': '359746009', 'Patient\'s condition worsened (finding)': '271299001', 'Patient condition undetermined (finding)': '709137006', + 'Cancer in complete remission(finding)': 'USCRS-352236', + 'Cancer in partial remission (finding)': 'USCRS-352237', }; // Code mapping is based on initial values still in use by icare implementors From 492fae7750130717fddea2296cf90f872cf611f1 Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Thu, 15 Dec 2022 13:44:31 -0500 Subject: [PATCH 3/7] Allowing CSVs with no rows + adding trimming to CSV parsing --- src/helpers/csvParsingUtils.js | 10 ++++++++++ src/helpers/csvValidator.js | 8 ++++++-- src/modules/CSVFileModule.js | 8 +++++--- src/modules/CSVURLModule.js | 5 +++-- test/helpers/csvValidator.test.js | 16 ++++++++++------ 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/helpers/csvParsingUtils.js b/src/helpers/csvParsingUtils.js index ca3b9f0c..6edab0d9 100644 --- a/src/helpers/csvParsingUtils.js +++ b/src/helpers/csvParsingUtils.js @@ -44,6 +44,8 @@ const DEFAULT_OPTIONS = { columns: (header) => header.map((column) => stringNormalizer(column)), // https://csv.js.org/parse/options/bom/ bom: true, + // https://csv.js.org/parse/options/trim/ + trim: true, // https://csv.js.org/parse/options/skip_empty_lines/ skip_empty_lines: true, // NOTE: This will skip any records with empty values, not just skip the empty values themselves @@ -57,9 +59,17 @@ function csvParse(csvData, options = {}) { return parse(csvData, { ...DEFAULT_OPTIONS, ...options }); } +function getCSVHeader(csvData) { + return parse(csvData, { + bom: true, + trim: true, + to: 1, + })[0]; +} module.exports = { stringNormalizer, normalizeEmptyValues, csvParse, + getCSVHeader, }; diff --git a/src/helpers/csvValidator.js b/src/helpers/csvValidator.js index b764e9af..f507f8cd 100644 --- a/src/helpers/csvValidator.js +++ b/src/helpers/csvValidator.js @@ -3,11 +3,11 @@ const logger = require('./logger'); // Validates csvData against the csvSchema // Uses the csvFileIdentifier in logs for readability -function validateCSV(csvFileIdentifier, csvSchema, csvData) { +function validateCSV(csvFileIdentifier, csvSchema, csvData, header) { let isValid = true; // Check headers - const headers = Object.keys(csvData[0]).map((h) => h.toLowerCase()); + const headers = header.map((h) => h.toLowerCase()); const schemaDiff = _.difference(csvSchema.headers.map((h) => h.name.toLowerCase()), headers); const fileDiff = _.difference(headers, csvSchema.headers.map((h) => h.name.toLowerCase())); @@ -27,6 +27,10 @@ function validateCSV(csvFileIdentifier, csvSchema, csvData) { }); } + if (csvData.length === 0) { + return true; + } + // Check values csvData.forEach((row, i) => { Object.entries(row).forEach(([key, value], j) => { diff --git a/src/modules/CSVFileModule.js b/src/modules/CSVFileModule.js index f6b067d2..e7fc5c02 100644 --- a/src/modules/CSVFileModule.js +++ b/src/modules/CSVFileModule.js @@ -2,14 +2,16 @@ const fs = require('fs'); const moment = require('moment'); const logger = require('../helpers/logger'); const { validateCSV } = require('../helpers/csvValidator'); -const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils'); +const { csvParse, stringNormalizer, normalizeEmptyValues, getCSVHeader } = require('../helpers/csvParsingUtils'); class CSVFileModule { constructor(csvFilePath, unalterableColumns, parserOptions) { // Parse then normalize the data - const parsedData = csvParse(fs.readFileSync(csvFilePath), parserOptions); + const csvData = fs.readFileSync(csvFilePath); + const parsedData = csvParse(csvData, parserOptions); this.filePath = csvFilePath; this.data = normalizeEmptyValues(parsedData, unalterableColumns); + this.header = getCSVHeader(csvData); } async get(key, value, fromDate, toDate) { @@ -32,7 +34,7 @@ class CSVFileModule { async validate(csvSchema) { if (csvSchema) { logger.info(`Validating CSV file for ${this.filePath}`); - return validateCSV(this.filePath, csvSchema, this.data); + return validateCSV(this.filePath, csvSchema, this.data, this.header); } logger.warn(`No CSV schema provided for ${this.filePath}`); return true; diff --git a/src/modules/CSVURLModule.js b/src/modules/CSVURLModule.js index 13198f0c..1e7d64d8 100644 --- a/src/modules/CSVURLModule.js +++ b/src/modules/CSVURLModule.js @@ -2,7 +2,7 @@ const axios = require('axios'); const moment = require('moment'); const logger = require('../helpers/logger'); const { validateCSV } = require('../helpers/csvValidator'); -const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils'); +const { csvParse, stringNormalizer, normalizeEmptyValues, getCSVHeader } = require('../helpers/csvParsingUtils'); class CSVURLModule { constructor(url, unalterableColumns, parserOptions) { @@ -28,6 +28,7 @@ class CSVURLModule { const parsedData = csvParse(csvData, this.parserOptions); logger.debug('CSV Data parsing successful'); this.data = normalizeEmptyValues(parsedData, this.unalterableColumns); + this.header = getCSVHeader(csvData); } } @@ -55,7 +56,7 @@ class CSVURLModule { if (csvSchema) { this.data = normalizeEmptyValues(this.data, this.unalterableColumns); - return validateCSV(this.url, csvSchema, this.data); + return validateCSV(this.url, csvSchema, this.data, this.header); } logger.warn(`No CSV schema provided for data found at ${this.url}`); return true; diff --git a/test/helpers/csvValidator.test.js b/test/helpers/csvValidator.test.js index a575fab9..72dc2b18 100644 --- a/test/helpers/csvValidator.test.js +++ b/test/helpers/csvValidator.test.js @@ -83,26 +83,30 @@ const schema = { describe('csvValidator', () => { test('simple data validates', () => { - expect(validateCSV('', schema, SIMPLE_DATA)).toBe(true); + expect(validateCSV('', schema, SIMPLE_DATA, Object.keys(SIMPLE_DATA[0]))).toBe(true); }); test('data missing required value does not validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_MISSING_REQUIRED_VALUE)).toBe(false); + expect(validateCSV('', schema, SIMPLE_DATA_MISSING_REQUIRED_VALUE, Object.keys(SIMPLE_DATA_MISSING_REQUIRED_VALUE[0]))).toBe(false); }); test('data missing required header does not validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_MISSING_HEADER)).toBe(false); + expect(validateCSV('', schema, SIMPLE_DATA_MISSING_HEADER, Object.keys(SIMPLE_DATA_MISSING_HEADER[0]))).toBe(false); }); test('data with erroneous column should still validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_EXTRA_COLUMNS)).toBe(true); + expect(validateCSV('', schema, SIMPLE_DATA_EXTRA_COLUMNS, Object.keys(SIMPLE_DATA_EXTRA_COLUMNS[0]))).toBe(true); }); test('data missing an optional column should still validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_MISSING_OPTIONAL_COLUMN)).toBe(true); + expect(validateCSV('', schema, SIMPLE_DATA_MISSING_OPTIONAL_COLUMN, Object.keys(SIMPLE_DATA_MISSING_OPTIONAL_COLUMN[0]))).toBe(true); }); test('data with different casing in the column header should still validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_DIFFERENT_CASING)).toBe(true); + expect(validateCSV('', schema, SIMPLE_DATA_DIFFERENT_CASING, Object.keys(SIMPLE_DATA_DIFFERENT_CASING[0]))).toBe(true); + }); + + test('data with only the header but no rows should still validate', () => { + expect(validateCSV('', schema, [], ['header1', 'header2', 'header3'])).toBe(true); }); }); From c5d1184db66f70e14f9a86e1dbc0b18334b4f4f6 Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Thu, 15 Dec 2022 14:05:54 -0500 Subject: [PATCH 4/7] removing unnecessary data length check --- src/helpers/csvValidator.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/helpers/csvValidator.js b/src/helpers/csvValidator.js index f507f8cd..2b0fab06 100644 --- a/src/helpers/csvValidator.js +++ b/src/helpers/csvValidator.js @@ -27,10 +27,6 @@ function validateCSV(csvFileIdentifier, csvSchema, csvData, header) { }); } - if (csvData.length === 0) { - return true; - } - // Check values csvData.forEach((row, i) => { Object.entries(row).forEach(([key, value], j) => { From 90dc90d2d5977578cd92db886663fc9686e1430d Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Thu, 15 Dec 2022 15:17:36 -0500 Subject: [PATCH 5/7] Moving header lowercasing into getCSVHeader function --- src/helpers/csvParsingUtils.js | 2 +- src/helpers/csvValidator.js | 3 +-- test/helpers/csvValidator.test.js | 10 +++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/helpers/csvParsingUtils.js b/src/helpers/csvParsingUtils.js index 6edab0d9..2f58d2cd 100644 --- a/src/helpers/csvParsingUtils.js +++ b/src/helpers/csvParsingUtils.js @@ -64,7 +64,7 @@ function getCSVHeader(csvData) { bom: true, trim: true, to: 1, - })[0]; + })[0].map((h) => h.toLowerCase()); } module.exports = { diff --git a/src/helpers/csvValidator.js b/src/helpers/csvValidator.js index 2b0fab06..f295bec3 100644 --- a/src/helpers/csvValidator.js +++ b/src/helpers/csvValidator.js @@ -3,11 +3,10 @@ const logger = require('./logger'); // Validates csvData against the csvSchema // Uses the csvFileIdentifier in logs for readability -function validateCSV(csvFileIdentifier, csvSchema, csvData, header) { +function validateCSV(csvFileIdentifier, csvSchema, csvData, headers) { let isValid = true; // Check headers - const headers = header.map((h) => h.toLowerCase()); const schemaDiff = _.difference(csvSchema.headers.map((h) => h.name.toLowerCase()), headers); const fileDiff = _.difference(headers, csvSchema.headers.map((h) => h.name.toLowerCase())); diff --git a/test/helpers/csvValidator.test.js b/test/helpers/csvValidator.test.js index 72dc2b18..bdf5e69f 100644 --- a/test/helpers/csvValidator.test.js +++ b/test/helpers/csvValidator.test.js @@ -87,23 +87,23 @@ describe('csvValidator', () => { }); test('data missing required value does not validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_MISSING_REQUIRED_VALUE, Object.keys(SIMPLE_DATA_MISSING_REQUIRED_VALUE[0]))).toBe(false); + expect(validateCSV('', schema, SIMPLE_DATA_MISSING_REQUIRED_VALUE, Object.keys(SIMPLE_DATA_MISSING_REQUIRED_VALUE[0]).map((h) => h.toLowerCase()))).toBe(false); }); test('data missing required header does not validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_MISSING_HEADER, Object.keys(SIMPLE_DATA_MISSING_HEADER[0]))).toBe(false); + expect(validateCSV('', schema, SIMPLE_DATA_MISSING_HEADER, Object.keys(SIMPLE_DATA_MISSING_HEADER[0]).map((h) => h.toLowerCase()))).toBe(false); }); test('data with erroneous column should still validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_EXTRA_COLUMNS, Object.keys(SIMPLE_DATA_EXTRA_COLUMNS[0]))).toBe(true); + expect(validateCSV('', schema, SIMPLE_DATA_EXTRA_COLUMNS, Object.keys(SIMPLE_DATA_EXTRA_COLUMNS[0]).map((h) => h.toLowerCase()))).toBe(true); }); test('data missing an optional column should still validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_MISSING_OPTIONAL_COLUMN, Object.keys(SIMPLE_DATA_MISSING_OPTIONAL_COLUMN[0]))).toBe(true); + expect(validateCSV('', schema, SIMPLE_DATA_MISSING_OPTIONAL_COLUMN, Object.keys(SIMPLE_DATA_MISSING_OPTIONAL_COLUMN[0]).map((h) => h.toLowerCase()))).toBe(true); }); test('data with different casing in the column header should still validate', () => { - expect(validateCSV('', schema, SIMPLE_DATA_DIFFERENT_CASING, Object.keys(SIMPLE_DATA_DIFFERENT_CASING[0]))).toBe(true); + expect(validateCSV('', schema, SIMPLE_DATA_DIFFERENT_CASING, Object.keys(SIMPLE_DATA_DIFFERENT_CASING[0]).map((h) => h.toLowerCase()))).toBe(true); }); test('data with only the header but no rows should still validate', () => { From 4921f7415531fdb9e39732abfb5f20ea5e2519e6 Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Fri, 16 Dec 2022 13:50:59 -0500 Subject: [PATCH 6/7] 2.1.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 76438797..208767b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "mcode-extraction-framework", - "version": "2.0.1", + "version": "2.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 219d57d2..789ef233 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mcode-extraction-framework", - "version": "2.0.1", + "version": "2.1.0", "description": "", "contributors": [ "Julia Afeltra ", From 1bd1c18f16150d3d89d26192ec41c00e6eee1153 Mon Sep 17 00:00:00 2001 From: Dylan Mendelowitz Date: Fri, 16 Dec 2022 13:57:40 -0500 Subject: [PATCH 7/7] npm audit fix --- package-lock.json | 12 ++++++------ package.json | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 208767b4..f145a458 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8435,9 +8435,9 @@ "dev": true }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "requires": { "brace-expansion": "^1.1.7" @@ -8480,9 +8480,9 @@ } }, "moment": { - "version": "2.29.3", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz", - "integrity": "sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw==" + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==" }, "ms": { "version": "2.0.0", diff --git a/package.json b/package.json index 789ef233..88876bee 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "fhir-crud-client": "^1.2.2", "fhirpath": "2.1.5", "lodash": "^4.17.21", - "moment": "^2.29.3", + "moment": "^2.29.4", "nodemailer": "^6.7.2", "winston": "^3.2.1" },