From 7b31466c22b6c61cb2371db9c4d8acbb3d59d53c Mon Sep 17 00:00:00 2001 From: Richard Almeida Date: Mon, 1 Jul 2024 07:56:08 -0400 Subject: [PATCH] support library evaluate --- build-libraries.js | 80 +++++++++++++++++++++++++ config/development.json | 44 ++++++++++---- configLoader.js | 54 +++++++++++++---- cql-tests-runner.js | 129 +++++++++++++++++----------------------- package.json | 3 +- resultsShared.js | 120 +++++++++++++++++++++++++++++++++++++ 6 files changed, 331 insertions(+), 99 deletions(-) create mode 100644 build-libraries.js create mode 100644 resultsShared.js diff --git a/build-libraries.js b/build-libraries.js new file mode 100644 index 0000000..02cb016 --- /dev/null +++ b/build-libraries.js @@ -0,0 +1,80 @@ +#!/usr/bin/node + +const os = require('os'); +const fs = require('fs'); +const path = require('path'); +const ConfigLoader = require('./configLoader'); +const config = new ConfigLoader(); +const loadTests = require('./loadTests'); +const { generateEmptyResults } = require('./resultsShared'); + +async function main() { + + const tests = loadTests.load(); + // Set this to true to run only the first group of tests + const quickTest = config.Debug.QuickTest + const emptyResults = await generateEmptyResults(tests, quickTest); + + const skipMap = config.skipListMap(); + + + for (let testFile of emptyResults) { + await generateLibrariesFromTests(testFile, skipMap); + } + +} + +main(); + + +async function generateLibrariesFromTests(group, skipMap) { + if (!group || group.length === 0) return; + + const cqlFileVersion = config.Build.CqlFileVersion; + const cqlOutputPath = config.Build.CqlOutputPath; + + let testsName = ''; + let body = ''; + + for (let r of group) { + if (!testsName) { + testsName = r.testsName; + } + + if (r.invalid !== 'semantic') { + const defineVal = `define "${r.groupName}.${r.testName}": ${r.expression}`; + const key = `${r.testsName}-${r.groupName}-${r.testName}`; + let reason = ''; + + if (r.testStatus === 'skip') { + console.log(`Skipping ${key}`); + reason = "Skipped by cql-tests-runner"; + } else if (skipMap.has(key)) { + console.log(`Skipping ${key}`); + reason = skipMap.get(key); + } + + if (reason) { + body += `/* ${os.EOL} Skipped: ${reason} ${os.EOL} ${defineVal} ${os.EOL}*/${os.EOL}${os.EOL}`; + } else { + body += `${defineVal}${os.EOL}${os.EOL}`; + } + } + } + + if (!testsName) return; + + body = `library ${testsName} version '${cqlFileVersion}'${os.EOL}${os.EOL}${body}`; + + if (!fs.existsSync(cqlOutputPath)) { + fs.mkdirSync(cqlOutputPath, { recursive: true }); + } + + const fileName = `${testsName}.cql`; + const filePath = path.join(cqlOutputPath, fileName); + fs.writeFileSync(filePath, body, (error) => { + if (error) throw error; + }); + +} + diff --git a/config/development.json b/config/development.json index 9fab76e..6dcb103 100644 --- a/config/development.json +++ b/config/development.json @@ -1,12 +1,36 @@ { - "FhirServer": { - "BaseUrl": "https://cloud.alphora.com/sandbox/r4/cds/fhir/", - "CqlOperation": "$cql" - }, - "Tests": { - "ResultsPath": "./results" - }, - "Debug": { - "QuickTest": true - } + "FhirServer": { + "BaseUrl": "https://cloud.alphora.com/sandbox/r4/cds/fhir/", + "CqlOperation": "$cql" + }, + "Build": { + "CqlFileVersion": "1.0.000", + "CqlOutputPath": "./cql" + }, + "Debug": { + "QuickTest": true + }, + "Tests": { + "ResultsPath": "./results", + "SkipList": [ + { + "testsName": "CqlAggregateTest", + "groupName": "AggregateTests", + "testName": "RolledOutIntervals", + "reason": "CQLtoELM - Could not resolve identifier MedicationRequestIntervals in the current library" + }, + { + "testsName": "CqlDateTimeOperatorsTest", + "groupName": "DateTimeComponentFrom", + "testName": "DateTimeComponentFromTimezone", + "reason": "CQLtoElm - Timezone keyword is only valid in 1.3 or lower" + }, + { + "testsName": "CqlDateTimeOperatorsTest", + "groupName": "Uncertainty tests", + "testName": "TimeDurationBetweenHourDiffPrecision", + "reason": "CQLtoELM - Syntax error at Z" + } + ] + } } \ No newline at end of file diff --git a/configLoader.js b/configLoader.js index e9edcfd..a0ff1a7 100644 --- a/configLoader.js +++ b/configLoader.js @@ -2,47 +2,77 @@ const config = require('config'); class ConfigLoader { - + constructor() { const baseURL = process.env.SERVER_BASE_URL || config.get('FhirServer.BaseUrl') || 'https://cloud.alphora.com/sandbox/r4/cds/fhir'; this.FhirServer = { - BaseUrl: this.removeTrailingSlash(baseURL), + BaseUrl: this.#removeTrailingSlash(baseURL), CqlOperation: process.env.CQL_OPERATION || config.get('FhirServer.CqlOperation') || '$cql' }; + this.Build = { + CqlFileVersion: process.env.CQL_FILE_VERSION || config.get('Build.CqlFileVersion') || '1.0.000', + CqlOutputPath: process.env.CQL_OUTPUT_PATH || config.get('Build.CqlOutputPath') || './cql' + + } this.Tests = { - ResultsPath: process.env.RESULTS_PATH || config.get('Tests.ResultsPath') || './results' + ResultsPath: process.env.RESULTS_PATH || config.get('Tests.ResultsPath') || './results', + SkipList: process.env.SKIP_LIST || config.get('Tests.SkipList') || [] }; this.Debug = { - QuickTest: this.getQuickTestSetting() + QuickTest: this.#setQuickTestSetting() }; - } - removeTrailingSlash(url) { + this.CqlEndpoint = this.#cqlEndPoint(); + + } + // TODO: validate the config values + #removeTrailingSlash(url) { return url.endsWith('/') ? url.slice(0, -1) : url; } + #cqlEndPoint(){ + if (this.FhirServer.CqlOperation === '$cql') { + return '$cql'; + } + else { + return 'Library' + '/$evaluate'; + } + } + get apiUrl() { if (this.FhirServer.CqlOperation === '$cql') { return this.FhirServer.BaseUrl + '/$cql'; } else { - return this.FhirServer.BaseUrl + 'Library' + '/$evaluate'; + return this.FhirServer.BaseUrl + '/Library' + '/$evaluate'; } } - getQuickTestSetting() { + + #setQuickTestSetting() { if (process.env.QUICK_TEST !== undefined) { - return process.env.QUICK_TEST === 'true'; + return process.env.QUICK_TEST === 'true'; } - + const configValue = config.get('Debug.QuickTest'); if (configValue !== undefined) { - return configValue; + return configValue; } return true; - } + } + + skipListMap() { + const skipList = this.Tests.SkipList; + const skipMap = new Map( + skipList.map(skipItem => [ + `${skipItem.testsName}-${skipItem.groupName}-${skipItem.testName}`, + skipItem.reason + ]) + ); + return skipMap + } } module.exports = ConfigLoader; diff --git a/cql-tests-runner.js b/cql-tests-runner.js index dca0d4c..9ff28c2 100644 --- a/cql-tests-runner.js +++ b/cql-tests-runner.js @@ -10,7 +10,7 @@ const axios = require('axios'); const CQLEngine = require('./CQLEngine'); const ConfigLoader = require('./configLoader'); const config = new ConfigLoader(); -// TODO: Read server-url from environment path... +const resultsShared = require('./resultsShared'); // Setup for running both $cql and Library/$evaluate // Expand outputType to allow Parameters representation @@ -49,9 +49,6 @@ class Test { output: String([]) | { text: String, type: boolean | code | date | dateTime | decimal | integer | long | quantity | string | time }([]) } */ - - - class Result { testStatus; // String: pass | fail | skip | error responseStatus; // Integer @@ -91,101 +88,81 @@ class Result { async function main() { let serverBaseUrl = config.FhirServer.BaseUrl - let cqlEndpoint = config.FhirServer.CqlOperation; + let cqlEndpoint = config.CqlEndpoint; let outputPath = config.Tests.ResultsPath; - + + //TODO: CQLEngine needs adjustments to handle Library/$evaluate. Config forces use of proper operation name and baseURL let cqlEngine = new CQLEngine(serverBaseUrl, cqlEndpoint); cqlEngine.cqlVersion = '1.5'; - const tests = loadTests.load(); - var x; await require('./cvl/cvlLoader.js').then(([{ default: cvl }]) => { x = cvl }); + + const tests = loadTests.load(); // Set this to true to run only the first group of tests const quickTest = config.Debug.QuickTest - //const onlyTestsName = "CqlArithmeticFunctionsTest"; - //const onlyGroupName = "Ceiling"; - //const onlyTestName = "CeilingNeg1D1"; + const emptyResults = await resultsShared.generateEmptyResults(tests, quickTest); + + const skipMap = config.skipListMap(); let results = []; - for (const ts of tests) { - if (typeof onlyTestsName === 'undefined' || onlyTestsName === ts.name) { - console.log('Tests: ' + ts.name); - for (const group of ts.group) { - if (typeof onlyGroupName === 'undefined' || onlyGroupName === group.name) { - console.log(' Group: ' + group.name); - let test = group.test; - if (test != undefined) { - for (const t of test) { - if (typeof onlyTestName === 'undefined' || onlyTestName === t.name) { - console.log(' Test: ' + t.name); - results.push(new Result(ts.name, group.name, t)); - } - } - } - if (quickTest) { - break; // Only load 1 group for testing - } - } - } - if (quickTest) { - break; // Only load 1 test set for testing - } + for (let testFile of emptyResults) { + for (let result of testFile) { + await runTest(result, cqlEngine.apiUrl, x, skipMap); + results.push(result); } } - - for (let r of results) { - await runTest(r, cqlEngine.apiUrl, x); - } - logResults(cqlEngine, results, outputPath); }; main(); -async function runTest(result, apiUrl, cvl) { - if (result.testStatus !== 'skip') { - const data = { - "resourceType": "Parameters", - "parameter": [{ - "name": "expression", - "valueString": result.expression - }] - }; - - try { - console.log('Running test %s:%s:%s', result.testsName, result.groupName, result.testName); - const response = await axios.post(apiUrl, data, { - headers: { - 'Content-Type': 'application/json', - } - }); +async function runTest(result, apiUrl, cvl, skipMap) { + const key = `${result.testsName}-${result.groupName}-${result.testName}`; + if (result.testStatus === 'skip') { + result.SkipMessage = 'Skipped by cql-tests-runner'; + return result; + } else if (skipMap.has(key)) { + let reason = skipMap.get(key); + result.SkipMessage = `Skipped by config: ${reason}"`; + result.testStatus = 'skip'; + return result; + } + //TODO: handle instance api location for library/$evaluate + let data = resultsShared.generateParametersResource(result, config.FhirServer.CqlOperation); + + try { + console.log('Running test %s:%s:%s', result.testsName, result.groupName, result.testName); + const response = await axios.post(apiUrl, data, { + headers: { + 'Content-Type': 'application/json', + } + }); - result.responseStatus = response.status; + result.responseStatus = response.status; - const responseBody = response.data; - result.actual = extractResult(responseBody); + const responseBody = response.data; + result.actual = extractResult(responseBody); - const invalid = result.invalid; - if (invalid === 'true' || invalid === 'semantic') { - // TODO: Validate the error message is as expected... - result.testStatus = response.status === 200 ? 'fail' : 'pass'; + const invalid = result.invalid; + if (invalid === 'true' || invalid === 'semantic') { + // TODO: Validate the error message is as expected... + result.testStatus = response.status === 200 ? 'fail' : 'pass'; + } + else { + if (response.status === 200) { + result.testStatus = resultsEqual(cvl.parse(result.expected), result.actual) ? 'pass' : 'fail'; } else { - if (response.status === 200) { - result.testStatus = resultsEqual(cvl.parse(result.expected), result.actual) ? 'pass' : 'fail'; - } - else { - result.testStatus = 'fail'; - } + result.testStatus = 'fail'; } } - catch (error) { - result.testStatus = 'error'; - result.error = error; - }; } + catch (error) { + result.testStatus = 'error'; + result.error = error; + }; console.log('Test %s:%s:%s status: %s expected: %s actual: %s', result.testsName, result.groupName, result.name, result.testStatus, result.expected, result.actual); return result; @@ -207,7 +184,7 @@ function extractResult(response) { var result; if (response.hasOwnProperty('resourceType') && response.resourceType === 'Parameters') { for (let p of response.parameter) { - if (p.name === 'return') { + // if (p.name === 'return') { if (result === undefined) { if (p.hasOwnProperty("valueBoolean")) { result = p.valueBoolean; @@ -247,7 +224,7 @@ function extractResult(response) { result = undefined; break; } - } + // } } if (result !== undefined) { @@ -273,7 +250,7 @@ function logResult(result, outputPath) { } function logResults(cqlengine, results, outputPath) { - if(cqlengine instanceof CQLEngine){ + if (cqlengine instanceof CQLEngine) { const fileName = `${currentDate}_results.json`; if (!fs.existsSync(outputPath)) { fs.mkdirSync(outputPath, { recursive: true }); diff --git a/package.json b/package.json index 60b6617..001103b 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "Node application for running CQL tests", "main": "cql-tests-runner.js", "scripts": { - "test": "node cql-tests-runner.js" + "test": "node cql-tests-runner.js", + "build": "node build-libraries.js" }, "author": "", "license": "Apache-2.0", diff --git a/resultsShared.js b/resultsShared.js new file mode 100644 index 0000000..2b915f0 --- /dev/null +++ b/resultsShared.js @@ -0,0 +1,120 @@ + +class Result { + testStatus; // String: pass | fail | skip | error + responseStatus; // Integer + actual; // String + expected; // String + error; // Error + constructor(testsName, groupName, test) { + this.testsName = testsName; + this.groupName = groupName; + this.testName = test.name; + + if (typeof test.expression !== 'string') { + this.invalid = test.expression.invalid; + this.expression = test.expression.text; + } + else { + this.invalid = 'false'; + this.expression = test.expression; + } + + if (test.output !== undefined) { + if (typeof test.output !== 'string') { + // TODO: Structure the result if it can be structured (i.e. is one of the expected types) + this.expected = test.output.text; + } + else { + this.expected = test.output; + } + } + else { + this.testStatus = 'skip'; + } + } +} + + +async function generateEmptyResults(tests, quickTest) { + //const tests = loadTests.load(); + + // Set this to true to run only the first group of tests + //const quickTest = config.Debug.QuickTest + console.log('QuickTest: ' + quickTest) + // const onlyTestsName = "ValueLiteralsAndSelectors"; + // const onlyGroupName = "Decimal"; + // const onlyTestName = "DecimalNeg10Pow28ToZeroOneStepDecimalMinValue"; + + let results = []; + let groupResults = []; + for (const ts of tests) { + if (typeof onlyTestsName === 'undefined' || onlyTestsName === ts.name) { + console.log('Tests: ' + ts.name); + let groupTests = []; + for (const group of ts.group) { + if (typeof onlyGroupName === 'undefined' || onlyGroupName === group.name) { + console.log(' Group: ' + group.name); + let test = group.test; + if (test != undefined) { + for (const t of test) { + if (typeof onlyTestName === 'undefined' || onlyTestName === t.name) { + console.log(' Test: ' + t.name); + var r = new Result(ts.name, group.name, t); + results.push(r); + groupTests.push(r); + } + } + } + if (quickTest) { + break; // Only load 1 group for testing + } + } + + }//push array for each test file + groupResults.push(groupTests); + + if (quickTest) { + break; // Only load 1 test set for testing + } + } + } + return groupResults +} + +function generateParametersResource(result, cqlEndpoint) { + let data = ''; + + // Check if the last part is $cql or $evaluate + if (cqlEndpoint === '$cql') { + data = { + "resourceType": "Parameters", + "parameter": [{ + "name": "expression", + "valueString": result.expression + }] + }; + } else if (cqlEndpoint === '$evaluate') { + data = { + "resourceType": "Parameters", + "parameter": [{ + "name": "url", + "valueCanonical": "https://hl7.org/fhir/uv/cql/Library/" + result.testsName + "|1.0.000" + }, + { + "name": "expression", + "valueString": "" + result.groupName + '.' + result.testName + "" + } + ] + }; + } else { + console.log('The URL does not end with $cql or $evaluate'); + } + + return data; +} + + +module.exports = { + generateEmptyResults, + generateParametersResource +} \ No newline at end of file