From 35b6efdb4c84aea920706435be2105825c513a64 Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Fri, 27 Sep 2024 13:27:15 -0400 Subject: [PATCH] Improve US Core category slicing - When looking at Condition.category, also look for the US Core "health-concern" code with its custom system as well as the 16100001 (death diagnosis) SNOMED code. These two extra codes are mentioned by US Core but are not mentioned in the base R4 spec. - When looking at DiagnosticReport.category, instead of allowing all codes in the LOINC and the diagnostic section systems, only allow the very few codes mentioned in the US Core valuesets for Lab and Note profiles: "LAB" and the three LOINC codes for Notes. --- cumulus_library_data_metrics/__init__.py | 2 +- .../c_system_use/c_system_use.jinja | 10 ++--- .../c_system_use/c_system_use.py | 18 ++------ cumulus_library_data_metrics/resource_info.py | 18 +++++++- cumulus_library_data_metrics/systems.py | 1 + cumulus_library_data_metrics/utils.jinja | 45 +++++++++++-------- .../general/condition/0.ndjson | 4 +- .../general/diagnosticreport/0.ndjson | 6 +-- .../general/expected_condition_month.csv | 4 +- .../general/expected_condition_year.csv | 4 +- .../expected_diagnosticreport_month.csv | 4 +- .../expected_diagnosticreport_year.csv | 4 +- 12 files changed, 65 insertions(+), 55 deletions(-) diff --git a/cumulus_library_data_metrics/__init__.py b/cumulus_library_data_metrics/__init__.py index 15ce0cd..8cc7399 100644 --- a/cumulus_library_data_metrics/__init__.py +++ b/cumulus_library_data_metrics/__init__.py @@ -1,3 +1,3 @@ """Data Metrics study for Cumulus Library""" -__version__ = "5.0.0" +__version__ = "5.0.1" diff --git a/cumulus_library_data_metrics/c_system_use/c_system_use.jinja b/cumulus_library_data_metrics/c_system_use/c_system_use.jinja index 0aee100..2b85dd0 100644 --- a/cumulus_library_data_metrics/c_system_use/c_system_use.jinja +++ b/cumulus_library_data_metrics/c_system_use/c_system_use.jinja @@ -5,8 +5,8 @@ CREATE TABLE data_metrics__count_c_system_use_{{ src|lower }}_{{ field|lower }} WITH src_status AS {{ utils.extract_status(src) }}, -{% if category_system %} -src_categories AS {{ utils.extract_codes(src, 'category', system=category_system, is_array=true) }}, +{% if use_category %} +src_categories AS {{ utils.extract_codes(src, cat_field, system=cat_systems, is_array=true) }}, {% endif %} src_systems_flat AS ( @@ -57,7 +57,7 @@ simplified AS ( {{ utils.get_date_string(dates, 'year') }} AS "year", {% endif %} - {% if category_system %} + {% if use_category %} {{ utils.array_to_string('src_categories.codes') }} AS category, {% endif %} @@ -69,7 +69,7 @@ simplified AS ( LEFT JOIN src_status ON src.id = src_status.id - {% if category_system %} + {% if use_category %} LEFT JOIN src_categories ON src.id = src_categories.id {% endif %} @@ -80,7 +80,7 @@ simplified AS ( {% call utils.make_counts('simplified', output_mode) %} - {% if category_system %} + {% if use_category %} category, {% endif %} diff --git a/cumulus_library_data_metrics/c_system_use/c_system_use.py b/cumulus_library_data_metrics/c_system_use/c_system_use.py index e1c9474..73ad867 100644 --- a/cumulus_library_data_metrics/c_system_use/c_system_use.py +++ b/cumulus_library_data_metrics/c_system_use/c_system_use.py @@ -18,22 +18,10 @@ def make_table(self, **kwargs) -> None: self.queries.append(self.render_sql(self.name, system_names=systems.NAMES, **kwargs)) def add_metric_queries(self) -> None: - self.make_table( - src="Observation", - field="code", - category_system=systems.OBSERVATION_CATEGORY, - ) - self.make_table( - src="Observation", - field="valueCodeableConcept", - category_system=systems.OBSERVATION_CATEGORY, - ) + self.make_table(src="Observation", field="code", use_category=True) + self.make_table(src="Observation", field="valueCodeableConcept", use_category=True) self.make_table(src="AllergyIntolerance", field="code") - self.make_table( - src="Condition", - field="code", - category_system=systems.CONDITION_CATEGORY, - ) + self.make_table(src="Condition", field="code", use_category=True) self.make_table(src="Device", field="type") self.make_table(src="DiagnosticReport", field="code") self.make_table(src="DocumentReference", field="type") diff --git a/cumulus_library_data_metrics/resource_info.py b/cumulus_library_data_metrics/resource_info.py index 6337dde..f6afd82 100644 --- a/cumulus_library_data_metrics/resource_info.py +++ b/cumulus_library_data_metrics/resource_info.py @@ -9,26 +9,40 @@ }, "Condition": { "cat_field": "category", - "cat_systems": [systems.CONDITION_CATEGORY], + "cat_systems": [ + # https://hl7.org/fhir/us/core/stu4/ValueSet-us-core-condition-category.html + systems.CONDITION_CATEGORY, + [systems.USCORE_CONDITION_CATEGORY, ["health-concern"]], + [systems.SNOMED, ["16100001"]], + ], }, "DiagnosticReport": { "cat_field": "category", - "cat_systems": [systems.LOINC, systems.DIAGNOSTIC_SECTION], + # http://hl7.org/fhir/us/core/STU4/StructureDefinition-us-core-diagnosticreport-lab.html + # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-diagnosticreport-category.html + "cat_systems": [ + [systems.DIAGNOSTIC_SECTION, ["LAB"]], # for labs + [systems.LOINC, ["LP29684-5", "LP29708-2", "LP7839-6"]], # for notes + ], }, "DocumentReference": { "cat_field": "category", + # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-documentreference-category.html "cat_systems": [systems.USCORE_DOCREF_CATEGORY], }, "Encounter": { "cat_field": "type", + # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-encounter-type.html "cat_systems": [systems.CPT, systems.SNOMED], }, "MedicationRequest": { "cat_field": "category", + # http://hl7.org/fhir/R4/valueset-medicationrequest-category.html "cat_systems": [systems.MEDREQ_CATEGORY], }, "Observation": { "cat_field": "category", + # https://www.hl7.org/fhir/R4/valueset-observation-category.html "cat_systems": [systems.OBSERVATION_CATEGORY], }, } diff --git a/cumulus_library_data_metrics/systems.py b/cumulus_library_data_metrics/systems.py index cebe7a0..f53779f 100644 --- a/cumulus_library_data_metrics/systems.py +++ b/cumulus_library_data_metrics/systems.py @@ -16,6 +16,7 @@ DIAGNOSTIC_SECTION = "http://terminology.hl7.org/CodeSystem/v2-0074" MEDREQ_CATEGORY = "http://terminology.hl7.org/CodeSystem/medicationrequest-category" OBSERVATION_CATEGORY = "http://terminology.hl7.org/CodeSystem/observation-category" +USCORE_CONDITION_CATEGORY = "http://hl7.org/fhir/us/core/CodeSystem/condition-category" USCORE_DOCREF_CATEGORY = "http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category" # Object Identifiers (OIDs). diff --git a/cumulus_library_data_metrics/utils.jinja b/cumulus_library_data_metrics/utils.jinja index 7e887fe..adb1951 100644 --- a/cumulus_library_data_metrics/utils.jinja +++ b/cumulus_library_data_metrics/utils.jinja @@ -1,13 +1,30 @@ --- Extracts the codes from a codeableConcept and returns a select with (id, codes). -{% macro extract_codes(src, src_field, system=null, is_array=false) -%} -( - -- Support both a list of systems or a single system +{% macro _compare_systems(system) -%} + {#- Support both a list of systems or a single system #} {%- if system is string %} - {%- set system_list = "('" + system + "')" %} + c.coding.system = '{{ system }}' {%- elif system %} - {%- set system_list = "('" + system|join("', '") + "')" %} + ( + FALSE -- just here so rest of these can start with OR + {%- for sys_info in system %} + {#- sys_info can either by a string or a list of string then codes #} + {%- if sys_info is string %} + OR c.coding.system = '{{ sys_info }}' + {%- else %} + OR ( + c.coding.system = '{{ sys_info[0] }}' + AND c.coding.code in ('{{ sys_info[1]|join("', '") }}') + ) + {%- endif %} + {%- endfor %} + ) + {%- else %} + TRUE {%- endif %} +{%- endmacro %} +-- Extracts the codes from a codeableConcept and returns a select with (id, codes). +{% macro extract_codes(src, src_field, system=null, is_array=false) -%} +( -- Flatten codings of the provided system and recombine the codes as a list SELECT id, @@ -21,9 +38,8 @@ {%- endif %} WHERE c.coding.code IS NOT NULL - {% if system_list %} - AND c.coding.system in {{ system_list }} - {% endif %} + AND {{ _compare_systems(system) }} + GROUP BY id ) {%- endmacro %} @@ -32,13 +48,6 @@ -- Extracts the codes from a codeableConcept and returns a flat select with (id, code). {% macro extract_codes_flat(src, src_field, system=null, is_array=false) -%} ( - -- Support both a list of systems or a single system - {%- if system is string %} - {%- set system_list = "('" + system + "')" %} - {%- elif system %} - {%- set system_list = "('" + system|join("', '") + "')" %} - {%- endif %} - -- Flatten codings of the provided system and recombine the codes as a list SELECT id, @@ -53,9 +62,7 @@ {%- endif %} WHERE c.coding.code IS NOT NULL - {% if system_list %} - AND c.coding.system in {{ system_list }} - {% endif %} + AND {{ _compare_systems(system) }} ) {%- endmacro %} diff --git a/tests/data/c_resource_count/general/condition/0.ndjson b/tests/data/c_resource_count/general/condition/0.ndjson index 3326597..d1d6fb0 100644 --- a/tests/data/c_resource_count/general/condition/0.ndjson +++ b/tests/data/c_resource_count/general/condition/0.ndjson @@ -2,8 +2,8 @@ {"resourceType": "Condition", "id": "onsetDateTime", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "onsetDateTime": "2023-10-04", "onsetPeriod": {"start": "2002-02-02"}} {"resourceType": "Condition", "id": "onsetPeriod", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "onsetPeriod": {"start": "2023-10-04"}} {"resourceType": "Condition", "id": "no-date", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}]} -{"resourceType": "Condition", "id": "multiple-categories", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}, {"coding": [{"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"} -{"resourceType": "Condition", "id": "multiple-codings", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2022-11-28"} +{"resourceType": "Condition", "id": "multiple-categories", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}, {"coding": [{"code": "health-concern", "system": "http://hl7.org/fhir/us/core/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"} +{"resourceType": "Condition", "id": "multiple-codings", "category": [{"coding": [{"code": "16100001", "system": "http://snomed.info/sct"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2022-11-28"} {"resourceType": "Condition", "id": "unrelated-category", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "nope"}]}, {"coding": [{"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-10-28"} {"resourceType": "Condition", "id": "unrelated-coding", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "nope"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"} {"resourceType": "Condition", "id": "no-category", "recordedDate": "2023-10-03"} diff --git a/tests/data/c_resource_count/general/diagnosticreport/0.ndjson b/tests/data/c_resource_count/general/diagnosticreport/0.ndjson index 381aede..51bc6e6 100644 --- a/tests/data/c_resource_count/general/diagnosticreport/0.ndjson +++ b/tests/data/c_resource_count/general/diagnosticreport/0.ndjson @@ -1,6 +1,6 @@ -{"resourceType": "DiagnosticReport", "id": "with-date", "category": [{"coding": [{"code": "CT", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}], "effectiveDateTime": "2023-10-04"} +{"resourceType": "DiagnosticReport", "id": "with-date", "category": [{"coding": [{"code": "LAB", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}], "effectiveDateTime": "2023-10-04"} {"resourceType": "DiagnosticReport", "id": "with-issued", "issued": "2021-02-15", "status": "partial"} -{"resourceType": "DiagnosticReport", "id": "no-date", "category": [{"coding": [{"code": "CT", "system": "http://loinc.org"}]}]} -{"resourceType": "DiagnosticReport", "id": "unrelated-category", "category": [{"coding": [{"code": "1234", "system": "nope"}]}]} +{"resourceType": "DiagnosticReport", "id": "no-date", "category": [{"coding": [{"code": "LP29684-5", "system": "http://loinc.org"}]}]} +{"resourceType": "DiagnosticReport", "id": "unrelated-category", "category": [{"coding": [{"code": "12345-6", "system": "http://loinc.org"}, {"code": "CT", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}]} {"resourceType": "DiagnosticReport", "id": "no-category", "effectivePeriod": {"start": "2022-09-04"}} {"resourceType": "DiagnosticReport", "id": "nothing"} diff --git a/tests/data/c_resource_count/general/expected_condition_month.csv b/tests/data/c_resource_count/general/expected_condition_month.csv index 5b0fbcd..cc21849 100644 --- a/tests/data/c_resource_count/general/expected_condition_month.csv +++ b/tests/data/c_resource_count/general/expected_condition_month.csv @@ -2,9 +2,9 @@ cnt,category,month,status 2,encounter-diagnosis,2023-10,cumulus__none 1,problem-list-item,2023-11,cumulus__none 1,problem-list-item,2023-10,cumulus__none -1,encounter-diagnosis; problem-list-item,2023-11,cumulus__none -1,encounter-diagnosis; problem-list-item,2022-11,cumulus__none +1,encounter-diagnosis; health-concern,2023-11,cumulus__none 1,encounter-diagnosis,cumulus__none,cumulus__none 1,encounter-diagnosis,2023-11,cumulus__none 1,cumulus__none,cumulus__none,cumulus__none 1,cumulus__none,2023-10,cumulus__none +1,16100001; problem-list-item,2022-11,cumulus__none diff --git a/tests/data/c_resource_count/general/expected_condition_year.csv b/tests/data/c_resource_count/general/expected_condition_year.csv index d155ff7..b3321d8 100644 --- a/tests/data/c_resource_count/general/expected_condition_year.csv +++ b/tests/data/c_resource_count/general/expected_condition_year.csv @@ -1,8 +1,8 @@ cnt,category,year,status 3,encounter-diagnosis,2023,cumulus__none 2,problem-list-item,2023,cumulus__none -1,encounter-diagnosis; problem-list-item,2023,cumulus__none -1,encounter-diagnosis; problem-list-item,2022,cumulus__none +1,encounter-diagnosis; health-concern,2023,cumulus__none 1,encounter-diagnosis,cumulus__none,cumulus__none 1,cumulus__none,cumulus__none,cumulus__none 1,cumulus__none,2023,cumulus__none +1,16100001; problem-list-item,2022,cumulus__none diff --git a/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv b/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv index 8404cb1..ee24ccd 100644 --- a/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv +++ b/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv @@ -2,5 +2,5 @@ cnt,category,month,status 2,cumulus__none,cumulus__none,cumulus__none 1,cumulus__none,2022-09,cumulus__none 1,cumulus__none,2021-02,partial -1,CT,cumulus__none,cumulus__none -1,CT,2023-10,cumulus__none +1,LP29684-5,cumulus__none,cumulus__none +1,LAB,2023-10,cumulus__none diff --git a/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv b/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv index 1031de2..75af675 100644 --- a/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv +++ b/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv @@ -2,5 +2,5 @@ cnt,category,year,status 2,cumulus__none,cumulus__none,cumulus__none 1,cumulus__none,2022,cumulus__none 1,cumulus__none,2021,partial -1,CT,cumulus__none,cumulus__none -1,CT,2023,cumulus__none +1,LP29684-5,cumulus__none,cumulus__none +1,LAB,2023,cumulus__none