Improve US Core category slicing

- When looking at Condition.category, also look for the US Core "health-concern" code with its custom system as well as the 16100001 (death diagnosis) SNOMED code. These two extra codes are mentioned by US Core but are not mentioned in the base R4 spec. - When looking at DiagnosticReport.category, instead of allowing all codes in the LOINC and the diagnostic section systems, only allow the very few codes mentioned in the US Core valuesets for Lab and Note profiles: "LAB" and the three LOINC codes for Notes.
smart-on-fhir · Sep 27, 2024 · 35b6efd · 35b6efd
1 parent 73a8743
commit 35b6efd
Show file tree

Hide file tree

Showing 12 changed files with 65 additions and 55 deletions.
diff --git a/cumulus_library_data_metrics/__init__.py b/cumulus_library_data_metrics/__init__.py
@@ -1,3 +1,3 @@
 """Data Metrics study for Cumulus Library"""
 
-__version__ = "5.0.0"
+__version__ = "5.0.1"
diff --git a/cumulus_library_data_metrics/c_system_use/c_system_use.jinja b/cumulus_library_data_metrics/c_system_use/c_system_use.jinja
@@ -5,8 +5,8 @@ CREATE TABLE data_metrics__count_c_system_use_{{ src|lower }}_{{ field|lower }}
 WITH
 src_status AS {{ utils.extract_status(src) }},
 
-{% if category_system %}
-src_categories AS {{ utils.extract_codes(src, 'category', system=category_system, is_array=true) }},
+{% if use_category %}
+src_categories AS {{ utils.extract_codes(src, cat_field, system=cat_systems, is_array=true) }},
 {% endif %}
 
 src_systems_flat AS (
@@ -57,7 +57,7 @@ simplified AS (
         {{ utils.get_date_string(dates, 'year') }} AS "year",
         {% endif %}
 
-        {% if category_system %}
+        {% if use_category %}
         {{ utils.array_to_string('src_categories.codes') }} AS category,
         {% endif %}
 
@@ -69,7 +69,7 @@ simplified AS (
     LEFT JOIN src_status
     ON src.id = src_status.id
 
-    {% if category_system %}
+    {% if use_category %}
     LEFT JOIN src_categories
     ON src.id = src_categories.id
     {% endif %}
@@ -80,7 +80,7 @@ simplified AS (
 
 
 {% call utils.make_counts('simplified', output_mode) %}
-    {% if category_system %}
+    {% if use_category %}
     category,
     {% endif %}
 

diff --git a/cumulus_library_data_metrics/c_system_use/c_system_use.py b/cumulus_library_data_metrics/c_system_use/c_system_use.py
@@ -18,22 +18,10 @@ def make_table(self, **kwargs) -> None:
         self.queries.append(self.render_sql(self.name, system_names=systems.NAMES, **kwargs))
 
     def add_metric_queries(self) -> None:
-        self.make_table(
-            src="Observation",
-            field="code",
-            category_system=systems.OBSERVATION_CATEGORY,
-        )
-        self.make_table(
-            src="Observation",
-            field="valueCodeableConcept",
-            category_system=systems.OBSERVATION_CATEGORY,
-        )
+        self.make_table(src="Observation", field="code", use_category=True)
+        self.make_table(src="Observation", field="valueCodeableConcept", use_category=True)
         self.make_table(src="AllergyIntolerance", field="code")
-        self.make_table(
-            src="Condition",
-            field="code",
-            category_system=systems.CONDITION_CATEGORY,
-        )
+        self.make_table(src="Condition", field="code", use_category=True)
         self.make_table(src="Device", field="type")
         self.make_table(src="DiagnosticReport", field="code")
         self.make_table(src="DocumentReference", field="type")

diff --git a/cumulus_library_data_metrics/resource_info.py b/cumulus_library_data_metrics/resource_info.py
@@ -9,26 +9,40 @@
     },
     "Condition": {
         "cat_field": "category",
-        "cat_systems": [systems.CONDITION_CATEGORY],
+        "cat_systems": [
+            # https://hl7.org/fhir/us/core/stu4/ValueSet-us-core-condition-category.html
+            systems.CONDITION_CATEGORY,
+            [systems.USCORE_CONDITION_CATEGORY, ["health-concern"]],
+            [systems.SNOMED, ["16100001"]],
+        ],
     },
     "DiagnosticReport": {
         "cat_field": "category",
-        "cat_systems": [systems.LOINC, systems.DIAGNOSTIC_SECTION],
+        # http://hl7.org/fhir/us/core/STU4/StructureDefinition-us-core-diagnosticreport-lab.html
+        # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-diagnosticreport-category.html
+        "cat_systems": [
+            [systems.DIAGNOSTIC_SECTION, ["LAB"]],  # for labs
+            [systems.LOINC, ["LP29684-5", "LP29708-2", "LP7839-6"]],  # for notes
+        ],
     },
     "DocumentReference": {
         "cat_field": "category",
+        # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-documentreference-category.html
         "cat_systems": [systems.USCORE_DOCREF_CATEGORY],
     },
     "Encounter": {
         "cat_field": "type",
+        # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-encounter-type.html
         "cat_systems": [systems.CPT, systems.SNOMED],
     },
     "MedicationRequest": {
         "cat_field": "category",
+        # http://hl7.org/fhir/R4/valueset-medicationrequest-category.html
         "cat_systems": [systems.MEDREQ_CATEGORY],
     },
     "Observation": {
         "cat_field": "category",
+        # https://www.hl7.org/fhir/R4/valueset-observation-category.html
         "cat_systems": [systems.OBSERVATION_CATEGORY],
     },
 }

diff --git a/cumulus_library_data_metrics/systems.py b/cumulus_library_data_metrics/systems.py
@@ -16,6 +16,7 @@
 DIAGNOSTIC_SECTION = "http://terminology.hl7.org/CodeSystem/v2-0074"
 MEDREQ_CATEGORY = "http://terminology.hl7.org/CodeSystem/medicationrequest-category"
 OBSERVATION_CATEGORY = "http://terminology.hl7.org/CodeSystem/observation-category"
+USCORE_CONDITION_CATEGORY = "http://hl7.org/fhir/us/core/CodeSystem/condition-category"
 USCORE_DOCREF_CATEGORY = "http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category"
 
 # Object Identifiers (OIDs).

diff --git a/cumulus_library_data_metrics/utils.jinja b/cumulus_library_data_metrics/utils.jinja
@@ -1,13 +1,30 @@
--- Extracts the codes from a codeableConcept and returns a select with (id, codes).
-{% macro extract_codes(src, src_field, system=null, is_array=false) -%}
-(
-    -- Support both a list of systems or a single system
+{% macro _compare_systems(system) -%}
+    {#- Support both a list of systems or a single system #}
     {%- if system is string %}
-        {%- set system_list = "('" + system + "')" %}
+        c.coding.system = '{{ system }}'
     {%- elif system %}
-        {%- set system_list = "('" + system|join("', '") + "')" %}
+        (
+            FALSE -- just here so rest of these can start with OR
+        {%- for sys_info in system %}
+            {#- sys_info can either by a string or a list of string then codes #}
+            {%- if sys_info is string %}
+            OR c.coding.system = '{{ sys_info }}'
+            {%- else %}
+            OR (
+                c.coding.system = '{{ sys_info[0] }}'
+                AND c.coding.code in ('{{ sys_info[1]|join("', '") }}')
+            )
+            {%- endif %}
+        {%- endfor %}
+        )
+    {%- else %}
+        TRUE
     {%- endif %}
+{%- endmacro %}
 
+-- Extracts the codes from a codeableConcept and returns a select with (id, codes).
+{% macro extract_codes(src, src_field, system=null, is_array=false) -%}
+(
     -- Flatten codings of the provided system and recombine the codes as a list
     SELECT
         id,
@@ -21,9 +38,8 @@
         {%- endif %}
     WHERE
         c.coding.code IS NOT NULL
-    {% if system_list %}
-        AND c.coding.system in {{ system_list }}
-    {% endif %}
+        AND {{ _compare_systems(system) }}
+
     GROUP BY id
 )
 {%- endmacro %}
@@ -32,13 +48,6 @@
 -- Extracts the codes from a codeableConcept and returns a flat select with (id, code).
 {% macro extract_codes_flat(src, src_field, system=null, is_array=false) -%}
 (
-    -- Support both a list of systems or a single system
-    {%- if system is string %}
-        {%- set system_list = "('" + system + "')" %}
-    {%- elif system %}
-        {%- set system_list = "('" + system|join("', '") + "')" %}
-    {%- endif %}
-
     -- Flatten codings of the provided system and recombine the codes as a list
     SELECT
         id,
@@ -53,9 +62,7 @@
         {%- endif %}
     WHERE
         c.coding.code IS NOT NULL
-    {% if system_list %}
-        AND c.coding.system in {{ system_list }}
-    {% endif %}
+        AND {{ _compare_systems(system) }}
 )
 {%- endmacro %}
 

diff --git a/tests/data/c_resource_count/general/condition/0.ndjson b/tests/data/c_resource_count/general/condition/0.ndjson
@@ -2,8 +2,8 @@
 {"resourceType": "Condition", "id": "onsetDateTime", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "onsetDateTime": "2023-10-04", "onsetPeriod": {"start": "2002-02-02"}}
 {"resourceType": "Condition", "id": "onsetPeriod", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "onsetPeriod": {"start": "2023-10-04"}}
 {"resourceType": "Condition", "id": "no-date", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}]}
-{"resourceType": "Condition", "id": "multiple-categories", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}, {"coding": [{"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"}
-{"resourceType": "Condition", "id": "multiple-codings", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2022-11-28"}
+{"resourceType": "Condition", "id": "multiple-categories", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}, {"coding": [{"code": "health-concern", "system": "http://hl7.org/fhir/us/core/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"}
+{"resourceType": "Condition", "id": "multiple-codings", "category": [{"coding": [{"code": "16100001", "system": "http://snomed.info/sct"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2022-11-28"}
 {"resourceType": "Condition", "id": "unrelated-category", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "nope"}]}, {"coding": [{"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-10-28"}
 {"resourceType": "Condition", "id": "unrelated-coding", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "nope"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"}
 {"resourceType": "Condition", "id": "no-category", "recordedDate": "2023-10-03"}

diff --git a/tests/data/c_resource_count/general/diagnosticreport/0.ndjson b/tests/data/c_resource_count/general/diagnosticreport/0.ndjson
@@ -1,6 +1,6 @@
-{"resourceType": "DiagnosticReport", "id": "with-date", "category": [{"coding": [{"code": "CT", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}], "effectiveDateTime": "2023-10-04"}
+{"resourceType": "DiagnosticReport", "id": "with-date", "category": [{"coding": [{"code": "LAB", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}], "effectiveDateTime": "2023-10-04"}
 {"resourceType": "DiagnosticReport", "id": "with-issued", "issued": "2021-02-15", "status": "partial"}
-{"resourceType": "DiagnosticReport", "id": "no-date", "category": [{"coding": [{"code": "CT", "system": "http://loinc.org"}]}]}
-{"resourceType": "DiagnosticReport", "id": "unrelated-category", "category": [{"coding": [{"code": "1234", "system": "nope"}]}]}
+{"resourceType": "DiagnosticReport", "id": "no-date", "category": [{"coding": [{"code": "LP29684-5", "system": "http://loinc.org"}]}]}
+{"resourceType": "DiagnosticReport", "id": "unrelated-category", "category": [{"coding": [{"code": "12345-6", "system": "http://loinc.org"}, {"code": "CT", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}]}
 {"resourceType": "DiagnosticReport", "id": "no-category", "effectivePeriod": {"start": "2022-09-04"}}
 {"resourceType": "DiagnosticReport", "id": "nothing"}
diff --git a/tests/data/c_resource_count/general/expected_condition_month.csv b/tests/data/c_resource_count/general/expected_condition_month.csv
@@ -2,9 +2,9 @@ cnt,category,month,status
 2,encounter-diagnosis,2023-10,cumulus__none
 1,problem-list-item,2023-11,cumulus__none
 1,problem-list-item,2023-10,cumulus__none
-1,encounter-diagnosis; problem-list-item,2023-11,cumulus__none
-1,encounter-diagnosis; problem-list-item,2022-11,cumulus__none
+1,encounter-diagnosis; health-concern,2023-11,cumulus__none
 1,encounter-diagnosis,cumulus__none,cumulus__none
 1,encounter-diagnosis,2023-11,cumulus__none
 1,cumulus__none,cumulus__none,cumulus__none
 1,cumulus__none,2023-10,cumulus__none
+1,16100001; problem-list-item,2022-11,cumulus__none
diff --git a/tests/data/c_resource_count/general/expected_condition_year.csv b/tests/data/c_resource_count/general/expected_condition_year.csv
@@ -1,8 +1,8 @@
 cnt,category,year,status
 3,encounter-diagnosis,2023,cumulus__none
 2,problem-list-item,2023,cumulus__none
-1,encounter-diagnosis; problem-list-item,2023,cumulus__none
-1,encounter-diagnosis; problem-list-item,2022,cumulus__none
+1,encounter-diagnosis; health-concern,2023,cumulus__none
 1,encounter-diagnosis,cumulus__none,cumulus__none
 1,cumulus__none,cumulus__none,cumulus__none
 1,cumulus__none,2023,cumulus__none
+1,16100001; problem-list-item,2022,cumulus__none
diff --git a/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv b/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv
@@ -2,5 +2,5 @@ cnt,category,month,status
 2,cumulus__none,cumulus__none,cumulus__none
 1,cumulus__none,2022-09,cumulus__none
 1,cumulus__none,2021-02,partial
-1,CT,cumulus__none,cumulus__none
-1,CT,2023-10,cumulus__none
+1,LP29684-5,cumulus__none,cumulus__none
+1,LAB,2023-10,cumulus__none
diff --git a/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv b/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv
@@ -2,5 +2,5 @@ cnt,category,year,status
 2,cumulus__none,cumulus__none,cumulus__none
 1,cumulus__none,2022,cumulus__none
 1,cumulus__none,2021,partial
-1,CT,cumulus__none,cumulus__none
-1,CT,2023,cumulus__none
+1,LP29684-5,cumulus__none,cumulus__none
+1,LAB,2023,cumulus__none