Merge pull request #60 from smart-on-fhir/mikix/category-fixes

Improve US Core category slicing

cumulus_library_data_metrics/__init__.py

@@ -1,3 +1,3 @@
 """Data Metrics study for Cumulus Library"""
 
-__version__ = "5.0.0"
+__version__ = "5.0.1"

cumulus_library_data_metrics/c_system_use/c_system_use.jinja

@@ -5,8 +5,8 @@ CREATE TABLE data_metrics__count_c_system_use_{{ src|lower }}_{{ field|lower }}
 WITH
 src_status AS {{ utils.extract_status(src) }},
 
-{% if category_system %}
-src_categories AS {{ utils.extract_codes(src, 'category', system=category_system, is_array=true) }},
+{% if use_category %}
+src_categories AS {{ utils.extract_codes(src, cat_field, system=cat_systems, is_array=true) }},
 {% endif %}
 
 src_systems_flat AS (
@@ -57,7 +57,7 @@ simplified AS (
         {{ utils.get_date_string(dates, 'year') }} AS "year",
         {% endif %}
 
-        {% if category_system %}
+        {% if use_category %}
         {{ utils.array_to_string('src_categories.codes') }} AS category,
         {% endif %}
 
@@ -69,7 +69,7 @@ simplified AS (
     LEFT JOIN src_status
     ON src.id = src_status.id
 
-    {% if category_system %}
+    {% if use_category %}
     LEFT JOIN src_categories
     ON src.id = src_categories.id
     {% endif %}
@@ -80,7 +80,7 @@ simplified AS (
 
 
 {% call utils.make_counts('simplified', output_mode) %}
-    {% if category_system %}
+    {% if use_category %}
     category,
     {% endif %}
 

cumulus_library_data_metrics/c_system_use/c_system_use.py

@@ -18,22 +18,10 @@ def make_table(self, **kwargs) -> None:
         self.queries.append(self.render_sql(self.name, system_names=systems.NAMES, **kwargs))
 
     def add_metric_queries(self) -> None:
-        self.make_table(
-            src="Observation",
-            field="code",
-            category_system=systems.OBSERVATION_CATEGORY,
-        )
-        self.make_table(
-            src="Observation",
-            field="valueCodeableConcept",
-            category_system=systems.OBSERVATION_CATEGORY,
-        )
+        self.make_table(src="Observation", field="code", use_category=True)
+        self.make_table(src="Observation", field="valueCodeableConcept", use_category=True)
         self.make_table(src="AllergyIntolerance", field="code")
-        self.make_table(
-            src="Condition",
-            field="code",
-            category_system=systems.CONDITION_CATEGORY,
-        )
+        self.make_table(src="Condition", field="code", use_category=True)
         self.make_table(src="Device", field="type")
         self.make_table(src="DiagnosticReport", field="code")
         self.make_table(src="DocumentReference", field="type")

cumulus_library_data_metrics/resource_info.py

@@ -9,26 +9,40 @@
     },
     "Condition": {
         "cat_field": "category",
-        "cat_systems": [systems.CONDITION_CATEGORY],
+        "cat_systems": [
+            # https://hl7.org/fhir/us/core/stu4/ValueSet-us-core-condition-category.html
+            systems.CONDITION_CATEGORY,
+            [systems.USCORE_CONDITION_CATEGORY, ["health-concern"]],
+            [systems.SNOMED, ["16100001"]],
+        ],
     },
     "DiagnosticReport": {
         "cat_field": "category",
-        "cat_systems": [systems.LOINC, systems.DIAGNOSTIC_SECTION],
+        # http://hl7.org/fhir/us/core/STU4/StructureDefinition-us-core-diagnosticreport-lab.html
+        # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-diagnosticreport-category.html
+        "cat_systems": [
+            [systems.DIAGNOSTIC_SECTION, ["LAB"]],  # for labs
+            [systems.LOINC, ["LP29684-5", "LP29708-2", "LP7839-6"]],  # for notes
+        ],
     },
     "DocumentReference": {
         "cat_field": "category",
+        # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-documentreference-category.html
         "cat_systems": [systems.USCORE_DOCREF_CATEGORY],
     },
     "Encounter": {
         "cat_field": "type",
+        # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-encounter-type.html
         "cat_systems": [systems.CPT, systems.SNOMED],
     },
     "MedicationRequest": {
         "cat_field": "category",
+        # http://hl7.org/fhir/R4/valueset-medicationrequest-category.html
         "cat_systems": [systems.MEDREQ_CATEGORY],
     },
     "Observation": {
         "cat_field": "category",
+        # https://www.hl7.org/fhir/R4/valueset-observation-category.html
         "cat_systems": [systems.OBSERVATION_CATEGORY],
     },
 }

cumulus_library_data_metrics/systems.py

@@ -16,6 +16,7 @@
 DIAGNOSTIC_SECTION = "http://terminology.hl7.org/CodeSystem/v2-0074"
 MEDREQ_CATEGORY = "http://terminology.hl7.org/CodeSystem/medicationrequest-category"
 OBSERVATION_CATEGORY = "http://terminology.hl7.org/CodeSystem/observation-category"
+USCORE_CONDITION_CATEGORY = "http://hl7.org/fhir/us/core/CodeSystem/condition-category"
 USCORE_DOCREF_CATEGORY = "http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category"
 
 # Object Identifiers (OIDs).

cumulus_library_data_metrics/utils.jinja

@@ -1,13 +1,30 @@
--- Extracts the codes from a codeableConcept and returns a select with (id, codes).
-{% macro extract_codes(src, src_field, system=null, is_array=false) -%}
-(
-    -- Support both a list of systems or a single system
+{% macro _compare_systems(system) -%}
+    {#- Support both a list of systems or a single system #}
     {%- if system is string %}
-        {%- set system_list = "('" + system + "')" %}
+        c.coding.system = '{{ system }}'
     {%- elif system %}
-        {%- set system_list = "('" + system|join("', '") + "')" %}
+        (
+            FALSE -- just here so rest of these can start with OR
+        {%- for sys_info in system %}
+            {#- sys_info can either by a string or a list of string then codes #}
+            {%- if sys_info is string %}
+            OR c.coding.system = '{{ sys_info }}'
+            {%- else %}
+            OR (
+                c.coding.system = '{{ sys_info[0] }}'
+                AND c.coding.code in ('{{ sys_info[1]|join("', '") }}')
+            )
+            {%- endif %}
+        {%- endfor %}
+        )
+    {%- else %}
+        TRUE
     {%- endif %}
+{%- endmacro %}
 
+-- Extracts the codes from a codeableConcept and returns a select with (id, codes).
+{% macro extract_codes(src, src_field, system=null, is_array=false) -%}
+(
     -- Flatten codings of the provided system and recombine the codes as a list
     SELECT
         id,
@@ -21,9 +38,8 @@
         {%- endif %}
     WHERE
         c.coding.code IS NOT NULL
-    {% if system_list %}
-        AND c.coding.system in {{ system_list }}
-    {% endif %}
+        AND {{ _compare_systems(system) }}
+
     GROUP BY id
 )
 {%- endmacro %}
@@ -32,13 +48,6 @@
 -- Extracts the codes from a codeableConcept and returns a flat select with (id, code).
 {% macro extract_codes_flat(src, src_field, system=null, is_array=false) -%}
 (
-    -- Support both a list of systems or a single system
-    {%- if system is string %}
-        {%- set system_list = "('" + system + "')" %}
-    {%- elif system %}
-        {%- set system_list = "('" + system|join("', '") + "')" %}
-    {%- endif %}
-
     -- Flatten codings of the provided system and recombine the codes as a list
     SELECT
         id,
@@ -53,9 +62,7 @@
         {%- endif %}
     WHERE
         c.coding.code IS NOT NULL
-    {% if system_list %}
-        AND c.coding.system in {{ system_list }}
-    {% endif %}
+        AND {{ _compare_systems(system) }}
 )
 {%- endmacro %}
 

tests/data/c_resource_count/general/condition/0.ndjson

@@ -2,8 +2,8 @@
 {"resourceType": "Condition", "id": "onsetDateTime", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "onsetDateTime": "2023-10-04", "onsetPeriod": {"start": "2002-02-02"}}
 {"resourceType": "Condition", "id": "onsetPeriod", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "onsetPeriod": {"start": "2023-10-04"}}
 {"resourceType": "Condition", "id": "no-date", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}]}
-{"resourceType": "Condition", "id": "multiple-categories", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}, {"coding": [{"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"}
-{"resourceType": "Condition", "id": "multiple-codings", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2022-11-28"}
+{"resourceType": "Condition", "id": "multiple-categories", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}, {"coding": [{"code": "health-concern", "system": "http://hl7.org/fhir/us/core/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"}
+{"resourceType": "Condition", "id": "multiple-codings", "category": [{"coding": [{"code": "16100001", "system": "http://snomed.info/sct"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2022-11-28"}
 {"resourceType": "Condition", "id": "unrelated-category", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "nope"}]}, {"coding": [{"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-10-28"}
 {"resourceType": "Condition", "id": "unrelated-coding", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "nope"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"}
 {"resourceType": "Condition", "id": "no-category", "recordedDate": "2023-10-03"}

tests/data/c_resource_count/general/diagnosticreport/0.ndjson

@@ -1,6 +1,6 @@
-{"resourceType": "DiagnosticReport", "id": "with-date", "category": [{"coding": [{"code": "CT", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}], "effectiveDateTime": "2023-10-04"}
+{"resourceType": "DiagnosticReport", "id": "with-date", "category": [{"coding": [{"code": "LAB", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}], "effectiveDateTime": "2023-10-04"}
 {"resourceType": "DiagnosticReport", "id": "with-issued", "issued": "2021-02-15", "status": "partial"}
-{"resourceType": "DiagnosticReport", "id": "no-date", "category": [{"coding": [{"code": "CT", "system": "http://loinc.org"}]}]}
-{"resourceType": "DiagnosticReport", "id": "unrelated-category", "category": [{"coding": [{"code": "1234", "system": "nope"}]}]}
+{"resourceType": "DiagnosticReport", "id": "no-date", "category": [{"coding": [{"code": "LP29684-5", "system": "http://loinc.org"}]}]}
+{"resourceType": "DiagnosticReport", "id": "unrelated-category", "category": [{"coding": [{"code": "12345-6", "system": "http://loinc.org"}, {"code": "CT", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}]}
 {"resourceType": "DiagnosticReport", "id": "no-category", "effectivePeriod": {"start": "2022-09-04"}}
 {"resourceType": "DiagnosticReport", "id": "nothing"}

tests/data/c_resource_count/general/expected_condition_month.csv

@@ -2,9 +2,9 @@ cnt,category,month,status
 2,encounter-diagnosis,2023-10,cumulus__none
 1,problem-list-item,2023-11,cumulus__none
 1,problem-list-item,2023-10,cumulus__none
-1,encounter-diagnosis; problem-list-item,2023-11,cumulus__none
-1,encounter-diagnosis; problem-list-item,2022-11,cumulus__none
+1,encounter-diagnosis; health-concern,2023-11,cumulus__none
 1,encounter-diagnosis,cumulus__none,cumulus__none
 1,encounter-diagnosis,2023-11,cumulus__none
 1,cumulus__none,cumulus__none,cumulus__none
 1,cumulus__none,2023-10,cumulus__none
+1,16100001; problem-list-item,2022-11,cumulus__none

tests/data/c_resource_count/general/expected_condition_year.csv

@@ -1,8 +1,8 @@
 cnt,category,year,status
 3,encounter-diagnosis,2023,cumulus__none
 2,problem-list-item,2023,cumulus__none
-1,encounter-diagnosis; problem-list-item,2023,cumulus__none
-1,encounter-diagnosis; problem-list-item,2022,cumulus__none
+1,encounter-diagnosis; health-concern,2023,cumulus__none
 1,encounter-diagnosis,cumulus__none,cumulus__none
 1,cumulus__none,cumulus__none,cumulus__none
 1,cumulus__none,2023,cumulus__none
+1,16100001; problem-list-item,2022,cumulus__none

tests/data/c_resource_count/general/expected_diagnosticreport_month.csv

@@ -2,5 +2,5 @@ cnt,category,month,status
 2,cumulus__none,cumulus__none,cumulus__none
 1,cumulus__none,2022-09,cumulus__none
 1,cumulus__none,2021-02,partial
-1,CT,cumulus__none,cumulus__none
-1,CT,2023-10,cumulus__none
+1,LP29684-5,cumulus__none,cumulus__none
+1,LAB,2023-10,cumulus__none

tests/data/c_resource_count/general/expected_diagnosticreport_year.csv

@@ -2,5 +2,5 @@ cnt,category,year,status
 2,cumulus__none,cumulus__none,cumulus__none
 1,cumulus__none,2022,cumulus__none
 1,cumulus__none,2021,partial
-1,CT,cumulus__none,cumulus__none
-1,CT,2023,cumulus__none
+1,LP29684-5,cumulus__none,cumulus__none
+1,LAB,2023,cumulus__none