Merge pull request #362 from smart-on-fhir/mikix/tweaks

A couple small improvements

cumulus_etl/deid/scrubber.py

@@ -316,13 +316,19 @@ def _check_extensions(
                 "http://open.epic.com/FHIR/R4/StructureDefinition/patient-preferred-provider-sex",
                 # A Netherlands extension used by Epic
                 "http://nictiz.nl/fhir/StructureDefinition/BodySite-Qualifier",
+                ### Synthea extensions
+                "http://synthetichealth.github.io/synthea/disability-adjusted-life-years",
+                "http://synthetichealth.github.io/synthea/quality-adjusted-life-years",
             }
             # Some extensions we know about, but aren't necessary to us (they duplicate standard
             # extensions, contain PHI, or are otherwise not relevant). We don't want to warn
             # the user about them as "unrecognized", so we just ignore them entirely.
             ignored_extensions = {
-                # Base spec extension holding a phone number
+                # Base spec extensions with PHI
+                "http://hl7.org/fhir/StructureDefinition/geolocation",
                 "http://hl7.org/fhir/StructureDefinition/iso21090-TEL-address",
+                "http://hl7.org/fhir/StructureDefinition/patient-birthPlace",
+                "http://hl7.org/fhir/StructureDefinition/patient-mothersMaidenName"
                 # Usually harmless, but we ignore it to avoid accidentally leaving in the
                 # rendered value of a PHI element that we removed or didn't allow-list.
                 "http://hl7.org/fhir/StructureDefinition/rendered-value",

cumulus_etl/etl/tasks/base.py

@@ -276,7 +276,9 @@ def _update_completion_table(self) -> None:
 
         # Write it out
         formatter = self.task_config.create_formatter(**completion.completion_format_args())
-        formatter.write_records(batch)
+        if not formatter.write_records(batch):
+            # Completion crosses output table boundaries, so just mark the first one as failed
+            self.summaries[0].had_errors = True
         formatter.finalize()
 
     def _get_formatter(self, table_index: int) -> formats.Format:

tests/etl/test_tasks.py

@@ -317,6 +317,22 @@ async def test_allow_empty_group(self):
             comp_format.write_records.call_args[0][0].rows,
         )
 
+    async def test_error_during_write(self):
+        """We should flag the task as failed if we can't write completion"""
+
+        # Change the way we mock formatters to insert an error
+        def make_formatter(dbname: str, **kwargs):
+            mock_formatter = mock.MagicMock(dbname=dbname, **kwargs)
+            if dbname == "etl__completion":
+                mock_formatter.write_records.return_value = False
+            return mock_formatter
+
+        self.job_config.create_formatter = mock.MagicMock(side_effect=make_formatter)
+
+        self.make_json("Device", "A")
+        summaries = await basic_tasks.DeviceTask(self.job_config, self.scrubber).run()
+        self.assertTrue(summaries[0].had_errors)
+
 
 @ddt.ddt
 class TestMedicationRequestTask(TaskTestCase):