ci: add full stack NLP regression test

This is overdue. We want to make sure that any NLP changes are not
suprises.

.github/workflows/ci.yaml

@@ -14,78 +14,116 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  unittest:
-    name: unit tests
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        # while we are still private, don't go crazy with the Python versions as they eat up CI minutes
-        python-version: ["3.10"]
+#  unittest:
+#    name: unit tests
+#    runs-on: ubuntu-22.04
+#    strategy:
+#      matrix:
+#        # while we are still private, don't go crazy with the Python versions as they eat up CI minutes
+#        python-version: ["3.10"]
+#
+#    steps:
+#      - uses: actions/checkout@v4
+#
+#      - name: Set up Python ${{ matrix.python-version }}
+#        uses: actions/setup-python@v4
+#        with:
+#          python-version: ${{ matrix.python-version }}
+#
+#      - name: Install dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install pytest
+#          pip install .[tests]
+#
+#      - name: Check out MS tool
+#        uses: actions/checkout@v4
+#        with:
+#          repository: microsoft/Tools-for-Health-Data-Anonymization
+#          path: mstool
+#
+#      - name: Build MS tool
+#        run: |
+#          sudo apt-get update
+#          sudo apt-get install dotnet6
+#          dotnet publish \
+#            --runtime=linux-x64 \
+#            --configuration=Release \
+#            -p:PublishSingleFile=true \
+#            --output=$HOME/.local/bin \
+#            mstool/FHIR/src/Microsoft.Health.Fhir.Anonymizer.R4.CommandLineTool
+#
+#      - name: Test with pytest
+#        run: |
+#          python -m pytest
 
+  nlp-regression:
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
+      - name: Install Docker
+        uses: docker/setup-buildx-action@v3
 
-      - name: Install dependencies
+      - name: Install Docker images
         run: |
-          python -m pip install --upgrade pip
-          pip install pytest
-          pip install .[tests]
+          export UMLS_API_KEY=${{ secrets.UMLS_API_KEY }}
+          docker compose --profile covid-symptom up -d --quiet-pull
 
-      - name: Check out MS tool
-        uses: actions/checkout@v3
+      - name: Build ETL image
+        uses: docker/build-push-action@v5
         with:
-          repository: microsoft/Tools-for-Health-Data-Anonymization
-          path: mstool
-
-      - name: Build MS tool
-        run: |
-          sudo apt-get update
-          sudo apt-get install dotnet6
-          dotnet publish \
-            --runtime=linux-x64 \
-            --configuration=Release \
-            -p:PublishSingleFile=true \
-            --output=$HOME/.local/bin \
-            mstool/FHIR/src/Microsoft.Health.Fhir.Anonymizer.R4.CommandLineTool
-
-      - name: Test with pytest
-        run: |
-          python -m pytest
-
-  lint:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install linters
-        # black is synced with the .pre-commit-hooks version
-        run: |
-          python -m pip install --upgrade pip
-          pip install bandit[toml] pycodestyle pylint black==23.11.0
+          push: false  # this is the default, but just for clarity: we are only building here
+          tags: smartonfhir/cumulus-etl:latest
 
-      - name: Run pycodestyle
-        # E203: pycodestyle is a little too rigid about slices & whitespace
-        #  See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#slices
-        # W503: a default ignore that we are restoring
+      - name: Run NLP
         run: |
-          pycodestyle --max-line-length=120 --ignore=E203,W503 .
+          export UMLS_API_KEY=${{ secrets.UMLS_API_KEY }}  # just to quiet warnings about it
+          export DATADIR=$(realpath tests/data/nlp-regression)
 
-      - name: Run pylint
-        if: success() || failure() # still run pylint if above checks fail
-        run: |
-          pylint cumulus_etl tests
+          # Run the NLP task
+          docker compose run --rm \
+            --volume $DATADIR:/in \
+            cumulus-etl \
+            /in/input \
+            /in/run-output \
+            /in/phi \
+            --output-format=ndjson \
+            --task covid_symptom__nlp_results
 
-      - name: Run bandit
-        if: success() || failure() # still run bandit if above checks fail
-        run: |
-          bandit -c pyproject.toml -r .
+          # Compare results
+          diff -upr $DATADIR/expected-output/covid_symptom__nlp_results \
+                    $DATADIR/run-output/covid_symptom__nlp_results
 
-      - name: Run black
-        if: success() || failure() # still run black if above checks fails
-        run: |
-          black --check --verbose --line-length 120 .
+#  lint:
+#    runs-on: ubuntu-22.04
+#    steps:
+#      - uses: actions/checkout@v4
+#
+#      - name: Install linters
+#        # black is synced with the .pre-commit-hooks version
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install bandit[toml] pycodestyle pylint black==23.11.0
+#
+#      - name: Run pycodestyle
+#        # E203: pycodestyle is a little too rigid about slices & whitespace
+#        #  See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#slices
+#        # W503: a default ignore that we are restoring
+#        run: |
+#          pycodestyle --max-line-length=120 --ignore=E203,W503 .
+#
+#      - name: Run pylint
+#        if: success() || failure() # still run pylint if above checks fail
+#        run: |
+#          pylint cumulus_etl tests
+#
+#      - name: Run bandit
+#        if: success() || failure() # still run bandit if above checks fail
+#        run: |
+#          bandit -c pyproject.toml -r .
+#
+#      - name: Run black
+#        if: success() || failure() # still run black if above checks fails
+#        run: |
+#          black --check --verbose --line-length 120 .

.github/workflows/docker-hub.yaml

@@ -13,23 +13,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Get Docker metadata
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           flavor: latest=true
           images: smartonfhir/cumulus-etl
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push image to Docker Hub
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
           platforms: |

compose.yaml

@@ -39,7 +39,7 @@ services:
   ctakes-covid-base:
     image: smartonfhir/ctakes-covid:1.1.0
     environment:
-      - ctakes_umlsuser=umls_api_key 
+      - ctakes_umlsuser=umls_api_key
       - ctakes_umlspw=$UMLS_API_KEY
     networks:
       - cumulus-etl

tests/data/nlp-regression/input/DocumentReference.ndjson

@@ -0,0 +1,4 @@
+{"resourceType":"DocumentReference","id":"0070382a-10ea-f48d-c8d5-96ac4d72313a","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"identifier":[{"system":"urn:ietf:rfc:3986","value":"urn:uuid:3815acef-2612-9459-085d-3dd6bbcba105"}],"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34111-5","display":"Emergency department note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation + Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/7bc002fa-dc52-17d6-1563-fd8901826f7d"},"date":"2006-07-21T11:46:59.829-04:00","author":[{"reference":"Practitioner?identifier=http://hl7.org/fhir/sid/us-npi|9999925990","display":"Dr. Leana211 Wehner319"}],"custodian":{"reference":"Organization?identifier=https://github.com/synthetichealth/synthea|6d897d1c-a732-346f-991e-6e1a5b3d5af1","display":"HEALTH AND WELLNESS CLINIC, LLC"},"content":[{"attachment":{"contentType":"text/plain; charset=utf-8","data":"CjIwMDYtMDctMjEKCiMgQ2hpZWYgQ29tcGxhaW50Ck5vIGNvbXBsYWludHMuCgojIEhpc3Rvcnkgb2YgUHJlc2VudCBJbGxuZXNzCkFuMTI1IFN1YW5uZTg1OAogaXMgYSAyOCB5ZWFyLW9sZCBub25oaXNwYW5pYyB3aGl0ZSBmZW1hbGUuIFBhdGllbnQgaGFzIGEgaGlzdG9yeSBvZiBmdWxsLXRpbWUgZW1wbG95bWVudCAoZmluZGluZyksIHN0cmVzcyAoZmluZGluZykuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgbWFycmllZC4gUGF0aWVudCBoYXMgbmV2ZXIgc21va2VkLgogUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuCiBQYXRpZW50IGhhcyBjb21wbGV0ZWQgc29tZSBjb2xsZWdlIGNvdXJzZXMuClBhdGllbnQgY3VycmVudGx5IGhhcyBBZXRuYS4KCiMgQWxsZXJnaWVzCk5vIEtub3duIEFsbGVyZ2llcy4KCiMgTWVkaWNhdGlvbnMKTm8gQWN0aXZlIE1lZGljYXRpb25zLgoKIyBBc3Nlc3NtZW50IGFuZCBQbGFuClBhdGllbnQgaXMgcHJlc2VudGluZyB3aXRoIHNvY2lhbCBpc29sYXRpb24gKGZpbmRpbmcpLiAKCiMjIFBsYW4KCg=="},"format":{"system":"http://ihe.net/fhir/ihe.formatcode.fhir/CodeSystem/formatcode","code":"urn:ihe:iti:xds:2017:mimeTypeSufficient","display":"mimeType Sufficient"}}],"context":{"encounter":[{"reference":"Encounter/dafdfd49-5ffe-56c9-af0a-e533e7663d59"}],"period":{"start":"2006-07-21T11:46:59-04:00","end":"2006-07-21T12:21:56-04:00"}}}
+{"resourceType":"DocumentReference","id":"b59e6e43-14a6-cce1-438d-56ac70b33f33","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"identifier":[{"system":"urn:ietf:rfc:3986","value":"urn:uuid:f59d2041-c153-5563-02c8-a1d00b851340"}],"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34111-5","display":"Emergency department note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation + Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/79a66c97-6131-3213-f3c9-4606946ab056"},"date":"1985-09-18T02:12:16.824-04:00","author":[{"reference":"Practitioner?identifier=http://hl7.org/fhir/sid/us-npi|9999974493","display":"Dr. Chelsey293 Simonis280"}],"custodian":{"reference":"Organization?identifier=https://github.com/synthetichealth/synthea|a261e1fc-9361-3633-a2c4-8569a04b818d","display":"NEWMAN MEMORIAL COUNTY HOSPITAL"},"content":[{"attachment":{"contentType":"text/plain; charset=utf-8","data":"CjE5ODUtMDktMTgKCiMgQ2hpZWYgQ29tcGxhaW50Ci0gSHVuZ2VyCi0gVGhpcnN0Ci0gRnJlcXVlbnQgVXJpbmF0aW9uCgoKIyBIaXN0b3J5IG9mIFByZXNlbnQgSWxsbmVzcwpNYXJpbmU1NDIgQWkxMjAKIGlzIGEgNTggeWVhci1vbGQgbm9uaGlzcGFuaWMgd2hpdGUgZmVtYWxlLiBQYXRpZW50IGhhcyBhIGhpc3Rvcnkgb2Ygc2V2ZXJlIGFueGlldHkgKHBhbmljKSAoZmluZGluZyksIGNocm9uaWMga2lkbmV5IGRpc2Vhc2Ugc3RhZ2UgMyAoZGlzb3JkZXIpLCBjaHJvbmljIGtpZG5leSBkaXNlYXNlIHN0YWdlIDIgKGRpc29yZGVyKSwgbGltaXRlZCBzb2NpYWwgY29udGFjdCAoZmluZGluZyksIG5vdCBpbiBsYWJvciBmb3JjZSAoZmluZGluZyksIGNocm9uaWMga2lkbmV5IGRpc2Vhc2Ugc3RhZ2UgMSAoZGlzb3JkZXIpLCB1bmhlYWx0aHkgYWxjb2hvbCBkcmlua2luZyBiZWhhdmlvciAoZmluZGluZyksIHJlcG9ydHMgb2YgdmlvbGVuY2UgaW4gdGhlIGVudmlyb25tZW50IChmaW5kaW5nKSwgZnVsbC10aW1lIGVtcGxveW1lbnQgKGZpbmRpbmcpLCB2aWN0aW0gb2YgaW50aW1hdGUgcGFydG5lciBhYnVzZSAoZmluZGluZyksIHN0cmVzcyAoZmluZGluZykuCgojIFNvY2lhbCBIaXN0b3J5ClBhdGllbnQgaXMgbWFycmllZC4gUGF0aWVudCBoYXMgbmV2ZXIgc21va2VkLgogUGF0aWVudCBpZGVudGlmaWVzIGFzIGhldGVyb3NleHVhbC4KClBhdGllbnQgY29tZXMgZnJvbSBhIG1pZGRsZSBzb2Npb2Vjb25vbWljIGJhY2tncm91bmQuCiBQYXRpZW50IGhhcyBjb21wbGV0ZWQgc29tZSBjb2xsZWdlIGNvdXJzZXMuClBhdGllbnQgY3VycmVudGx5IGhhcyBOTyBJTlNVUkFOQ0UuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmxpc2lub3ByaWwgMTAgbWcgb3JhbCB0YWJsZXQ7IDEgbWwgZXBvZXRpbiBhbGZhIDQwMDAgdW50L21sIGluamVjdGlvbiBbZXBvZ2VuXTsgMjQgaHIgbWV0Zm9ybWluIGh5ZHJvY2hsb3JpZGUgNTAwIG1nIGV4dGVuZGVkIHJlbGVhc2Ugb3JhbCB0YWJsZXQ7IGluc3VsaW4gaXNvcGhhbmUsIGh1bWFuIDcwIHVudC9tbCAvIGluc3VsaW4sIHJlZ3VsYXIsIGh1bWFuIDMwIHVudC9tbCBpbmplY3RhYmxlIHN1c3BlbnNpb24gW2h1bXVsaW5dOyBtZXBlcmlkaW5lIGh5ZHJvY2hsb3JpZGUgNTAgbWcgb3JhbCB0YWJsZXQKCiMgQXNzZXNzbWVudCBhbmQgUGxhbgoKCiMjIFBsYW4KClRoZSBmb2xsb3dpbmcgcHJvY2VkdXJlcyB3ZXJlIGNvbmR1Y3RlZDoKLSByZW5hbCBkaWFseXNpcyAocHJvY2VkdXJlKQpUaGUgcGF0aWVudCB3YXMgcHJlc2NyaWJlZCB0aGUgZm9sbG93aW5nIG1lZGljYXRpb25zOgotIDEgbWwgZXBvZXRpbiBhbGZhIDQwMDAgdW50L21sIGluamVjdGlvbiBbZXBvZ2VuXQo="},"format":{"system":"http://ihe.net/fhir/ihe.formatcode.fhir/CodeSystem/formatcode","code":"urn:ihe:iti:xds:2017:mimeTypeSufficient","display":"mimeType Sufficient"}}],"context":{"encounter":[{"reference":"Encounter/4531fe24-4355-d727-8c9d-30640cb47437"}],"period":{"start":"1985-09-18T02:12:16-04:00","end":"1985-09-18T04:18:16-04:00"}}}
+{"resourceType":"DocumentReference","id":"d2ecb6cc-e30e-a690-079a-bd060bd59e05","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"identifier":[{"system":"urn:ietf:rfc:3986","value":"urn:uuid:43aef025-9ced-b263-0ad4-09048b744824"}],"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34111-5","display":"Emergency department note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation + Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/79a66c97-6131-3213-f3c9-4606946ab056"},"date":"1983-06-11T23:58:16.824-04:00","author":[{"reference":"Practitioner?identifier=http://hl7.org/fhir/sid/us-npi|9999974394","display":"Dr. Olevia458 Hermiston71"}],"custodian":{"reference":"Organization?identifier=https://github.com/synthetichealth/synthea|61e67719-63e4-318e-91ab-c834166b4680","display":"NEWMAN MEMORIAL COUNTY HOSPITAL"},"content":[{"attachment":{"contentType":"text/plain; charset=utf-8","data":"CjE5ODMtMDYtMTEKCiMgQ2hpZWYgQ29tcGxhaW50Ci0gSHVuZ2VyCi0gVGhpcnN0Ci0gRnJlcXVlbnQgVXJpbmF0aW9uCgoKIyBIaXN0b3J5IG9mIFByZXNlbnQgSWxsbmVzcwpNYXJpbmU1NDIgQWkxMjAKIGlzIGEgNTYgeWVhci1vbGQgbm9uaGlzcGFuaWMgd2hpdGUgZmVtYWxlLiBQYXRpZW50IGhhcyBhIGhpc3Rvcnkgb2YgY2hyb25pYyBraWRuZXkgZGlzZWFzZSBzdGFnZSAzIChkaXNvcmRlciksIGNocm9uaWMga2lkbmV5IGRpc2Vhc2Ugc3RhZ2UgMiAoZGlzb3JkZXIpLCBsaW1pdGVkIHNvY2lhbCBjb250YWN0IChmaW5kaW5nKSwgY2hyb25pYyBraWRuZXkgZGlzZWFzZSBzdGFnZSAxIChkaXNvcmRlciksIGZ1bGwtdGltZSBlbXBsb3ltZW50IChmaW5kaW5nKSwgdmljdGltIG9mIGludGltYXRlIHBhcnRuZXIgYWJ1c2UgKGZpbmRpbmcpLCBzdHJlc3MgKGZpbmRpbmcpLgoKIyBTb2NpYWwgSGlzdG9yeQpQYXRpZW50IGlzIG1hcnJpZWQuIFBhdGllbnQgaGFzIG5ldmVyIHNtb2tlZC4KIFBhdGllbnQgaWRlbnRpZmllcyBhcyBoZXRlcm9zZXh1YWwuCgpQYXRpZW50IGNvbWVzIGZyb20gYSBtaWRkbGUgc29jaW9lY29ub21pYyBiYWNrZ3JvdW5kLgogUGF0aWVudCBoYXMgY29tcGxldGVkIHNvbWUgY29sbGVnZSBjb3Vyc2VzLgpQYXRpZW50IGN1cnJlbnRseSBoYXMgTWVkaWNhaWQuCgojIEFsbGVyZ2llcwpObyBLbm93biBBbGxlcmdpZXMuCgojIE1lZGljYXRpb25zCmxpc2lub3ByaWwgMTAgbWcgb3JhbCB0YWJsZXQ7IDI0IGhyIG1ldGZvcm1pbiBoeWRyb2NobG9yaWRlIDUwMCBtZyBleHRlbmRlZCByZWxlYXNlIG9yYWwgdGFibGV0OyBpbnN1bGluIGlzb3BoYW5lLCBodW1hbiA3MCB1bnQvbWwgLyBpbnN1bGluLCByZWd1bGFyLCBodW1hbiAzMCB1bnQvbWwgaW5qZWN0YWJsZSBzdXNwZW5zaW9uIFtodW11bGluXTsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KUGF0aWVudCBpcyBwcmVzZW50aW5nIHdpdGggZnVsbC10aW1lIGVtcGxveW1lbnQgKGZpbmRpbmcpLiAKCiMjIFBsYW4KClRoZSBwYXRpZW50IHdhcyBwcmVzY3JpYmVkIHRoZSBmb2xsb3dpbmcgbWVkaWNhdGlvbnM6Ci0gaW5zdWxpbiBpc29waGFuZSwgaHVtYW4gNzAgdW50L21sIC8gaW5zdWxpbiwgcmVndWxhciwgaHVtYW4gMzAgdW50L21sIGluamVjdGFibGUgc3VzcGVuc2lvbiBbaHVtdWxpbl0KLSAyNCBociBtZXRmb3JtaW4gaHlkcm9jaGxvcmlkZSA1MDAgbWcgZXh0ZW5kZWQgcmVsZWFzZSBvcmFsIHRhYmxldAotIGxpc2lub3ByaWwgMTAgbWcgb3JhbCB0YWJsZXQK"},"format":{"system":"http://ihe.net/fhir/ihe.formatcode.fhir/CodeSystem/formatcode","code":"urn:ihe:iti:xds:2017:mimeTypeSufficient","display":"mimeType Sufficient"}}],"context":{"encounter":[{"reference":"Encounter/9318db9c-692c-b351-d472-8be543cfeb8c"}],"period":{"start":"1983-06-11T23:58:16-04:00","end":"1983-06-12T00:38:22-04:00"}}}
+{"resourceType":"DocumentReference","id":"fefe58f2-fc42-56ea-4536-1494a1b288b4","meta":{"profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference"]},"identifier":[{"system":"urn:ietf:rfc:3986","value":"urn:uuid:713c7b85-55f4-5251-e571-298b8f1d1948"}],"status":"superseded","type":{"coding":[{"system":"http://loinc.org","code":"34111-5","display":"Emergency department note"},{"system":"http://loinc.org","code":"51847-2","display":"Evaluation + Plan note"}]},"category":[{"coding":[{"system":"http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category","code":"clinical-note","display":"Clinical Note"}]}],"subject":{"reference":"Patient/79a66c97-6131-3213-f3c9-4606946ab056"},"date":"1988-09-06T13:27:16.824-04:00","author":[{"reference":"Practitioner?identifier=http://hl7.org/fhir/sid/us-npi|9999974493","display":"Dr. Chelsey293 Simonis280"}],"custodian":{"reference":"Organization?identifier=https://github.com/synthetichealth/synthea|a261e1fc-9361-3633-a2c4-8569a04b818d","display":"NEWMAN MEMORIAL COUNTY HOSPITAL"},"content":[{"attachment":{"contentType":"text/plain; charset=utf-8","data":"CjE5ODgtMDktMDYKCiMgQ2hpZWYgQ29tcGxhaW50Ci0gSHVuZ2VyCi0gVGhpcnN0Ci0gRnJlcXVlbnQgVXJpbmF0aW9uCgoKIyBIaXN0b3J5IG9mIFByZXNlbnQgSWxsbmVzcwpNYXJpbmU1NDIgQWkxMjAKIGlzIGEgNjEgeWVhci1vbGQgbm9uaGlzcGFuaWMgd2hpdGUgZmVtYWxlLiBQYXRpZW50IGhhcyBhIGhpc3Rvcnkgb2Ygc2V2ZXJlIGFueGlldHkgKHBhbmljKSAoZmluZGluZyksIGNocm9uaWMga2lkbmV5IGRpc2Vhc2Ugc3RhZ2UgMyAoZGlzb3JkZXIpLCB2aWN0aW0gb2YgaW50aW1hdGUgcGFydG5lciBhYnVzZSAoZmluZGluZyksIHBhcnQtdGltZSBlbXBsb3ltZW50IChmaW5kaW5nKSwgY2hyb25pYyBraWRuZXkgZGlzZWFzZSBzdGFnZSAyIChkaXNvcmRlciksIGxpbWl0ZWQgc29jaWFsIGNvbnRhY3QgKGZpbmRpbmcpLCBub3QgaW4gbGFib3IgZm9yY2UgKGZpbmRpbmcpLCBjaHJvbmljIGtpZG5leSBkaXNlYXNlIHN0YWdlIDEgKGRpc29yZGVyKSwgdW5oZWFsdGh5IGFsY29ob2wgZHJpbmtpbmcgYmVoYXZpb3IgKGZpbmRpbmcpLCByZXBvcnRzIG9mIHZpb2xlbmNlIGluIHRoZSBlbnZpcm9ubWVudCAoZmluZGluZyksIGZ1bGwtdGltZSBlbXBsb3ltZW50IChmaW5kaW5nKSwgYXdhaXRpbmcgdHJhbnNwbGFudGF0aW9uIG9mIGtpZG5leSAoc2l0dWF0aW9uKSwgZXNjaGVyaWNoaWEgY29saSB1cmluYXJ5IHRyYWN0IGluZmVjdGlvbiwgc3RyZXNzIChmaW5kaW5nKS4KCiMgU29jaWFsIEhpc3RvcnkKUGF0aWVudCBpcyBtYXJyaWVkLiBQYXRpZW50IGhhcyBuZXZlciBzbW9rZWQuCiBQYXRpZW50IGlkZW50aWZpZXMgYXMgaGV0ZXJvc2V4dWFsLgoKUGF0aWVudCBjb21lcyBmcm9tIGEgbWlkZGxlIHNvY2lvZWNvbm9taWMgYmFja2dyb3VuZC4KIFBhdGllbnQgaGFzIGNvbXBsZXRlZCBzb21lIGNvbGxlZ2UgY291cnNlcy4KUGF0aWVudCBjdXJyZW50bHkgaGFzIE5PIElOU1VSQU5DRS4KCiMgQWxsZXJnaWVzCk5vIEtub3duIEFsbGVyZ2llcy4KCiMgTWVkaWNhdGlvbnMKbGlzaW5vcHJpbCAxMCBtZyBvcmFsIHRhYmxldDsgMSBtbCBlcG9ldGluIGFsZmEgNDAwMCB1bnQvbWwgaW5qZWN0aW9uIFtlcG9nZW5dOyAyNCBociBtZXRmb3JtaW4gaHlkcm9jaGxvcmlkZSA1MDAgbWcgZXh0ZW5kZWQgcmVsZWFzZSBvcmFsIHRhYmxldDsgbml0cm9mdXJhbnRvaW4gNSBtZy9tbCBvcmFsIHN1c3BlbnNpb247IHBoZW5hem9weXJpZGluZSBoeWRyb2NobG9yaWRlIDEwMCBtZyBvcmFsIHRhYmxldDsgaW5zdWxpbiBpc29waGFuZSwgaHVtYW4gNzAgdW50L21sIC8gaW5zdWxpbiwgcmVndWxhciwgaHVtYW4gMzAgdW50L21sIGluamVjdGFibGUgc3VzcGVuc2lvbiBbaHVtdWxpbl07IG5pdHJvZnVyYW50b2luLCBtYWNyb2NyeXN0YWxzIDUwIG1nIG9yYWwgY2Fwc3VsZTsgbWVwZXJpZGluZSBoeWRyb2NobG9yaWRlIDUwIG1nIG9yYWwgdGFibGV0CgojIEFzc2Vzc21lbnQgYW5kIFBsYW4KCgojIyBQbGFuCgpUaGUgZm9sbG93aW5nIHByb2NlZHVyZXMgd2VyZSBjb25kdWN0ZWQ6Ci0gcmVuYWwgZGlhbHlzaXMgKHByb2NlZHVyZSkKVGhlIHBhdGllbnQgd2FzIHByZXNjcmliZWQgdGhlIGZvbGxvd2luZyBtZWRpY2F0aW9uczoKLSAxIG1sIGVwb2V0aW4gYWxmYSA0MDAwIHVudC9tbCBpbmplY3Rpb24gW2Vwb2dlbl0K"},"format":{"system":"http://ihe.net/fhir/ihe.formatcode.fhir/CodeSystem/formatcode","code":"urn:ihe:iti:xds:2017:mimeTypeSufficient","display":"mimeType Sufficient"}}],"context":{"encounter":[{"reference":"Encounter/b2ea9444-9fcf-0fd0-d9cd-6f45028e9af9"}],"period":{"start":"1988-09-06T13:27:16-04:00","end":"1988-09-06T16:18:16-04:00"}}}

tests/data/nlp-regression/phi/codebook.json

@@ -0,0 +1 @@
+{"version": 1, "id_salt": "49205a25a9604006b4ad28016e559e1bd0417f2a915b04013fce03da46850320"}