Feature/upload error

.github/workflows/python.yml

@@ -18,14 +18,14 @@ jobs:
         python-version: [3.6]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: "Install Apache package"
+      - name: 'Install Apache package'
         run: sudo apt install -y apache2-dev
       - name: Install dependencies
         run: |

clarity_epp/upload/samples.py

@@ -89,31 +89,42 @@ def from_helix(lims, email_settings, input_file):
     sample_messages = {}
 
     # Parse samples
-    for line in input_file:
+    for line_index, line in enumerate(input_file):
         data = line.rstrip().strip('"').split('","')
 
         udf_data = {'Sample Type': 'DNA isolated', 'Dx Import warning': '', 'Dx Exoomequivalent': 1}  # required lims input
         for udf in udf_column:
             # Transform specific udf
-            if udf in ['Dx Overleden', 'Dx Spoed', 'Dx NICU Spoed']:
-                udf_data[udf] = clarity_epp.upload.utils.txt_to_bool(data[udf_column[udf]['index']])
-            elif udf in ['Dx Geslacht', 'Dx Foetus geslacht']:
-                udf_data[udf] = clarity_epp.upload.utils.transform_sex(data[udf_column[udf]['index']])
-            elif udf == 'Dx Foetus':
-                udf_data[udf] = bool(data[udf_column[udf]['index']].strip())
-            elif udf == 'Dx Concentratie (ng/ul)':
-                udf_data[udf] = data[udf_column[udf]['index']].replace(',', '.')
-                if udf_data[udf]:
-                    udf_data[udf] = float(udf_data[udf])
-            elif udf in ['Dx Monsternummer', 'Dx Fractienummer']:
-                udf_data[udf] = clarity_epp.upload.utils.transform_sample_name(data[udf_column[udf]['index']])
-            elif udf == 'Dx Gerelateerde onderzoeken':
-                udf_data[udf] = data[udf_column[udf]['index']].replace(',', ';')
-            elif udf == 'Dx Einddatum':
-                date = datetime.strptime(data[udf_column[udf]['index']], '%d-%m-%Y')  # Helix format (14-01-2021)
-                udf_data[udf] = date.strftime('%Y-%m-%d')  # LIMS format (2021-01-14)
-            else:
-                udf_data[udf] = data[udf_column[udf]['index']]
+            try:
+                if udf in ['Dx Overleden', 'Dx Spoed', 'Dx NICU Spoed']:
+                    udf_data[udf] = clarity_epp.upload.utils.txt_to_bool(data[udf_column[udf]['index']])
+                elif udf in ['Dx Geslacht', 'Dx Foetus geslacht']:
+                    udf_data[udf] = clarity_epp.upload.utils.transform_sex(data[udf_column[udf]['index']])
+                elif udf == 'Dx Foetus':
+                    udf_data[udf] = bool(data[udf_column[udf]['index']].strip())
+                elif udf == 'Dx Concentratie (ng/ul)':
+                    udf_data[udf] = data[udf_column[udf]['index']].replace(',', '.')
+                    if udf_data[udf]:
+                        udf_data[udf] = float(udf_data[udf])
+                elif udf in ['Dx Monsternummer', 'Dx Fractienummer']:
+                    udf_data[udf] = clarity_epp.upload.utils.transform_sample_name(data[udf_column[udf]['index']])
+                elif udf == 'Dx Gerelateerde onderzoeken':
+                    udf_data[udf] = data[udf_column[udf]['index']].replace(',', ';')
+                elif udf == 'Dx Einddatum':
+                    date = datetime.strptime(data[udf_column[udf]['index']], '%d-%m-%Y')  # Helix format (14-01-2021)
+                    udf_data[udf] = date.strftime('%Y-%m-%d')  # LIMS format (2021-01-14)
+                else:
+                    udf_data[udf] = data[udf_column[udf]['index']]
+            except (IndexError, ValueError):
+                # Catch parsing errors and send email
+                subject = "ERROR Lims Helix Upload: {0}".format(project_name)
+                message = (
+                    "Could not correctly parse data from helix export file (werklijst).\n"
+                    f"Row = {line_index+1} \t Column = {udf_column[udf]['column']} \t UDF = {udf}.\n"
+                    "Please check/update the file and try again. Make sure to remove the project from LIMS before retrying."
+                )
+                send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message)
+                sys.exit(message)
 
         sample_name = '{0}_{1}'.format(udf_data['Dx Monsternummer'], udf_data['Dx Meet ID'])
 

tests/data/WL24_01_HelixError.csv

@@ -0,0 +1,7 @@
+Familienummer,Persoons_id,Geslacht,Geboortejaar,Overleden,Foetus,Foet_id,Foetus_geslacht,Onderzoeknummer,Onderzoeksindicatie,Onderzoeksreden,Einddatum,Gerelateerde onderzoeken,Spoed,NICU Spoed,Monsternummer,Materiaal,Fractienummer,Concentratie (ng/ul),Opslagpositie,Werklijstnummer,Protocolcode,Protocolomschrijving,Stof_meet_id,Stoftestcode,Stoftestomschrijving,Gerelateerd aan,Aantal gerelateerde onderzoeken.
+"U000000","1","V","1988","N"," - 1","3","O","DN2024/1","VBE01",
+"Prenataal onderzoek","11-07-2024","DN2024/2, DN2024/3","J","N","2024D00001","VW","2024D00001_D01","131.25","AA1234BB: 1","WL24_01_HelixError","EX_EXO_004","Exoom.analy_IL_elidS34226467_Exoomver.","1","NGS_025","NGS_IL_WES","","2"
+"U000000","1","V","1988","N","","","","DN2024/2","VBE01","Informativiteitstest","11-07-2024","","J","J","2024D00002","BL","2024D00002_D01","311.7","AA1234BB: 2","WL24_01_HelixError","EX_EXO_004","Exoom.analy_IL_elidS34226467_Exoomver.","2","NGS_025","NGS_IL_WES","DN2024/1","0"
+"U000000","1","V","1988","N","","","","DN2024/2","VBE01","Informativiteitstest","11-07-2024","","J","J","2024D00003","BL","2024D00003_D01","343.9","AA1234BB: 3","WL24_01_HelixError","NGS_DUPLOCHECK","NGS duplocheck mengfractie","3","NGS_028","NGS_IL_WES_duplocheck","DN2024/1","0"
+"U000000","2","M","1988","N","","","","DN2024/3","VBE01","Informativiteitstest","11-07-2024","","J","N","2024D00004","BL","2024D00004_D01","196.7","AA1234BB: 4","WL24_01_HelixError","EX_EXO_004","Exoom.analy_IL_elidS34226467_Exoomver.","4","NGS_025","NGS_IL_WES","DN2024/1","0"
+"U000000","2","M","1988","N","","","","DN2024/3","VBE01","Informativiteitstest","11-07-2024","","J","N","2024D00005","BL","2024D00005_D01","215.2","AA1234BB: 5","WL24_01_HelixError","NGS_DUPLOCHECK","NGS duplocheck mengfractie","5","NGS_028","NGS_IL_WES_duplocheck","DN2024/1","0"

tests/test_upload_sample.py

@@ -0,0 +1,55 @@
+import genologics
+import pytest
+
+from clarity_epp.upload import samples
+
+
+class ResearcherMock:
+    def __init__(self, fax, email):
+        self.fax = fax
+        self.email = email
+
+
+class ProjectMock:
+    def __init__(self, name, researcher):
+        self.name = name
+        self.researcher = researcher
+
+
+class ContainertypeMock:
+    def __init__(self, name):
+        self.name = name
+
+
+def test_from_helix_file_error(mocker):
+    mocker.patch('genologics.lims.Lims.check_version', return_value=None)
+
+    researchers_mock = [ResearcherMock('HelixError', '[email protected]')]
+    mocker.patch('genologics.lims.Lims.get_researchers', return_value=researchers_mock)
+
+    mocker.patch('genologics.lims.Lims.get_projects', return_value=[])
+
+    project_mock = ProjectMock('TEST', researchers_mock[0])
+    mocker.patch('genologics.entities.Project.create', return_value=project_mock)
+
+    container_type_mock = ContainertypeMock('Tube')
+    mocker.patch('genologics.entities.Containertype.__new__', return_value=[container_type_mock])
+
+    mocker.patch('clarity_epp.upload.samples.send_email', return_value=None)
+    email_settings = {
+        'server': 'smtp.test.nl',
+        'from': '[email protected]',
+        'to_import_helix': []
+
+    }
+
+    with pytest.raises(SystemExit) as exception_info:
+        with open('tests/data/WL24_01_HelixError.csv', 'r') as helix_file:
+            samples.from_helix(genologics.lims.Lims, email_settings, helix_file)
+
+    assert exception_info.type == SystemExit
+    assert exception_info.value.code == (
+        "Could not correctly parse data from helix export file (werklijst).\n"
+        "Row = 1 \t Column = Fractienummer \t UDF = Dx Fractienummer.\n"
+        "Please check/update the file and try again. Make sure to remove the project from LIMS before retrying."
+    )