diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 5f7fcff..c2d4858 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -18,14 +18,14 @@ jobs: python-version: [3.6] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: "Install Apache package" + - name: 'Install Apache package' run: sudo apt install -y apache2-dev - name: Install dependencies run: | diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index cba9527..daa6584 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -89,31 +89,42 @@ def from_helix(lims, email_settings, input_file): sample_messages = {} # Parse samples - for line in input_file: + for line_index, line in enumerate(input_file): data = line.rstrip().strip('"').split('","') udf_data = {'Sample Type': 'DNA isolated', 'Dx Import warning': '', 'Dx Exoomequivalent': 1} # required lims input for udf in udf_column: # Transform specific udf - if udf in ['Dx Overleden', 'Dx Spoed', 'Dx NICU Spoed']: - udf_data[udf] = clarity_epp.upload.utils.txt_to_bool(data[udf_column[udf]['index']]) - elif udf in ['Dx Geslacht', 'Dx Foetus geslacht']: - udf_data[udf] = clarity_epp.upload.utils.transform_sex(data[udf_column[udf]['index']]) - elif udf == 'Dx Foetus': - udf_data[udf] = bool(data[udf_column[udf]['index']].strip()) - elif udf == 'Dx Concentratie (ng/ul)': - udf_data[udf] = data[udf_column[udf]['index']].replace(',', '.') - if udf_data[udf]: - udf_data[udf] = float(udf_data[udf]) - elif udf in ['Dx Monsternummer', 'Dx Fractienummer']: - udf_data[udf] = clarity_epp.upload.utils.transform_sample_name(data[udf_column[udf]['index']]) - elif udf == 'Dx Gerelateerde onderzoeken': - udf_data[udf] = data[udf_column[udf]['index']].replace(',', ';') - elif udf == 'Dx Einddatum': - date = datetime.strptime(data[udf_column[udf]['index']], '%d-%m-%Y') # Helix format (14-01-2021) - udf_data[udf] = date.strftime('%Y-%m-%d') # LIMS format (2021-01-14) - else: - udf_data[udf] = data[udf_column[udf]['index']] + try: + if udf in ['Dx Overleden', 'Dx Spoed', 'Dx NICU Spoed']: + udf_data[udf] = clarity_epp.upload.utils.txt_to_bool(data[udf_column[udf]['index']]) + elif udf in ['Dx Geslacht', 'Dx Foetus geslacht']: + udf_data[udf] = clarity_epp.upload.utils.transform_sex(data[udf_column[udf]['index']]) + elif udf == 'Dx Foetus': + udf_data[udf] = bool(data[udf_column[udf]['index']].strip()) + elif udf == 'Dx Concentratie (ng/ul)': + udf_data[udf] = data[udf_column[udf]['index']].replace(',', '.') + if udf_data[udf]: + udf_data[udf] = float(udf_data[udf]) + elif udf in ['Dx Monsternummer', 'Dx Fractienummer']: + udf_data[udf] = clarity_epp.upload.utils.transform_sample_name(data[udf_column[udf]['index']]) + elif udf == 'Dx Gerelateerde onderzoeken': + udf_data[udf] = data[udf_column[udf]['index']].replace(',', ';') + elif udf == 'Dx Einddatum': + date = datetime.strptime(data[udf_column[udf]['index']], '%d-%m-%Y') # Helix format (14-01-2021) + udf_data[udf] = date.strftime('%Y-%m-%d') # LIMS format (2021-01-14) + else: + udf_data[udf] = data[udf_column[udf]['index']] + except (IndexError, ValueError): + # Catch parsing errors and send email + subject = "ERROR Lims Helix Upload: {0}".format(project_name) + message = ( + "Could not correctly parse data from helix export file (werklijst).\n" + f"Row = {line_index+1} \t Column = {udf_column[udf]['column']} \t UDF = {udf}.\n" + "Please check/update the file and try again. Make sure to remove the project from LIMS before retrying." + ) + send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) + sys.exit(message) sample_name = '{0}_{1}'.format(udf_data['Dx Monsternummer'], udf_data['Dx Meet ID']) diff --git a/tests/data/WL24_01_HelixError.csv b/tests/data/WL24_01_HelixError.csv new file mode 100755 index 0000000..d7f3453 --- /dev/null +++ b/tests/data/WL24_01_HelixError.csv @@ -0,0 +1,7 @@ +Familienummer,Persoons_id,Geslacht,Geboortejaar,Overleden,Foetus,Foet_id,Foetus_geslacht,Onderzoeknummer,Onderzoeksindicatie,Onderzoeksreden,Einddatum,Gerelateerde onderzoeken,Spoed,NICU Spoed,Monsternummer,Materiaal,Fractienummer,Concentratie (ng/ul),Opslagpositie,Werklijstnummer,Protocolcode,Protocolomschrijving,Stof_meet_id,Stoftestcode,Stoftestomschrijving,Gerelateerd aan,Aantal gerelateerde onderzoeken. +"U000000","1","V","1988","N"," - 1","3","O","DN2024/1","VBE01", +"Prenataal onderzoek","11-07-2024","DN2024/2, DN2024/3","J","N","2024D00001","VW","2024D00001_D01","131.25","AA1234BB: 1","WL24_01_HelixError","EX_EXO_004","Exoom.analy_IL_elidS34226467_Exoomver.","1","NGS_025","NGS_IL_WES","","2" +"U000000","1","V","1988","N","","","","DN2024/2","VBE01","Informativiteitstest","11-07-2024","","J","J","2024D00002","BL","2024D00002_D01","311.7","AA1234BB: 2","WL24_01_HelixError","EX_EXO_004","Exoom.analy_IL_elidS34226467_Exoomver.","2","NGS_025","NGS_IL_WES","DN2024/1","0" +"U000000","1","V","1988","N","","","","DN2024/2","VBE01","Informativiteitstest","11-07-2024","","J","J","2024D00003","BL","2024D00003_D01","343.9","AA1234BB: 3","WL24_01_HelixError","NGS_DUPLOCHECK","NGS duplocheck mengfractie","3","NGS_028","NGS_IL_WES_duplocheck","DN2024/1","0" +"U000000","2","M","1988","N","","","","DN2024/3","VBE01","Informativiteitstest","11-07-2024","","J","N","2024D00004","BL","2024D00004_D01","196.7","AA1234BB: 4","WL24_01_HelixError","EX_EXO_004","Exoom.analy_IL_elidS34226467_Exoomver.","4","NGS_025","NGS_IL_WES","DN2024/1","0" +"U000000","2","M","1988","N","","","","DN2024/3","VBE01","Informativiteitstest","11-07-2024","","J","N","2024D00005","BL","2024D00005_D01","215.2","AA1234BB: 5","WL24_01_HelixError","NGS_DUPLOCHECK","NGS duplocheck mengfractie","5","NGS_028","NGS_IL_WES_duplocheck","DN2024/1","0" diff --git a/tests/test_upload_sample.py b/tests/test_upload_sample.py new file mode 100644 index 0000000..e5196c9 --- /dev/null +++ b/tests/test_upload_sample.py @@ -0,0 +1,55 @@ +import genologics +import pytest + +from clarity_epp.upload import samples + + +class ResearcherMock: + def __init__(self, fax, email): + self.fax = fax + self.email = email + + +class ProjectMock: + def __init__(self, name, researcher): + self.name = name + self.researcher = researcher + + +class ContainertypeMock: + def __init__(self, name): + self.name = name + + +def test_from_helix_file_error(mocker): + mocker.patch('genologics.lims.Lims.check_version', return_value=None) + + researchers_mock = [ResearcherMock('HelixError', 'test@test.nl')] + mocker.patch('genologics.lims.Lims.get_researchers', return_value=researchers_mock) + + mocker.patch('genologics.lims.Lims.get_projects', return_value=[]) + + project_mock = ProjectMock('TEST', researchers_mock[0]) + mocker.patch('genologics.entities.Project.create', return_value=project_mock) + + container_type_mock = ContainertypeMock('Tube') + mocker.patch('genologics.entities.Containertype.__new__', return_value=[container_type_mock]) + + mocker.patch('clarity_epp.upload.samples.send_email', return_value=None) + email_settings = { + 'server': 'smtp.test.nl', + 'from': 'from@test.nl', + 'to_import_helix': [] + + } + + with pytest.raises(SystemExit) as exception_info: + with open('tests/data/WL24_01_HelixError.csv', 'r') as helix_file: + samples.from_helix(genologics.lims.Lims, email_settings, helix_file) + + assert exception_info.type == SystemExit + assert exception_info.value.code == ( + "Could not correctly parse data from helix export file (werklijst).\n" + "Row = 1 \t Column = Fractienummer \t UDF = Dx Fractienummer.\n" + "Please check/update the file and try again. Make sure to remove the project from LIMS before retrying." + ) \ No newline at end of file