From 44c4b4461c73c866cfda60ac31b5fc5140278b9c Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 16 Aug 2022 17:23:45 +0200 Subject: [PATCH 01/84] Copy templates --- ...CONVERT_Reverse_Complement_Samplesheet.csv | 65 +++++++++++++++++++ templates/NovaSeq_BCLCONVERT_Samplesheet.csv | 58 +++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv create mode 100644 templates/NovaSeq_BCLCONVERT_Samplesheet.csv diff --git a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv new file mode 100644 index 0000000..e0eb5a8 --- /dev/null +++ b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv @@ -0,0 +1,65 @@ +SORT.BY.${INPUT.CONTAINER.ROW}${INPUT.CONTAINER.COLUMN} +OUTPUT.FILE.NAME,${OUTPUT.CONTAINER.NAME}.csv +PROCESS.POOLED.ARTIFACTS +CONTROL.SAMPLE.DEFAULT.PROJECT.NAME,Controls +HIDE, ${PROCESS.UDF.UMI - Trim}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 1 Length}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 2 Length}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 1 Start From Cycle}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 2 Start From Cycle}, IF, NODATA + +[Header] +Investigator Name,${PROCESS.TECHNICIAN} +Experiment Name,${PROCESS.UDF.Experiment Name} +Date,${DATE} +[Reads] +${PROCESS.UDF.Read 1 Cycles} +${PROCESS.UDF.Read 2 Cycles} +[Settings] +Adapter,${PROCESS.UDF.Adapter} +AdapterRead2,${PROCESS.UDF.Adapter Read 2} +TrimUMI,${PROCESS.UDF.UMI - Trim} +Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} +Read2UMILength,${PROCESS.UDF.UMI - Read 2 Length} +Read1StartFromCycle,${PROCESS.UDF.UMI - Read 1 Start From Cycle} +Read2StartFromCycle,${PROCESS.UDF.UMI - Read 2 Start From Cycle} +[Data] + +
+Sample_ID,Sample_Name,Sample_Plate,Sample_Well,index,index2,Sample_Project,Description +
+ +${INPUT.LIMSID},${SAMPLE.NAME##NoSpecialCharacters},${INPUT.CONTAINER.NAME},${INPUT.CONTAINER.PLACEMENT},${INPUT.REAGENT.SEQUENCE##Single},${INPUT.REAGENT.SEQUENCE##Index2RC},${SAMPLE.PROJECT.NAME##NoSpecialCharacters}, + + +${INPUT.REAGENT.SEQUENCE##Single} +def index = token.indexOf("-") +if (index > 0) { + return token.substring(0,index) +} else { + return token +} + +${INPUT.REAGENT.SEQUENCE##Index2RC} +def index = token.indexOf("-") +if (index > 0) { + return token.split('-')[1].toUpperCase().reverse().collect { base -> + switch (base) { + case 'A': return 'T' + case 'C': return 'G' + case 'G': return 'C' + case 'T': return 'A' + default : throw new Exception("Input sequence '${sequence}' contains an invalid base '${base}'.") + } + }.join('') +} else { + return '' +} + +${SAMPLE.NAME##NoSpecialCharacters} +return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") + +${SAMPLE.PROJECT.NAME##NoSpecialCharacters} +return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") + + \ No newline at end of file diff --git a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv new file mode 100644 index 0000000..cf86d3d --- /dev/null +++ b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv @@ -0,0 +1,58 @@ +SORT.BY.${INPUT.CONTAINER.ROW}${INPUT.CONTAINER.COLUMN} +OUTPUT.FILE.NAME,${OUTPUT.CONTAINER.NAME}.csv +PROCESS.POOLED.ARTIFACTS +CONTROL.SAMPLE.DEFAULT.PROJECT.NAME,Controls +HIDE, ${INPUT.REAGENT.SEQUENCE##Dual}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Trim}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 1 Length}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 2 Length}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 1 Start From Cycle}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 2 Start From Cycle}, IF, NODATA + +[Header] +Investigator Name,${PROCESS.TECHNICIAN} +Experiment Name,${PROCESS.UDF.Experiment Name} +Date,${DATE} +[Reads] +${PROCESS.UDF.Read 1 Cycles} +${PROCESS.UDF.Read 2 Cycles} +[Settings] +Adapter,${PROCESS.UDF.Adapter} +AdapterRead2,${PROCESS.UDF.Adapter Read 2} +TrimUMI,${PROCESS.UDF.UMI - Trim} +Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} +Read2UMILength,${PROCESS.UDF.UMI - Read 2 Length} +Read1StartFromCycle,${PROCESS.UDF.UMI - Read 1 Start From Cycle} +Read2StartFromCycle,${PROCESS.UDF.UMI - Read 2 Start From Cycle} +[Data] + +
+Sample_ID,Sample_Name,Sample_Plate,Sample_Well,index,index2,Sample_Project,Description +
+ +${INPUT.LIMSID},${SAMPLE.NAME##NoSpecialCharacters},${INPUT.CONTAINER.NAME},${INPUT.CONTAINER.PLACEMENT},${INPUT.REAGENT.SEQUENCE##Single},${INPUT.REAGENT.SEQUENCE##Dual},${SAMPLE.PROJECT.NAME##NoSpecialCharacters}, + + +${INPUT.REAGENT.SEQUENCE##Single} +def index = token.indexOf("-") +if (index > 0) { + return token.substring(0,index) +} else { + return token +} + +${INPUT.REAGENT.SEQUENCE##Dual} +def index = token.indexOf("-") +if (index >= 0) { + return token.substring(index + 1) +} else { + return null +} + +${SAMPLE.NAME##NoSpecialCharacters} +return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") + +${SAMPLE.PROJECT.NAME##NoSpecialCharacters} +return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") + + \ No newline at end of file From c1cf7f77dfb2a8e47adb43356abcee57d860d6e0 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 16 Aug 2022 17:26:20 +0200 Subject: [PATCH 02/84] Update Adapter --- templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv | 2 +- templates/NovaSeq_BCLCONVERT_Samplesheet.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv index e0eb5a8..56c0c8e 100644 --- a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv @@ -16,7 +16,7 @@ Date,${DATE} ${PROCESS.UDF.Read 1 Cycles} ${PROCESS.UDF.Read 2 Cycles} [Settings] -Adapter,${PROCESS.UDF.Adapter} +AdapterRead1,${PROCESS.UDF.Adapter} AdapterRead2,${PROCESS.UDF.Adapter Read 2} TrimUMI,${PROCESS.UDF.UMI - Trim} Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} diff --git a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv index cf86d3d..e61c370 100644 --- a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv @@ -17,7 +17,7 @@ Date,${DATE} ${PROCESS.UDF.Read 1 Cycles} ${PROCESS.UDF.Read 2 Cycles} [Settings] -Adapter,${PROCESS.UDF.Adapter} +AdapterRead1,${PROCESS.UDF.Adapter} AdapterRead2,${PROCESS.UDF.Adapter Read 2} TrimUMI,${PROCESS.UDF.UMI - Trim} Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} From a6f673bdfa6a0bb39ce119255c161575f367f424 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 19 Aug 2022 17:39:24 +0200 Subject: [PATCH 03/84] Add OverrideCycles --- clarity_epp/export/illumina.py | 52 ++++++++++++++++--- ...CONVERT_Reverse_Complement_Samplesheet.csv | 5 -- templates/NovaSeq_BCLCONVERT_Samplesheet.csv | 5 -- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index 574bfe0..9827d53 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -181,17 +181,57 @@ def get_project(projects, urgent=False): samplesheet_artifact = Artifact(lims, id=artifact_id) file_id = samplesheet_artifact.files[0].id + # Setup custom settings + custom_settings = '' + # Setup OverrideCycles + if trim_last_base or process.udf['UMI - Trim']: + override_cycles = [ + '', # read 1 + 'I{0}'.format(process.udf['Index Read 1']), # index 1 + 'I{0}'.format(process.udf['Index Read 2']), # index 2 + '', # read 2 + ] + + if trim_last_base and process.udf['UMI - Trim']: + override_cycles[0] = 'U{umi}Y{read}N1'.format( + umi=process.udf['UMI - Read 1 Length'], + read=process.udf['Read 1 Cycles'] - process.udf['UMI - Read 1 Length'] - 1 + ) + override_cycles[3] = 'U{umi}Y{read}N1'.format( + umi=process.udf['UMI - Read 2 Length'], + read=process.udf['Read 2 Cycles'] - process.udf['UMI - Read 2 Length'] - 1 + ) + custom_settings = 'TrimUMI,1' + + elif trim_last_base: + override_cycles[0] = 'Y{read}N1'.format(read=process.udf['Read 1 Cycles'] - 1) + override_cycles[3] = 'Y{read}N1'.format(read=process.udf['Read 2 Cycles'] - 1) + + elif process.udf['UMI - Trim']: + override_cycles[0] = 'U{umi}Y{read}'.format( + umi=process.udf['UMI - Read 1 Length'], + read=process.udf['Read 1 Cycles'] - process.udf['UMI - Read 1 Length'] + ) + override_cycles[3] = 'U{umi}Y{read}'.format( + umi=process.udf['UMI - Read 2 Length'], + read=process.udf['Read 2 Cycles'] - process.udf['UMI - Read 2 Length'] + ) + custom_settings = 'TrimUMI,1' + + custom_settings = '{settings}\nOverrideCycles,{override_cycles}\n'.format( + settings=custom_settings, + override_cycles=';'.join(override_cycles) + ) + for line in lims.get_file_contents(id=file_id).rstrip().split('\n'): - if line.startswith('[Settings]') and trim_last_base: + if line.startswith('[Settings]') and custom_settings: output_file.write('{line}\n'.format(line=line)) - output_file.write('Read1EndWithCycle,{value}\n'.format(value=process.udf['Read 1 Cycles']-1)) - output_file.write('Read2EndWithCycle,{value}\n'.format(value=process.udf['Read 2 Cycles']-1)) + output_file.write(custom_settings) settings_section = True - elif line.startswith('[Data]') and trim_last_base and not settings_section: + elif line.startswith('[Data]') and custom_settings and not settings_section: output_file.write('[Settings]\n') - output_file.write('Read1EndWithCycle,{value}\n'.format(value=process.udf['Read 1 Cycles']-1)) - output_file.write('Read2EndWithCycle,{value}\n'.format(value=process.udf['Read 2 Cycles']-1)) + output_file.write(custom_settings) output_file.write('{line}\n'.format(line=line)) elif line.startswith('Sample_ID'): # Samples header line diff --git a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv index 56c0c8e..72e6965 100644 --- a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv @@ -18,11 +18,6 @@ ${PROCESS.UDF.Read 2 Cycles} [Settings] AdapterRead1,${PROCESS.UDF.Adapter} AdapterRead2,${PROCESS.UDF.Adapter Read 2} -TrimUMI,${PROCESS.UDF.UMI - Trim} -Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} -Read2UMILength,${PROCESS.UDF.UMI - Read 2 Length} -Read1StartFromCycle,${PROCESS.UDF.UMI - Read 1 Start From Cycle} -Read2StartFromCycle,${PROCESS.UDF.UMI - Read 2 Start From Cycle} [Data]
diff --git a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv index e61c370..b73dc7b 100644 --- a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv @@ -19,11 +19,6 @@ ${PROCESS.UDF.Read 2 Cycles} [Settings] AdapterRead1,${PROCESS.UDF.Adapter} AdapterRead2,${PROCESS.UDF.Adapter Read 2} -TrimUMI,${PROCESS.UDF.UMI - Trim} -Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} -Read2UMILength,${PROCESS.UDF.UMI - Read 2 Length} -Read1StartFromCycle,${PROCESS.UDF.UMI - Read 1 Start From Cycle} -Read2StartFromCycle,${PROCESS.UDF.UMI - Read 2 Start From Cycle} [Data]
From a548e2a8352dc7e369ec1f3068435d45f00c410f Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 22 Aug 2022 11:27:21 +0200 Subject: [PATCH 04/84] Move newline --- clarity_epp/export/illumina.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index 9827d53..3d915f5 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -201,7 +201,7 @@ def get_project(projects, urgent=False): umi=process.udf['UMI - Read 2 Length'], read=process.udf['Read 2 Cycles'] - process.udf['UMI - Read 2 Length'] - 1 ) - custom_settings = 'TrimUMI,1' + custom_settings = 'TrimUMI,1\n' elif trim_last_base: override_cycles[0] = 'Y{read}N1'.format(read=process.udf['Read 1 Cycles'] - 1) @@ -216,9 +216,9 @@ def get_project(projects, urgent=False): umi=process.udf['UMI - Read 2 Length'], read=process.udf['Read 2 Cycles'] - process.udf['UMI - Read 2 Length'] ) - custom_settings = 'TrimUMI,1' + custom_settings = 'TrimUMI,1\n' - custom_settings = '{settings}\nOverrideCycles,{override_cycles}\n'.format( + custom_settings = '{settings}OverrideCycles,{override_cycles}\n'.format( settings=custom_settings, override_cycles=';'.join(override_cycles) ) From 5c53fe9259eb65cd0cc43213854acec23e4486a3 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 17 Jan 2023 14:29:19 +0100 Subject: [PATCH 05/84] Replace special chars with '-' --- templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv | 2 +- templates/NovaSeq_BCLCONVERT_Samplesheet.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv index 72e6965..60b6124 100644 --- a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv @@ -52,7 +52,7 @@ if (index > 0) { } ${SAMPLE.NAME##NoSpecialCharacters} -return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") +return token.replaceAll(\"[^a-zA-Z0-9-]\", \"-\") ${SAMPLE.PROJECT.NAME##NoSpecialCharacters} return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") diff --git a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv index b73dc7b..6c9133e 100644 --- a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv @@ -45,7 +45,7 @@ if (index >= 0) { } ${SAMPLE.NAME##NoSpecialCharacters} -return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") +return token.replaceAll(\"[^a-zA-Z0-9-]\", \"-\") ${SAMPLE.PROJECT.NAME##NoSpecialCharacters} return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") From 85d151b29f8f1a8584904bb765fff5624b392a07 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 1 Jun 2023 13:20:49 +0200 Subject: [PATCH 06/84] Add Q support --- scripts/management_review.py | 61 +++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/scripts/management_review.py b/scripts/management_review.py index 477e4fc..372178b 100644 --- a/scripts/management_review.py +++ b/scripts/management_review.py @@ -10,8 +10,17 @@ password = 'lims_user_password' lims = Lims(baseuri, username, password) +# month quarter +quarters = ['Q1', 'Q2', 'Q3', 'Q4'] +month_quarter = { + '01': 'Q1', '02': 'Q1', '03': 'Q1', + '04': 'Q2', '05': 'Q2', '06': 'Q2', + '07': 'Q3', '08': 'Q3', '09': 'Q3', + '10': 'Q4', '11': 'Q4', '12': 'Q4' +} + # Get DX projects and filter on year based on project name -dx_projects = [project for project in lims.get_projects(udf={'Application': 'DX'}) if project.name.startswith('Dx WL21')] +dx_projects = [project for project in lims.get_projects(udf={'Application': 'DX'}) if project.name.startswith('Dx WL23')] sample_count = 0 # Expected actions @@ -25,7 +34,14 @@ 'Dx 3nM verdunning Magnis', 'Dx Multiplexen Enrichment samples (3nM) Magnis', 'Dx Multiplexen Enrichment pools Magnis', 'Dx Multiplexen sequence pool', 'Dx Library pool denatureren en laden (NovaSeq)', 'AUTOMATED - NovaSeq Run (NovaSeq 6000)', 'Dx Library pool denatureren en laden (NovaSeq) Dx QC controle Lab sequencen', - 'Dx NGS labwerk afronden', 'Dx Bioinformatica analyses', 'Dx NGS onderzoeken afronden', + 'Dx Library pool denatureren en laden (NovaSeq) Dx NovaSeq QC controle Lab sequencen', + 'Dx NGS labwerk afronden', 'Dx Bioinformatica analyses', 'Dx Fingerprint match maken', 'Dx NGS onderzoeken afronden', + 'Dx sample registratie pool', 'Dx Capture', 'Dx Exonuclease behandeling', 'Dx PCR na exonuclease behandeling', + 'Dx smMIP multiplexen & BBSS sequence pool', 'Dx NGS smMIP onderzoeken afronden', 'Dx smMIP sequence pool verdunning', + 'Dx Library pool denatureren en laden (Nextseq)', 'Dx NextSeq Run', + 'Dx Library pool denatureren en laden (Nextseq) Dx QC controle Lab sequencen', + 'Dx Sample registratie', 'Dx gDNA handmatige normalisatie', 'Dx Uitvullen en zuiveren (Fluent 480)', + 'Dx Uitvullen en zuiveren (Fluent 480)', 'Dx Normaliseren (Fluent 480)', 'Dx gDNA handmatige normalisatie WES', # CREv2 'Dx Fragmenteren & BBSS', 'Dx LibraryPrep Caliper KAPA', 'Dx Library Prep amplificatie & clean up KAPA', @@ -40,8 +56,11 @@ 'Dx Tapestation 2200 QC', 'Dx Tapestation 4200 QC', 'Dx Aggregate QC' ] processes_before_qc = [ - 'Dx Hamilton zuiveren', 'Dx Zuiveren gDNA manueel', 'Dx Placement Enrichment Magnis', - 'Dx Multiplexen Enrichment pools Magnis', + 'Dx Hamilton uitvullen', 'Dx Hamilton zuiveren', 'Dx Zuiveren gDNA manueel', 'Dx Placement Enrichment Magnis', + 'Dx Multiplexen Enrichment pools Magnis', 'Dx sample registratie pool', 'Dx smMIP multiplexen & BBSS sequence pool', + 'Dx PCR na exonuclease behandeling', 'Dx Sample registratie', 'Dx Exonuclease behandeling', + 'Dx Sample registratie zuivering', + # CREv2 'Dx Fragmenteren & BBSS', 'Dx LibraryPrep Caliper KAPA', 'Dx Library Prep amplificatie & clean up KAPA', 'Dx Post Enrichment PCR & clean up', 'Dx Multiplexen library pool' @@ -53,16 +72,19 @@ # Setup count dictonary process_action_counts = OrderedDict() -for process in processes: - process_action_counts[process] = {} - for action in action_list: - process_action_counts[process][action] = 0 +for quarter in quarters: + process_action_counts[quarter] = OrderedDict() + for process in processes: + process_action_counts[quarter][process] = {} + for action in action_list: + process_action_counts[quarter][process][action] = 0 for project in dx_projects: for sample in lims.get_samples(projectlimsid=project.id): - if sample.udf['Dx Onderzoeksreden'] == 'Research': # skip research + if 'Dx Onderzoeksreden' not in sample.udf or sample.udf['Dx Onderzoeksreden'] == 'Research': # skip research continue sample_count += 1 + sample_quarter = month_quarter[sample.date_received.split('-')[1]] for artifact in lims.get_artifacts(samplelimsid=sample.id, resolve=True, type='Analyte'): for process in lims.get_processes(inputartifactlimsid=artifact.id): @@ -93,14 +115,17 @@ # Get action for artifact for action in process.step.actions.get_next_actions(): if action['artifact'].id in output_artifacts: - process_action_counts[process_name]['total'] += 1 - process_action_counts[process_name][str(action['action'])] += 1 + process_action_counts[sample_quarter][process_name]['total'] += 1 + process_action_counts[sample_quarter][process_name][str(action['action'])] += 1 print('Total Sample count: {0}'.format(str(sample_count))) -print('Process\t{action_list}'.format(action_list='\t'.join(action_list))) -for process in process_action_counts: - if process_action_counts[process]['total']: - print('{process}\t{action_list}'.format( - process=process, - action_list='\t'.join([str(process_action_counts[process][action]) if action in process_action_counts[process] else '0' for action in action_list]) - )) + +for quarter in quarters: + print(quarter) + print('Process\t{action_list}'.format(action_list='\t'.join(action_list))) + for process in process_action_counts[quarter]: + if process_action_counts[quarter][process]['total']: + print('{process}\t{action_list}'.format( + process=process, + action_list='\t'.join([str(process_action_counts[quarter][process][action]) if action in process_action_counts[quarter][process] else '0' for action in action_list]) + )) From ad70c07a9fe2f4bf904b45cf5547a114837a526c Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 8 Jun 2023 11:46:00 +0200 Subject: [PATCH 07/84] WIP --- clarity_epp/placement/pool.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/clarity_epp/placement/pool.py b/clarity_epp/placement/pool.py index 5f8b86a..23258b4 100644 --- a/clarity_epp/placement/pool.py +++ b/clarity_epp/placement/pool.py @@ -36,9 +36,13 @@ def unpooling(lims, process_id): for node in pool_artifact_demux.getiterator('artifact'): if node.find('samples'): if len(node.find('samples').findall('sample')) == 1: + # if len(node.find('samples').findall('sample')) in [1, 2]: sample_artifact = Artifact(lims, uri=node.attrib['uri']) sample = sample_artifact.samples[0] # 1 sample per artifact. + # Skip non dx samples? + # Check if pool with 2 samples come from same person. + # Get sample sequencing run and project from samplesheet sample_artifact.udf['Dx Sequencing Run ID'] = run_id if 'Sample Type' in sample.udf and 'library' in sample.udf['Sample Type']: # Use sample.name for external (clarity_portal) samples @@ -49,5 +53,5 @@ def unpooling(lims, process_id): if sample_artifact.samples[0].project and sample_artifact.samples[0].project.udf['Application'] == 'DX': # Only move DX production samples to post sequencing workflow sample_artifacts.append(sample_artifact) - + # print(sample_artifacts) lims.route_artifacts(sample_artifacts, workflow_uri=Workflow(lims, id=config.post_sequencing_workflow).uri) From 5db491a581a4384eca7f2645e8f913a51385d9a9 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 8 Jun 2023 12:58:27 +0200 Subject: [PATCH 08/84] Add create patient pools and update sequence name. Sequence name change breaks ped, samplesheet, merge and helix export files. This will be fixed later. --- clarity_epp.py | 9 +++++++++ clarity_epp/export/sample.py | 1 - clarity_epp/placement/artifact.py | 3 +-- clarity_epp/placement/pool.py | 30 +++++++++++++++++++++++++++++- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/clarity_epp.py b/clarity_epp.py index dfc4ee6..2c93f17 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -219,6 +219,11 @@ def placement_unpooling(args): clarity_epp.placement.pool.unpooling(lims, args.process_id) +def placement_patient_pools(args): + """Create patient pools for Dx samples.""" + clarity_epp.placement.pool.create_patient_pools(lims, args.process_id) + + def placement_complete_step(args): """Complete protocol step (Dx Mark protocol complete).""" clarity_epp.placement.step.finish_protocol_complete(lims, args.process_id) @@ -428,6 +433,10 @@ def placement_tecan(args): parser_placement_unpooling.add_argument('process_id', help='Clarity lims process id') parser_placement_unpooling.set_defaults(func=placement_unpooling) + parser_placement_patient_pools = subparser_placement.add_parser('patient_pools', help='Create patient pools for Dx samples') + parser_placement_patient_pools.add_argument('process_id', help='Clarity lims process id') + parser_placement_patient_pools.set_defaults(func=placement_patient_pools) + parser_placement_tecan = subparser_placement.add_parser('tecan', help='Placement of samples in tecan step') parser_placement_tecan.add_argument('process_id', help='Clarity lims process id') parser_placement_tecan.set_defaults(func=placement_tecan) diff --git a/clarity_epp/export/sample.py b/clarity_epp/export/sample.py index 84d6844..74bee68 100644 --- a/clarity_epp/export/sample.py +++ b/clarity_epp/export/sample.py @@ -4,7 +4,6 @@ from genologics.entities import Process import clarity_epp.export.utils -from .. import get_sequence_name def removed_samples(lims, output_file): diff --git a/clarity_epp/placement/artifact.py b/clarity_epp/placement/artifact.py index 8963ea7..05cd257 100644 --- a/clarity_epp/placement/artifact.py +++ b/clarity_epp/placement/artifact.py @@ -10,8 +10,7 @@ def set_sequence_name(lims, process_id): """Change artifact name to sequnece name.""" process = Process(lims, id=process_id) for artifact in process.analytes()[0]: - sample = artifact.samples[0] - artifact.name = get_sequence_name(sample) + artifact.name = get_sequence_name(artifact) artifact.put() diff --git a/clarity_epp/placement/pool.py b/clarity_epp/placement/pool.py index 23258b4..8fe3a48 100644 --- a/clarity_epp/placement/pool.py +++ b/clarity_epp/placement/pool.py @@ -1,7 +1,8 @@ """Pool placement functions.""" -from genologics.entities import Artifact, Process, Workflow +from genologics.entities import Artifact, Process, Workflow, Step +from .. import get_sequence_name import config @@ -55,3 +56,30 @@ def unpooling(lims, process_id): sample_artifacts.append(sample_artifact) # print(sample_artifacts) lims.route_artifacts(sample_artifacts, workflow_uri=Workflow(lims, id=config.post_sequencing_workflow).uri) + + +def create_patient_pools(lims, process_id): + """Create patient pools for Dx samples based on UDF 'Dx Persoons ID'.""" + step = Step(lims, id=process_id) + step_pools = step.step_pools + patient_pools = {} + + # Create patient pools + for artifact in step_pools.available_inputs: + sample = artifact.samples[0] # Assume one sample per artifact + if sample.udf['Dx Persoons ID'] not in patient_pools: + patient_pools[sample.udf['Dx Persoons ID']] = { + 'name': str(sample.udf['Dx Persoons ID']), + 'inputs': [] + } + patient_pools[sample.udf['Dx Persoons ID']]['inputs'].append(artifact) + + # Transform patient pools to list and put to clarity + step_pools.set_pools(list(patient_pools.values())) + step_pools.put() + + # Rename pools to sequence name + process = Process(lims, id=process_id) + for artifact in process.all_outputs(): + artifact.name = get_sequence_name(artifact) + artifact.put() From e3338b7e33931e2deba59e211d506ea60e6ef2a1 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 8 Jun 2023 12:59:20 +0200 Subject: [PATCH 09/84] pep8 --- clarity_epp/placement/pool.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/clarity_epp/placement/pool.py b/clarity_epp/placement/pool.py index 8fe3a48..e53e003 100644 --- a/clarity_epp/placement/pool.py +++ b/clarity_epp/placement/pool.py @@ -39,20 +39,21 @@ def unpooling(lims, process_id): if len(node.find('samples').findall('sample')) == 1: # if len(node.find('samples').findall('sample')) in [1, 2]: sample_artifact = Artifact(lims, uri=node.attrib['uri']) - sample = sample_artifact.samples[0] # 1 sample per artifact. + sample = sample_artifact.samples[0] # 1 sample per artifact. # Skip non dx samples? # Check if pool with 2 samples come from same person. # Get sample sequencing run and project from samplesheet sample_artifact.udf['Dx Sequencing Run ID'] = run_id - if 'Sample Type' in sample.udf and 'library' in sample.udf['Sample Type']: # Use sample.name for external (clarity_portal) samples + # Use sample.name for external (clarity_portal) samples + if 'Sample Type' in sample.udf and 'library' in sample.udf['Sample Type']: sample_artifact.udf['Dx Sequencing Run Project'] = sample_projects[sample.name] else: # Use sample_artifact.name for Dx samples (upload via Helix) sample_artifact.udf['Dx Sequencing Run Project'] = sample_projects[sample_artifact.name] sample_artifact.put() - - if sample_artifact.samples[0].project and sample_artifact.samples[0].project.udf['Application'] == 'DX': # Only move DX production samples to post sequencing workflow + # Only move DX production samples to post sequencing workflow + if sample_artifact.samples[0].project and sample_artifact.samples[0].project.udf['Application'] == 'DX': sample_artifacts.append(sample_artifact) # print(sample_artifacts) lims.route_artifacts(sample_artifacts, workflow_uri=Workflow(lims, id=config.post_sequencing_workflow).uri) From 08f01a4921fc47ae629eb668c59b48aac005f1e8 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 8 Jun 2023 13:09:16 +0200 Subject: [PATCH 10/84] Remove sequence name, does not work on server. --- clarity_epp/placement/pool.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/clarity_epp/placement/pool.py b/clarity_epp/placement/pool.py index e53e003..49bf3b7 100644 --- a/clarity_epp/placement/pool.py +++ b/clarity_epp/placement/pool.py @@ -78,9 +78,3 @@ def create_patient_pools(lims, process_id): # Transform patient pools to list and put to clarity step_pools.set_pools(list(patient_pools.values())) step_pools.put() - - # Rename pools to sequence name - process = Process(lims, id=process_id) - for artifact in process.all_outputs(): - artifact.name = get_sequence_name(artifact) - artifact.put() From f44d512ebeafc678eca88c0fb1759eb3da2a0bf6 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 8 Jun 2023 13:14:31 +0200 Subject: [PATCH 11/84] Update get_sequence_name --- clarity_epp/__init__.py | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/clarity_epp/__init__.py b/clarity_epp/__init__.py index 2380286..11ab01d 100644 --- a/clarity_epp/__init__.py +++ b/clarity_epp/__init__.py @@ -8,34 +8,18 @@ import mimetypes -def get_sequence_name(sample): +def get_sequence_name(artifact): """Generate sequence name.""" - try: - # Set fam_status - if sample.udf['Dx Familie status'] == 'Kind': - fam_status = 'C' - elif sample.udf['Dx Familie status'] == 'Ouder': - fam_status = 'P' + sample_numbers = [] + for sample in artifact.samples: + if 'Dx Monsternummer' in sample.udf: + sample_numbers.append(sample.udf['Dx Monsternummer']) + + if sample_numbers: + sequence_name = '-'.join(sample_numbers) + else: # non Dx sample + sequence_name = artifact.sample[0].name - # Set sex - if sample.udf['Dx Geslacht'] == 'Man': - sex = 'M' - elif sample.udf['Dx Geslacht'] == 'Vrouw': - sex = 'F' - elif sample.udf['Dx Geslacht'] == 'Onbekend': - sex = 'O' - except KeyError: # None DX sample, use sample.name as sequence name. - sequence_name = sample.name - else: - if not sample.name.startswith(sample.udf['Dx Familienummer']): - sequence_name = '{familienummer}{fam_status}{sex}{monsternummer}'.format( - familienummer=sample.udf['Dx Familienummer'], - fam_status=fam_status, - sex=sex, - monsternummer=sample.udf['Dx Monsternummer'] - ) - else: - sequence_name = sample.name return sequence_name From d0a911fcc6b100c131f2e6269deb9f1964782a22 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 8 Jun 2023 13:14:40 +0200 Subject: [PATCH 12/84] Revert --- clarity_epp/placement/pool.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clarity_epp/placement/pool.py b/clarity_epp/placement/pool.py index 49bf3b7..e53e003 100644 --- a/clarity_epp/placement/pool.py +++ b/clarity_epp/placement/pool.py @@ -78,3 +78,9 @@ def create_patient_pools(lims, process_id): # Transform patient pools to list and put to clarity step_pools.set_pools(list(patient_pools.values())) step_pools.put() + + # Rename pools to sequence name + process = Process(lims, id=process_id) + for artifact in process.all_outputs(): + artifact.name = get_sequence_name(artifact) + artifact.put() From eaa4dc2e69a123462940bf7ddb3d582ef5a91037 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Fri, 9 Jun 2023 17:40:57 +0200 Subject: [PATCH 13/84] added multiplier for sp/s1 --- clarity_epp/export/manual_pipetting.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index a8e249d..44745f1 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -689,6 +689,11 @@ def samplesheet_pool_magnis_pools(lims, process_id, output_file): """Create manual pipetting samplesheet for pooling magnis pools. Correct for pools with < 8 samples""" process = Process(lims, id=process_id) + # set up multiplier + multiplier = 1 + if 'Run type' in process.udf and 'Sp/S1' in process.udf['Run type']: + multiplier = 2.5 + # print header output_file.write('Pool\tContainer\tSample count\tVolume (ul)\n') @@ -706,6 +711,6 @@ def samplesheet_pool_magnis_pools(lims, process_id, output_file): pool=input_artifact.name, container=input_artifact.container.name, sample_count=sample_count, - volume=sample_count * 1.25 + volume=sample_count * 1.25 * multiplier ) ) From c708bff092f598946ffc2c7a0b08e33a1d78077e Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 12 Jun 2023 18:43:44 +0200 Subject: [PATCH 14/84] Add bcl2fastq option for backwards comptability. --- clarity_epp/export/illumina.py | 84 +++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index 617e0a7..f8db73a 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -9,7 +9,7 @@ import config -def update_samplesheet(lims, process_id, artifact_id, output_file): +def update_samplesheet(lims, process_id, artifact_id, output_file, conversion_tool='bcl2fastq'): """Update illumina samplesheet.""" process = Process(lims, id=process_id) trim_last_base = True # Used to set Read1EndWithCycle @@ -184,45 +184,55 @@ def get_project(projects, urgent=False): # Setup custom settings custom_settings = '' - # Setup OverrideCycles - if trim_last_base or process.udf['UMI - Trim']: - override_cycles = [ - '', # read 1 - 'I{0}'.format(process.udf['Index Read 1']), # index 1 - 'I{0}'.format(process.udf['Index Read 2']), # index 2 - '', # read 2 - ] - - if trim_last_base and process.udf['UMI - Trim']: - override_cycles[0] = 'U{umi}Y{read}N1'.format( - umi=process.udf['UMI - Read 1 Length'], - read=process.udf['Read 1 Cycles'] - process.udf['UMI - Read 1 Length'] - 1 - ) - override_cycles[3] = 'U{umi}Y{read}N1'.format( - umi=process.udf['UMI - Read 2 Length'], - read=process.udf['Read 2 Cycles'] - process.udf['UMI - Read 2 Length'] - 1 - ) - custom_settings = 'TrimUMI,1\n' - elif trim_last_base: - override_cycles[0] = 'Y{read}N1'.format(read=process.udf['Read 1 Cycles'] - 1) - override_cycles[3] = 'Y{read}N1'.format(read=process.udf['Read 2 Cycles'] - 1) + if conversion_tool == 'bcl2fastq': + custom_settings = ( + 'Read1EndWithCycle,{read_1_value}\n' + 'Read2EndWithCycle,{read_2_value}\n' + ).format( + read_1_value=process.udf['Read 1 Cycles']-1, read_2_value=process.udf['Read 2 Cycles']-1 + ) - elif process.udf['UMI - Trim']: - override_cycles[0] = 'U{umi}Y{read}'.format( - umi=process.udf['UMI - Read 1 Length'], - read=process.udf['Read 1 Cycles'] - process.udf['UMI - Read 1 Length'] + elif conversion_tool == 'bclconvert': + # Setup OverrideCycles + if trim_last_base or process.udf['UMI - Trim']: + override_cycles = [ + '', # read 1 + 'I{0}'.format(process.udf['Index Read 1']), # index 1 + 'I{0}'.format(process.udf['Index Read 2']), # index 2 + '', # read 2 + ] + + if trim_last_base and process.udf['UMI - Trim']: + override_cycles[0] = 'U{umi}Y{read}N1'.format( + umi=process.udf['UMI - Read 1 Length'], + read=process.udf['Read 1 Cycles'] - process.udf['UMI - Read 1 Length'] - 1 + ) + override_cycles[3] = 'U{umi}Y{read}N1'.format( + umi=process.udf['UMI - Read 2 Length'], + read=process.udf['Read 2 Cycles'] - process.udf['UMI - Read 2 Length'] - 1 + ) + custom_settings = 'TrimUMI,1\n' + + elif trim_last_base: + override_cycles[0] = 'Y{read}N1'.format(read=process.udf['Read 1 Cycles'] - 1) + override_cycles[3] = 'Y{read}N1'.format(read=process.udf['Read 2 Cycles'] - 1) + + elif process.udf['UMI - Trim']: + override_cycles[0] = 'U{umi}Y{read}'.format( + umi=process.udf['UMI - Read 1 Length'], + read=process.udf['Read 1 Cycles'] - process.udf['UMI - Read 1 Length'] + ) + override_cycles[3] = 'U{umi}Y{read}'.format( + umi=process.udf['UMI - Read 2 Length'], + read=process.udf['Read 2 Cycles'] - process.udf['UMI - Read 2 Length'] + ) + custom_settings = 'TrimUMI,1\n' + + custom_settings = '{settings}OverrideCycles,{override_cycles}\n'.format( + settings=custom_settings, + override_cycles=';'.join(override_cycles) ) - override_cycles[3] = 'U{umi}Y{read}'.format( - umi=process.udf['UMI - Read 2 Length'], - read=process.udf['Read 2 Cycles'] - process.udf['UMI - Read 2 Length'] - ) - custom_settings = 'TrimUMI,1\n' - - custom_settings = '{settings}OverrideCycles,{override_cycles}\n'.format( - settings=custom_settings, - override_cycles=';'.join(override_cycles) - ) for line in lims.get_file_contents(id=file_id).rstrip().split('\n'): if line.startswith('[Settings]') and custom_settings: From d49158c6aee32dea0c5b8fb4d4176c33962ecc31 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 12 Jun 2023 21:57:01 +0200 Subject: [PATCH 15/84] Add CLI --- clarity_epp.py | 7 ++++++- clarity_epp/export/illumina.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/clarity_epp.py b/clarity_epp.py index dfc4ee6..78e9064 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -48,7 +48,9 @@ def export_hamilton(args): def export_illumina(args): """Export (updated) illumina samplesheet.""" - clarity_epp.export.illumina.update_samplesheet(lims, args.process_id, args.artifact_id, args.output_file) + clarity_epp.export.illumina.update_samplesheet( + lims, args.process_id, args.artifact_id, args.output_file, args.conversion_tool + ) def export_labels(args): @@ -271,6 +273,9 @@ def placement_tecan(args): ) parser_export_illumina.add_argument('process_id', help='Clarity lims process id') parser_export_illumina.add_argument('artifact_id', help='Clarity lims samplesheet artifact id') + parser_export_illumina.add_argument( + '-c', '--conversion_tool', choices=['bcl2fastq', 'bclconvert'], default='bcl2fastq', help='Illumina conversion tool' + ) parser_export_illumina.set_defaults(func=export_illumina) parser_export_labels = subparser_export.add_parser('labels', help='Export container labels', parents=[output_parser]) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index f8db73a..109ed93 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -9,7 +9,7 @@ import config -def update_samplesheet(lims, process_id, artifact_id, output_file, conversion_tool='bcl2fastq'): +def update_samplesheet(lims, process_id, artifact_id, output_file, conversion_tool): """Update illumina samplesheet.""" process = Process(lims, id=process_id) trim_last_base = True # Used to set Read1EndWithCycle From c9f61f7c03f5723d9e576878678db36cf7c4b6e7 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 13 Jun 2023 17:41:10 +0200 Subject: [PATCH 16/84] Remove unused fields --- .../NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv | 5 ----- templates/NovaSeq_BCLCONVERT_Samplesheet.csv | 5 ----- 2 files changed, 10 deletions(-) diff --git a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv index 60b6124..d0a9ca3 100644 --- a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv @@ -2,11 +2,6 @@ SORT.BY.${INPUT.CONTAINER.ROW}${INPUT.CONTAINER.COLUMN} OUTPUT.FILE.NAME,${OUTPUT.CONTAINER.NAME}.csv PROCESS.POOLED.ARTIFACTS CONTROL.SAMPLE.DEFAULT.PROJECT.NAME,Controls -HIDE, ${PROCESS.UDF.UMI - Trim}, IF, NODATA -HIDE, ${PROCESS.UDF.UMI - Read 1 Length}, IF, NODATA -HIDE, ${PROCESS.UDF.UMI - Read 2 Length}, IF, NODATA -HIDE, ${PROCESS.UDF.UMI - Read 1 Start From Cycle}, IF, NODATA -HIDE, ${PROCESS.UDF.UMI - Read 2 Start From Cycle}, IF, NODATA [Header] Investigator Name,${PROCESS.TECHNICIAN} diff --git a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv index 6c9133e..7dfb88f 100644 --- a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv @@ -3,11 +3,6 @@ OUTPUT.FILE.NAME,${OUTPUT.CONTAINER.NAME}.csv PROCESS.POOLED.ARTIFACTS CONTROL.SAMPLE.DEFAULT.PROJECT.NAME,Controls HIDE, ${INPUT.REAGENT.SEQUENCE##Dual}, IF, NODATA -HIDE, ${PROCESS.UDF.UMI - Trim}, IF, NODATA -HIDE, ${PROCESS.UDF.UMI - Read 1 Length}, IF, NODATA -HIDE, ${PROCESS.UDF.UMI - Read 2 Length}, IF, NODATA -HIDE, ${PROCESS.UDF.UMI - Read 1 Start From Cycle}, IF, NODATA -HIDE, ${PROCESS.UDF.UMI - Read 2 Start From Cycle}, IF, NODATA [Header] Investigator Name,${PROCESS.TECHNICIAN} From 236b74080a8df88d477b26e7ab4371ff0cb507c2 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 14 Jun 2023 12:11:11 +0200 Subject: [PATCH 17/84] Skip barcode check for same person id --- clarity_epp/placement/barcode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clarity_epp/placement/barcode.py b/clarity_epp/placement/barcode.py index b36eb98..453c191 100644 --- a/clarity_epp/placement/barcode.py +++ b/clarity_epp/placement/barcode.py @@ -9,7 +9,7 @@ def check_family(lims, process_id): for artifact in process.analytes()[0]: sample = artifact.samples[0] barcode = artifact.reagent_labels[0] - + try: query_udf = {'Dx Familienummer': sample.udf['Dx Familienummer']} except KeyError: @@ -18,7 +18,7 @@ def check_family(lims, process_id): else: family_samples = lims.get_samples(udf=query_udf) for family_sample in family_samples: - if family_sample.id != sample.id: + if family_sample.id != sample.id and family_sample.udf['Dx Persoons ID'] != sample.udf['Dx Persoons ID']: family_sample_artifacts = lims.get_artifacts(samplelimsid=family_sample.id, reagent_label=barcode, process_type=process.type.name) if family_sample_artifacts: artifact.udf['Dx monster met BC duplicaat'] = "{sample}".format(sample=family_sample.name) From 31dd7353eb372e4bd49410be853bb3dad84e8a78 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 14 Jun 2023 12:29:20 +0200 Subject: [PATCH 18/84] Skip duplicate samples --- clarity_epp/export/manual_pipetting.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index a8e249d..326361f 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -688,6 +688,7 @@ def samplesheet_pool_samples(lims, process_id, output_file): def samplesheet_pool_magnis_pools(lims, process_id, output_file): """Create manual pipetting samplesheet for pooling magnis pools. Correct for pools with < 8 samples""" process = Process(lims, id=process_id) + sample_ids = [] # print header output_file.write('Pool\tContainer\tSample count\tVolume (ul)\n') @@ -696,6 +697,13 @@ def samplesheet_pool_magnis_pools(lims, process_id, output_file): for input_artifact in sorted(process.all_inputs(resolve=True), key=lambda artifact: artifact.id): sample_count = 0 for sample in input_artifact.samples: + # Check persoons ID to skip duplicate samples + if 'Dx Persoons ID' in sample.udf: + if sample.udf['Dx Persoons ID'] in sample_ids: + continue # skip to next sample + else: + sample_ids.append(sample.udf['Dx Persoons ID']) + if 'Dx Exoomequivalent' in sample.udf: sample_count += sample.udf['Dx Exoomequivalent'] else: From 5501eae7e2bcc29719308f441f08a2ec51fc5627 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 14 Jun 2023 14:32:56 +0200 Subject: [PATCH 19/84] Skip duplicate samples --- clarity_epp/export/manual_pipetting.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 326361f..5a5fbd2 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -328,6 +328,7 @@ def samplesheet_multiplex_sequence_pool(lims, process_id, output_file): final_volume = float(process.udf['Final volume'].split()[0]) for input_pool in process.all_inputs(): + input_pool_sample_ids = [] input_pool_conc = float(input_pool.udf['Dx Concentratie fluorescentie (ng/ul)']) input_pool_size = float(input_pool.udf['Dx Fragmentlengte (bp)']) input_pool_nM = (input_pool_conc * 1000 * (1.0/660.0) * (1/input_pool_size)) * 1000 @@ -336,10 +337,18 @@ def samplesheet_multiplex_sequence_pool(lims, process_id, output_file): input_pool_sample_count = 0 for sample in input_pool.samples: + # Check persoons ID to skip duplicate samples + if 'Dx Persoons ID' in sample.udf: + if sample.udf['Dx Persoons ID'] in input_pool_sample_ids: + continue # skip to next sample + else: + input_pool_sample_ids.append(sample.udf['Dx Persoons ID']) + if 'Dx Exoomequivalent' in sample.udf: input_pool_sample_count += sample.udf['Dx Exoomequivalent'] else: input_pool_sample_count += 1 + total_sample_count += input_pool_sample_count input_pools.append({ 'name': input_pool.name, @@ -688,7 +697,6 @@ def samplesheet_pool_samples(lims, process_id, output_file): def samplesheet_pool_magnis_pools(lims, process_id, output_file): """Create manual pipetting samplesheet for pooling magnis pools. Correct for pools with < 8 samples""" process = Process(lims, id=process_id) - sample_ids = [] # print header output_file.write('Pool\tContainer\tSample count\tVolume (ul)\n') @@ -696,13 +704,14 @@ def samplesheet_pool_magnis_pools(lims, process_id, output_file): # Get input pools, sort by name and print volume for input_artifact in sorted(process.all_inputs(resolve=True), key=lambda artifact: artifact.id): sample_count = 0 + input_artifact_sample_ids = [] for sample in input_artifact.samples: # Check persoons ID to skip duplicate samples if 'Dx Persoons ID' in sample.udf: - if sample.udf['Dx Persoons ID'] in sample_ids: + if sample.udf['Dx Persoons ID'] in input_artifact_sample_ids: continue # skip to next sample else: - sample_ids.append(sample.udf['Dx Persoons ID']) + input_artifact_sample_ids.append(sample.udf['Dx Persoons ID']) if 'Dx Exoomequivalent' in sample.udf: sample_count += sample.udf['Dx Exoomequivalent'] From 80144268836b7ca00d4d81e1c1822da6bd9988a7 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 14 Jun 2023 15:03:18 +0200 Subject: [PATCH 20/84] Fix ped for 'meng fracties' --- clarity_epp/export/ped.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/clarity_epp/export/ped.py b/clarity_epp/export/ped.py index 7dd42dd..33e5858 100644 --- a/clarity_epp/export/ped.py +++ b/clarity_epp/export/ped.py @@ -1,20 +1,21 @@ """Export ped functions.""" from genologics.entities import Process -from .. import get_sequence_name +from .. import get_sequence_name, get_sample_artifacts_from_pool def create_file(lims, process_id, output_file): """Create ped file.""" process = Process(lims, id=process_id) - samples = process.analytes()[0][0].samples - + sample_artifacts = get_sample_artifacts_from_pool(lims, process.analytes()[0][0]) ped_families = {} - for sample in samples: + for sample_artifact in sample_artifacts: + sample = sample_artifact.samples[0] # Asume all samples are identical. + if 'Dx Familienummer' in sample.udf and sample.udf['Dx Onderzoeksreden'] != 'Research': family = sample.udf['Dx Familienummer'] - sample_name = get_sequence_name(sample) + sample_name = get_sequence_name(sample_artifact) ped_sample = {'name': sample_name} if family not in ped_families: From 8d752a33fdd4ee28bf6c4af2d4c40dd7c3dd98a9 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 14 Jun 2023 15:04:42 +0200 Subject: [PATCH 21/84] Add get_sample_artifacts_from_pool --- clarity_epp/__init__.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/clarity_epp/__init__.py b/clarity_epp/__init__.py index 11ab01d..2c10e95 100644 --- a/clarity_epp/__init__.py +++ b/clarity_epp/__init__.py @@ -7,6 +7,8 @@ import smtplib import mimetypes +from genologics.entities import Artifact + def get_sequence_name(artifact): """Generate sequence name.""" @@ -23,6 +25,32 @@ def get_sequence_name(artifact): return sequence_name +def get_sample_artifacts_from_pool(lims, pool_artifact): + """Get sample artifacts from (sequence) pool.""" + sample_artifacts = [] + pool_artifact_demux = lims.get(pool_artifact.uri + '/demux') + for node in pool_artifact_demux.getiterator('artifact'): + if node.find('samples'): + if len(node.find('samples').findall('sample')) in [1, 2]: + sample_artifact = Artifact(lims, uri=node.attrib['uri']) + + # Check if sample_artifact with 2 samples are from the same person + if len(sample_artifact.samples) == 2: + if ( + 'Dx Persoons ID' in sample_artifact.samples[0].udf or + 'Dx Persoons ID' in sample_artifact.samples[1].udf or + sample_artifact.samples[0].udf['Dx Persoons ID'] == sample_artifact.samples[1].udf['Dx Persoons ID'] + ): + sample_artifacts.append(sample_artifact) + else: + sample_artifacts.append(sample_artifact) + return sample_artifacts + + + + + + def send_email(server, sender, receivers, subject, text, attachment=None): """Send emails.""" mail = MIMEMultipart() From c5ba1d1d035f8d98ef4abd995d454486a2acc4ab Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 14 Jun 2023 15:11:49 +0200 Subject: [PATCH 22/84] Fix merge table --- clarity_epp/export/merge.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/clarity_epp/export/merge.py b/clarity_epp/export/merge.py index 8f71555..bab9afd 100644 --- a/clarity_epp/export/merge.py +++ b/clarity_epp/export/merge.py @@ -1,26 +1,28 @@ """Export merge file functions.""" from genologics.entities import Process -from .. import get_sequence_name +from .. import get_sequence_name, get_sample_artifacts_from_pool def create_file(lims, process_id, output_file): """Create mege file.""" process = Process(lims, id=process_id) - samples = process.analytes()[0][0].samples + sample_artifacts = get_sample_artifacts_from_pool(lims, process.analytes()[0][0]) output_file.write('Sample\tMerge 1 Sample\tMerge 1 Sequencing Run\tMerge 2 Sample\tMerge 2 Sequencing Run\n') - for sample in samples: - sample_merge = [] - if 'Dx Mergen' in sample.udf and sample.udf['Dx Mergen']: - for udf in ['Dx Merge 1 Samplenaam', 'Dx Merge 1 Runnaam', 'Dx Merge 2 Samplenaam', 'Dx Merge 2 Runnaam']: - if udf in sample.udf: - sample_merge.append(sample.udf[udf]) - else: - sample_merge.append('') + # for sample in samples: + for sample_artifact in sample_artifacts: + for sample in sample_artifact.samples: # Asumme one sample per sample_artifact contains merge information + sample_merge = [] + if 'Dx Mergen' in sample.udf and sample.udf['Dx Mergen']: + for udf in ['Dx Merge 1 Samplenaam', 'Dx Merge 1 Runnaam', 'Dx Merge 2 Samplenaam', 'Dx Merge 2 Runnaam']: + if udf in sample.udf: + sample_merge.append(sample.udf[udf]) + else: + sample_merge.append('') - output_file.write('{sample}\t{merge}\n'.format( - sample=get_sequence_name(sample), - merge='\t'.join(sample_merge) - )) + output_file.write('{sample}\t{merge}\n'.format( + sample=get_sequence_name(sample_artifact), + merge='\t'.join(sample_merge) + )) From a8eea816da3ed352ef0b0bf843ed39afab8cefbf Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 14 Jun 2023 16:42:48 +0200 Subject: [PATCH 23/84] Fix samplesheet --- clarity_epp/export/illumina.py | 65 +++++++++++++++++++--------------- clarity_epp/export/ped.py | 2 +- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index 7135130..9bc00d6 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -1,10 +1,11 @@ """Illumina export functions.""" import operator import re +import csv from genologics.entities import Process, Artifact -from .. import get_sequence_name +from .. import get_sequence_name, get_sample_artifacts_from_pool import clarity_epp.export.utils import config @@ -28,8 +29,10 @@ def get_project(projects, urgent=False): # Parse families families = {} - for artifact in process.all_inputs(): - for sample in artifact.samples: + sample_artifacts = get_sample_artifacts_from_pool(lims, process.analytes()[0][0]) + + for sample_artifact in sample_artifacts: + for sample in sample_artifact.samples: if ( 'Dx Familienummer' in list(sample.udf) and 'Dx NICU Spoed' in list(sample.udf) and @@ -113,8 +116,9 @@ def get_project(projects, urgent=False): 'deviating': False } - # Add sample to family - families[family]['samples'].append(sample) + # Add sample_artifact to family + if sample_artifact not in families[family]['samples']: + families[family]['samples'].append(sample_artifact) # Get all project types and count samples project_types = {} @@ -144,18 +148,20 @@ def get_project(projects, urgent=False): # Urgent families / samples, skip deviating for family in [family for family in families.values() if family['urgent'] and not family['deviating']]: family_project = get_project(project_types[family['project_type']]['projects'], urgent=True) - for sample in family['samples']: - sample_sequence_name = get_sequence_name(sample) - sample_sequence_names[sample.name] = sample_sequence_name + for sample_artifact in family['samples']: + sample_sequence_name = get_sequence_name(sample_artifact) + for sample in sample_artifact.samples: + sample_sequence_names[sample.name] = sample_sequence_name sample_projects[sample_sequence_name] = family_project project_types[family['project_type']]['projects'][family_project] += 1 # Deviating families / samples for family in [family for family in families.values() if family['deviating']]: family_project = get_project(project_types[family['project_type']]['projects']) - for sample in family['samples']: - sample_sequence_name = get_sequence_name(sample) - sample_sequence_names[sample.name] = sample_sequence_name + for sample_artifact in family['samples']: + sample_sequence_name = get_sequence_name(sample_artifact) + for sample in sample_artifact.samples: + sample_sequence_names[sample.name] = sample_sequence_name sample_projects[sample_sequence_name] = family_project project_types[family['project_type']]['projects'][family_project] += 1 @@ -163,9 +169,10 @@ def get_project(projects, urgent=False): normal_families = [family for family in families.values() if not family['urgent'] and not family['deviating']] for family in sorted(normal_families, key=lambda fam: (len(fam['samples'])), reverse=True): family_project = get_project(project_types[family['project_type']]['projects']) - for sample in family['samples']: - sample_sequence_name = get_sequence_name(sample) - sample_sequence_names[sample.name] = sample_sequence_name + for sample_artifact in family['samples']: + sample_sequence_name = get_sequence_name(sample_artifact) + for sample in sample_artifact.samples: + sample_sequence_names[sample.name] = sample_sequence_name sample_projects[sample_sequence_name] = family_project project_types[family['project_type']]['projects'][family_project] += 1 @@ -182,21 +189,25 @@ def get_project(projects, urgent=False): samplesheet_artifact = Artifact(lims, id=artifact_id) file_id = samplesheet_artifact.files[0].id - for line in lims.get_file_contents(id=file_id).rstrip().split('\n'): - if line.startswith('[Settings]') and trim_last_base: - output_file.write('{line}\n'.format(line=line)) + # for line in lims.get_file_contents(id=file_id).rstrip().split('\n'): + for data in csv.reader( + lims.get_file_contents(id=file_id).rstrip().split('\n'), + quotechar='"', delimiter=',', quoting=csv.QUOTE_ALL, skipinitialspace=True + ): + if data[0] == '[Settings]' and trim_last_base: + output_file.write('{line}\n'.format(line=','.join(data))) output_file.write('Read1EndWithCycle,{value}\n'.format(value=process.udf['Read 1 Cycles']-1)) output_file.write('Read2EndWithCycle,{value}\n'.format(value=process.udf['Read 2 Cycles']-1)) settings_section = True - elif line.startswith('[Data]') and trim_last_base and not settings_section: + elif data[0] == '[Data]' and trim_last_base and not settings_section: output_file.write('[Settings]\n') output_file.write('Read1EndWithCycle,{value}\n'.format(value=process.udf['Read 1 Cycles']-1)) output_file.write('Read2EndWithCycle,{value}\n'.format(value=process.udf['Read 2 Cycles']-1)) - output_file.write('{line}\n'.format(line=line)) + output_file.write('{line}\n'.format(line=','.join(data))) - elif line.startswith('Sample_ID'): # Samples header line - sample_header = line.rstrip().split(',') + elif data[0] == 'Sample_ID': # Samples header line + sample_header = data sample_id_index = sample_header.index('Sample_ID') sample_name_index = sample_header.index('Sample_Name') sample_project_index = sample_header.index('Sample_Project') @@ -206,14 +217,12 @@ def get_project(projects, urgent=False): else: index_index = sample_header.index('index') - output_file.write('{line}\n'.format(line=line)) + output_file.write('{line}\n'.format(line=','.join(data))) elif sample_header: # Samples header seen, so continue with samples. - data = line.rstrip().split(',') - - # Fix sample name -> use sequence name - if data[sample_name_index] in sample_sequence_names: - data[sample_name_index] = sample_sequence_names[data[sample_name_index]] + sample_name = data[sample_name_index].split(',')[0] + if sample_name in sample_sequence_names: + data[sample_name_index] = sample_sequence_names[sample_name] # Set Sample_Project if data[sample_name_index] in sample_projects: @@ -228,4 +237,4 @@ def get_project(projects, urgent=False): output_file.write('{line}\n'.format(line=','.join(data))) else: # Leave other lines untouched. - output_file.write('{line}\n'.format(line=line)) + output_file.write('{line}\n'.format(line=','.join(data))) diff --git a/clarity_epp/export/ped.py b/clarity_epp/export/ped.py index 33e5858..836d7af 100644 --- a/clarity_epp/export/ped.py +++ b/clarity_epp/export/ped.py @@ -11,7 +11,7 @@ def create_file(lims, process_id, output_file): ped_families = {} for sample_artifact in sample_artifacts: - sample = sample_artifact.samples[0] # Asume all samples are identical. + sample = sample_artifact.samples[0] # Asume all samples metadata is identical. if 'Dx Familienummer' in sample.udf and sample.udf['Dx Onderzoeksreden'] != 'Research': family = sample.udf['Dx Familienummer'] From 29af6fa0fd9d82000ba5e2f973500fedfeca1886 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Fri, 16 Jun 2023 12:04:39 +0200 Subject: [PATCH 24/84] Added Fluent 480 samplesheet filling out purify --- clarity_epp.py | 2 +- clarity_epp/export/tecan.py | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/clarity_epp.py b/clarity_epp.py index 2c93f17..36b4668 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -343,7 +343,7 @@ def placement_tecan(args): parser_export_tecan = subparser_export.add_parser('tecan', help='Create tecan samplesheets', parents=[output_parser]) parser_export_tecan.add_argument('process_id', help='Clarity lims process id') - parser_export_tecan.add_argument('type', choices=['qc', 'purify_normalise'], help='Samplesheet type') + parser_export_tecan.add_argument('type', choices=['qc', 'purify_normalise', 'filling_out_purify'], help='Samplesheet type') parser_export_tecan.set_defaults(func=export_tecan) parser_export_workflow = subparser_export.add_parser( diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 2708dc7..6c8f00f 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -39,3 +39,48 @@ def samplesheet(lims, process_id, type, output_file): well=well, index=clarity_epp.export.utils.get_well_index(well, one_based=True) )) + + elif type == 'filling_out_purify': + output_file.write( + 'SourceTubeID;VolSample;VolWater;PositionIndex;MengID\n' + ) + for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): + artifact = well_plate[well] + sample_mix = False + if len(artifact.samples) > 1: + sample_mix = True + sample_volumes = {} + water_volumes = {} + mix_names = {} + messages = {} + for sample in artifact.samples: + messages[sample] = "" + conc = sample.udf['Dx Concentratie (ng/ul)'] + if sample_mix: + dividend = 880 + max_volume = 30 + mix_names[sample] = artifact.name + else: + dividend = 1760 + max_volume = 60 + mix_names[sample] = sample.udf['Dx Monsternummer'] + calc_sample = dividend / conc + if calc_sample < 4: + volume_sample = 4 + elif calc_sample > max_volume: + volume_sample = max_volume + messages[sample] = ("Conc. too low - volume= {calc_sample} ul".format(calc_sample=calc_sample)) + else: + volume_sample = calc_sample + sample_volumes[sample] = volume_sample + water_volumes[sample] = max_volume - volume_sample + for sample in artifact.samples: + output_file.write('{sample};{volume_sample:.2f};{volume_water:.2f};{index};{name};{empty};{message}\n'.format( + sample=sample.udf['Dx Fractienummer'], + volume_sample=sample_volumes[sample], + volume_water=water_volumes[sample], + index=clarity_epp.export.utils.get_well_index(well, one_based=True), + name=mix_names[sample], + empty="", + message=messages[sample] + )) \ No newline at end of file From 2f1b53409103317229223754beb1a92056be0ac8 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 16 Jun 2023 15:17:56 +0200 Subject: [PATCH 25/84] Fix move to post seq workflow for 'meng fracties' --- clarity_epp/placement/pool.py | 45 ++++++++++++++++------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/clarity_epp/placement/pool.py b/clarity_epp/placement/pool.py index e53e003..0ef3170 100644 --- a/clarity_epp/placement/pool.py +++ b/clarity_epp/placement/pool.py @@ -1,8 +1,8 @@ """Pool placement functions.""" -from genologics.entities import Artifact, Process, Workflow, Step +from genologics.entities import Process, Workflow, Step -from .. import get_sequence_name +from .. import get_sequence_name, get_sample_artifacts_from_pool import config @@ -13,12 +13,12 @@ def unpooling(lims, process_id): if process.step.actions.next_actions[0]['action'] == 'complete': # Only unpool complete sequencing runs. pool_artifact = process.all_inputs()[0] pool_artifact_parent_process = pool_artifact.parent_process - pool_artifact_demux = lims.get(pool_artifact.uri + '/demux') run_id = pool_artifact.name # Assume run id is set as pool name using placement/artifact/set_runid_name sample_artifacts = [] # sample artifacts before pooling sample_projects = {} + # Get sample projects from samplesheet for artifact in pool_artifact_parent_process.result_files(): if (artifact.name == 'SampleSheet csv' or artifact.name == 'Sample Sheet') and artifact.files: file_id = artifact.files[0].id @@ -34,28 +34,23 @@ def unpooling(lims, process_id): elif project_index and len(data) >= project_index: sample_projects[data[sample_index]] = data[project_index] - for node in pool_artifact_demux.getiterator('artifact'): - if node.find('samples'): - if len(node.find('samples').findall('sample')) == 1: - # if len(node.find('samples').findall('sample')) in [1, 2]: - sample_artifact = Artifact(lims, uri=node.attrib['uri']) - sample = sample_artifact.samples[0] # 1 sample per artifact. - - # Skip non dx samples? - # Check if pool with 2 samples come from same person. - - # Get sample sequencing run and project from samplesheet - sample_artifact.udf['Dx Sequencing Run ID'] = run_id - # Use sample.name for external (clarity_portal) samples - if 'Sample Type' in sample.udf and 'library' in sample.udf['Sample Type']: - sample_artifact.udf['Dx Sequencing Run Project'] = sample_projects[sample.name] - else: # Use sample_artifact.name for Dx samples (upload via Helix) - sample_artifact.udf['Dx Sequencing Run Project'] = sample_projects[sample_artifact.name] - sample_artifact.put() - # Only move DX production samples to post sequencing workflow - if sample_artifact.samples[0].project and sample_artifact.samples[0].project.udf['Application'] == 'DX': - sample_artifacts.append(sample_artifact) - # print(sample_artifacts) + # Parse sequencing run samples and move Dx samples to post sequencing workflow + for sample_artifact in get_sample_artifacts_from_pool(lims, pool_artifact): + sample = sample_artifact.samples[0] # Asume all samples metadata is identical. + + # Set sample sequencing run and project + sample_artifact.udf['Dx Sequencing Run ID'] = run_id + # Use sample.name for external (clarity_portal) samples + if 'Sample Type' in sample.udf and 'library' in sample.udf['Sample Type']: + sample_artifact.udf['Dx Sequencing Run Project'] = sample_projects[sample.name] + else: # Use sample_artifact.name for Dx samples (upload via Helix) + sample_artifact.udf['Dx Sequencing Run Project'] = sample_projects[sample_artifact.name] + sample_artifact.put() + + # Only move DX production samples to post sequencing workflow + if sample.project and sample.project.udf['Application'] == 'DX': + sample_artifacts.append(sample_artifact) + lims.route_artifacts(sample_artifacts, workflow_uri=Workflow(lims, id=config.post_sequencing_workflow).uri) From 206205390a7883a35483b93e58de57c7564f2197 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 16 Jun 2023 16:20:01 +0200 Subject: [PATCH 26/84] Remove mip code from route post bioinf --- clarity_epp/placement/artifact.py | 48 ++++--------------------------- config.py | 11 ++----- 2 files changed, 9 insertions(+), 50 deletions(-) diff --git a/clarity_epp/placement/artifact.py b/clarity_epp/placement/artifact.py index 05cd257..2014cbc 100644 --- a/clarity_epp/placement/artifact.py +++ b/clarity_epp/placement/artifact.py @@ -43,48 +43,12 @@ def route_to_workflow(lims, process_id, workflow): ] if workflow == 'post_bioinf': - stoftest_artifacts = {} - for artifact in artifacts_completed: - sample = artifact.samples[0] # Asume 1 sample per artifact - - # Add stoftest to dict - if sample.udf['Dx Stoftest code'] not in stoftest_artifacts: - stoftest_artifacts[sample.udf['Dx Stoftest code']] = {'single': [], 'trio': []} - - # Remove research artifacts - if sample.udf['Dx Stoftest code'] != config.stoftestcode_research: - # Lookup trio samples - # if parent check family members if parent belongs to trio - if sample.udf['Dx Familie status'] == 'Ouder': - parent_status = 'single' - for family_sample in lims.get_samples(udf={'Dx Familienummer': sample.udf['Dx Familienummer']}): - if( - 'Dx Gerelateerde onderzoeken' in family_sample.udf - and sample.udf['Dx Onderzoeknummer'] in family_sample.udf['Dx Gerelateerde onderzoeken'] - and len(family_sample.udf['Dx Gerelateerde onderzoeken'].split(';')) >= 2 - ): - parent_status = 'trio' - stoftest_artifacts[sample.udf['Dx Stoftest code']][parent_status].append(artifact) - # If child check if part of trio - elif( - 'Dx Gerelateerde onderzoeken' in sample.udf - and len(sample.udf['Dx Gerelateerde onderzoeken'].split(';')) >= 2 - ): - stoftest_artifacts[sample.udf['Dx Stoftest code']]['trio'].append(artifact) - # Else not trio - else: - stoftest_artifacts[sample.udf['Dx Stoftest code']]['single'].append(artifact) - - for stoftest, route_artifacts in stoftest_artifacts.items(): - if route_artifacts['single']: - workflow = Workflow(lims, id=config.post_bioinf_workflow[stoftest]['single']['workflow']) - stage = workflow.stages[config.post_bioinf_workflow[stoftest]['single']['stage']] - lims.route_artifacts(route_artifacts['single'], workflow_uri=workflow.uri, stage_uri=stage.uri) - - if route_artifacts['trio']: - workflow = Workflow(lims, id=config.post_bioinf_workflow[stoftest]['trio']['workflow']) - stage = workflow.stages[config.post_bioinf_workflow[stoftest]['trio']['stage']] - lims.route_artifacts(route_artifacts['trio'], workflow_uri=workflow.uri, stage_uri=stage.uri) + # Remove research artifacts + route_artifacts = [ + artifact for artifact in artifacts_completed + if artifact.samples[0].udf['Dx Stoftest code'] != config.research_stoftest_code # Asume all samples metadata is identical. + ] + lims.route_artifacts(route_artifacts, workflow_uri=Workflow(lims, id=config.post_bioinf_workflow).uri) elif workflow == 'sequencing': lims.route_artifacts(artifacts_completed, workflow_uri=Workflow(lims, id=config.sequencing_workflow).uri) diff --git a/config.py b/config.py index 6e5a22a..c08b058 100755 --- a/config.py +++ b/config.py @@ -23,8 +23,8 @@ stoftestcode_research = 'NGS_023' stoftestcode_workflow = { - stoftestcode_wes: '1654', # DEV Dx Exoom Magnis v1.2 - stoftestcode_mip: '1651', # DEV Dx smMIP v1.2 + stoftestcode_wes: '1802', # DEV Dx Exoom Magnis v2.0 + stoftestcode_mip: '1802', # DEV Dx Exoom Magnis v2.0 } # Export meetw protocol steps WES @@ -83,12 +83,7 @@ # Post sequencing workflow sequencing_workflow = '1701' # DEV Dx Illumina Sequencing v1.2 post_sequencing_workflow = '902' # DEV Dx Bioinformatica analyses v1.0 -post_bioinf_workflow = { # Contains workflow and workflow stage (number) for single or trio samples - # WES : DEV Dx NGS WES onderzoeken afronden v1.1 - stoftestcode_wes: {'single': {'workflow': '1401', 'stage': 0}, 'trio': {'workflow': '1401', 'stage': 1}}, - # MIP : DEV Dx NGS smMIP onderzoeken afronden v1.0 - stoftestcode_mip: {'single': {'workflow': '1202', 'stage': 0}, 'trio': {'workflow': '1202', 'stage': 0}} -} +post_bioinf_workflow = '1803' # DEV Dx NGS WES onderzoeken afronden v2.0 # Research Onderzoeksindicatie research_onderzoeksindicatie_project = { From ffa13db13843e174fc15be101c396fe5e380c4d9 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 16 Jun 2023 16:22:48 +0200 Subject: [PATCH 27/84] Update config name --- clarity_epp/placement/artifact.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/placement/artifact.py b/clarity_epp/placement/artifact.py index 2014cbc..a070342 100644 --- a/clarity_epp/placement/artifact.py +++ b/clarity_epp/placement/artifact.py @@ -46,7 +46,7 @@ def route_to_workflow(lims, process_id, workflow): # Remove research artifacts route_artifacts = [ artifact for artifact in artifacts_completed - if artifact.samples[0].udf['Dx Stoftest code'] != config.research_stoftest_code # Asume all samples metadata is identical. + if artifact.samples[0].udf['Dx Stoftest code'] != config.stoftestcode_research # Asume all samples metadata is identical. ] lims.route_artifacts(route_artifacts, workflow_uri=Workflow(lims, id=config.post_bioinf_workflow).uri) From cf4682c8ac90ff6243dc9c1f9bb5fdd6899ce246 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 19 Jun 2023 09:16:48 +0200 Subject: [PATCH 28/84] Use artifact to get sequence name --- clarity_epp/export/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/export/workflow.py b/clarity_epp/export/workflow.py index a399c4d..840c00b 100644 --- a/clarity_epp/export/workflow.py +++ b/clarity_epp/export/workflow.py @@ -100,7 +100,7 @@ def helix_magnis(lims, process_id, output_file): meetw_sampleprep=meetw_sampleprep, meetw_sampleprep_herh=meetw_sampleprep_herh, meetw_seq=meetw_seq, meetw_seq_herh=meetw_seq_herh, meetw_bfx='J', - sample_name=get_sequence_name(sample), + sample_name=get_sequence_name(artifact), vcf_file=gatk_vcf, cnv_vcf_file=exomedepth_vcf, ) From a6e554d60c6baee2a76263403ee0c35b442d56ce Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Wed, 21 Jun 2023 11:41:25 +0200 Subject: [PATCH 29/84] Acquiring multiplier with regex --- clarity_epp/export/manual_pipetting.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 44745f1..509210b 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -691,8 +691,9 @@ def samplesheet_pool_magnis_pools(lims, process_id, output_file): # set up multiplier multiplier = 1 - if 'Run type' in process.udf and 'Sp/S1' in process.udf['Run type']: - multiplier = 2.5 + if 'Run type' in process.udf: + run_type = re.search(r'\(\*.+\)' ,process.udf['Run type']) + multiplier = float(run_type.string[run_type.start()+2:run_type.end()-1]) # print header output_file.write('Pool\tContainer\tSample count\tVolume (ul)\n') From 93d93223ef4ed97feea081dcb7e8ad9a72c05379 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Wed, 21 Jun 2023 11:50:16 +0200 Subject: [PATCH 30/84] Added if --- clarity_epp/export/manual_pipetting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 509210b..f496e89 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -693,7 +693,8 @@ def samplesheet_pool_magnis_pools(lims, process_id, output_file): multiplier = 1 if 'Run type' in process.udf: run_type = re.search(r'\(\*.+\)' ,process.udf['Run type']) - multiplier = float(run_type.string[run_type.start()+2:run_type.end()-1]) + if run_type: + multiplier = float(run_type.string[run_type.start()+2:run_type.end()-1]) # print header output_file.write('Pool\tContainer\tSample count\tVolume (ul)\n') From d3776fcd7a576fd50eff8343af72412849481fe1 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Wed, 21 Jun 2023 16:48:20 +0200 Subject: [PATCH 31/84] Added Tecan Fluent 480 normalization samplesheet --- clarity_epp.py | 4 +++- clarity_epp/export/tecan.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/clarity_epp.py b/clarity_epp.py index 36b4668..d539e91 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -343,7 +343,9 @@ def placement_tecan(args): parser_export_tecan = subparser_export.add_parser('tecan', help='Create tecan samplesheets', parents=[output_parser]) parser_export_tecan.add_argument('process_id', help='Clarity lims process id') - parser_export_tecan.add_argument('type', choices=['qc', 'purify_normalise', 'filling_out_purify'], help='Samplesheet type') + parser_export_tecan.add_argument( + 'type', choices=['qc', 'purify_normalise', 'filling_out_purify', 'normalise'], help='Samplesheet type' + ) parser_export_tecan.set_defaults(func=export_tecan) parser_export_workflow = subparser_export.add_parser( diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 6c8f00f..5d7631e 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -83,4 +83,15 @@ def samplesheet(lims, process_id, type, output_file): name=mix_names[sample], empty="", message=messages[sample] + )) + + elif type == 'normalise': + output_file.write('SourceTubeID;PositionID;PositionIndex\n') + for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): + artifact = well_plate[well] + for sample in artifact.samples: + output_file.write('{sample};{well};{index}\n'.format( + sample=sample.udf['Dx Fractienummer'], + well=well, + index=clarity_epp.export.utils.get_well_index(well, one_based=True) )) \ No newline at end of file From 0bd605fde42dac2ee88487fdb34570b9502e45e9 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 23 Jun 2023 11:56:56 +0200 Subject: [PATCH 32/84] Update helix import with new UDF fields --- clarity_epp/upload/samples.py | 45 ++++++++++++++++++++++++++++++----- config.py | 4 +++- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 93efb5d..2cebff2 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -8,6 +8,7 @@ from .. import send_email import clarity_epp.upload.utils +import config def from_helix(lims, email_settings, input_file): @@ -42,7 +43,7 @@ def from_helix(lims, email_settings, input_file): else: subject = "ERROR Lims Helix Upload: {0}".format(project_name) message = "Duplicate project / werklijst. Samples not loaded." - send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) + # send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) sys.exit(message) container_type = Containertype(lims, id='2') # Tube @@ -75,6 +76,8 @@ def from_helix(lims, email_settings, input_file): 'Dx Protocolomschrijving': {'column': 'Protocolomschrijving'}, 'Dx Einddatum': {'column': 'Einddatum'}, 'Dx Gerelateerde onderzoeken': {'column': 'Gerelateerde onderzoeken'}, + 'Dx gerelateerd aan oz': {'column': 'Gerelateerd aan'}, + 'Dx gerelateerde oz #': {'column': 'Aantal gerelateerde onderzoeken.'}, } header = input_file.readline().rstrip().split(',') # expect header on first line for udf in udf_column: @@ -99,6 +102,8 @@ def from_helix(lims, email_settings, input_file): udf_data[udf] = bool(data[udf_column[udf]['index']].strip()) elif udf == 'Dx Concentratie (ng/ul)': udf_data[udf] = data[udf_column[udf]['index']].replace(',', '.') + if udf_data[udf]: + udf_data[udf] = float(udf_data[udf]) elif udf in ['Dx Monsternummer', 'Dx Fractienummer']: udf_data[udf] = clarity_epp.upload.utils.transform_sample_name(data[udf_column[udf]['index']]) elif udf == 'Dx Gerelateerde onderzoeken': @@ -123,6 +128,12 @@ def from_helix(lims, email_settings, input_file): else: udf_data['Dx Handmatig'] = False + # Set 'Dx norm. manueel' udf + if udf_data['Dx Concentratie (ng/ul)'] and udf_data['Dx Concentratie (ng/ul)'] < 29.3: + udf_data['Dx norm. manueel'] = True + else: + udf_data['Dx norm. manueel'] = False + # Set 'Dx Familie status' udf if udf_data['Dx Onderzoeksreden'] == 'Bevestiging diagnose': udf_data['Dx Familie status'] = 'Kind' @@ -170,17 +181,37 @@ def from_helix(lims, email_settings, input_file): # Set NICU status for sample if related sample is NICU else: for related_sample in lims.get_samples(udf={'Dx Familienummer': udf_data['Dx Familienummer']}): - if( + if ( 'Dx Gerelateerde onderzoeken' in related_sample.udf and udf_data['Dx Onderzoeknummer'] in related_sample.udf['Dx Gerelateerde onderzoeken'] ): udf_data['Dx NICU Spoed'] = related_sample.udf['Dx NICU Spoed'] + # Set 'Dx Mengfractie' + if udf_data['Dx Stoftest code'] == config.stoftestcode_wes_duplo: + udf_data['Dx Mengfractie'] = True + for duplo_sample in lims.get_samples(udf={ + 'Dx Persoons ID': udf_data['Dx Persoons ID'], + 'Dx Onderzoeknummer': udf_data['Dx Onderzoeknummer'] + }): + duplo_sample.udf['Dx Mengfractie'] = True + duplo_sample.put() + + elif udf_data['Dx Stoftest code'] == config.stoftestcode_wes: + if lims.get_samples(udf={ + 'Dx Persoons ID': udf_data['Dx Persoons ID'], + 'Dx Onderzoeknummer': udf_data['Dx Onderzoeknummer'], + 'Dx Mengfractie': True + }): + udf_data['Dx Mengfractie'] = True + else: + udf_data['Dx Mengfractie'] = False + # Check other samples from patient sample_list = lims.get_samples(udf={'Dx Persoons ID': udf_data['Dx Persoons ID']}) for sample in sample_list: if sample.udf['Dx Monsternummer'] == udf_data['Dx Monsternummer']: - if( + if ( sample.udf['Dx Protocolomschrijving'] in udf_data['Dx Protocolomschrijving'] and sample.udf['Dx Foetus'] == udf_data['Dx Foetus'] ): @@ -193,7 +224,7 @@ def from_helix(lims, email_settings, input_file): '{sample}: Monsternummer hetzelfde, Protocolomschrijving uniek.'.format(sample=sample.name), udf_data['Dx Import warning'] ]) - elif( + elif ( sample.udf['Dx Protocolomschrijving'] in udf_data['Dx Protocolomschrijving'] and sample.udf['Dx Foetus'] == udf_data['Dx Foetus'] ): @@ -222,5 +253,7 @@ def from_helix(lims, email_settings, input_file): udf_data['Dx Stoftest code'] ) - # Send final email - send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) + # # Send final email + # send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) + print(subject) + print(message) diff --git a/config.py b/config.py index 6e5a22a..701125e 100755 --- a/config.py +++ b/config.py @@ -19,11 +19,13 @@ # Import samples: stoftestcode to workflow stoftestcode_wes = 'NGS_025' +stoftestcode_wes_duplo = 'NGS_028' stoftestcode_mip = 'NGS_027' stoftestcode_research = 'NGS_023' stoftestcode_workflow = { - stoftestcode_wes: '1654', # DEV Dx Exoom Magnis v1.2 + stoftestcode_wes: '1802', # DEV Dx Exoom Magnis v2.0 + stoftestcode_wes_duplo: '1802', # DEV Dx Exoom Magnis v2.0 stoftestcode_mip: '1651', # DEV Dx smMIP v1.2 } From 4bb29734a03e5c563d12765a0857d686e142e986 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 23 Jun 2023 11:57:39 +0200 Subject: [PATCH 33/84] Remove debug code --- clarity_epp/upload/samples.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 2cebff2..00a593d 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -43,7 +43,7 @@ def from_helix(lims, email_settings, input_file): else: subject = "ERROR Lims Helix Upload: {0}".format(project_name) message = "Duplicate project / werklijst. Samples not loaded." - # send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) + send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) sys.exit(message) container_type = Containertype(lims, id='2') # Tube @@ -253,7 +253,6 @@ def from_helix(lims, email_settings, input_file): udf_data['Dx Stoftest code'] ) - # # Send final email - # send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) - print(subject) - print(message) + # Send final email + send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) + From 2b0116d061288466f8ffe3d5c40fcb3db4bae74f Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 23 Jun 2023 11:58:30 +0200 Subject: [PATCH 34/84] pep8 --- clarity_epp/upload/samples.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 00a593d..36aadac 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -255,4 +255,3 @@ def from_helix(lims, email_settings, input_file): # Send final email send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) - From f355dadcab1a2c113d9d07c494637eb58766e6ae Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 26 Jun 2023 11:14:56 +0200 Subject: [PATCH 35/84] Use stoftestcodes --- clarity_epp/upload/samples.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 36aadac..91ba25c 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -192,7 +192,8 @@ def from_helix(lims, email_settings, input_file): udf_data['Dx Mengfractie'] = True for duplo_sample in lims.get_samples(udf={ 'Dx Persoons ID': udf_data['Dx Persoons ID'], - 'Dx Onderzoeknummer': udf_data['Dx Onderzoeknummer'] + 'Dx Onderzoeknummer': udf_data['Dx Onderzoeknummer'], + 'Dx Stoftest code': config.stoftestcode_wes, }): duplo_sample.udf['Dx Mengfractie'] = True duplo_sample.put() @@ -201,7 +202,7 @@ def from_helix(lims, email_settings, input_file): if lims.get_samples(udf={ 'Dx Persoons ID': udf_data['Dx Persoons ID'], 'Dx Onderzoeknummer': udf_data['Dx Onderzoeknummer'], - 'Dx Mengfractie': True + 'Dx Stoftest code': config.stoftestcode_wes_duplo, }): udf_data['Dx Mengfractie'] = True else: @@ -254,4 +255,6 @@ def from_helix(lims, email_settings, input_file): ) # Send final email + print(subject) + print(message) send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) From 8eac26ffbf9d844bc1f9fe00c40cbaac0ddc38ea Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 26 Jun 2023 11:15:41 +0200 Subject: [PATCH 36/84] Remove print --- clarity_epp/upload/samples.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 91ba25c..0f73e80 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -255,6 +255,4 @@ def from_helix(lims, email_settings, input_file): ) # Send final email - print(subject) - print(message) send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) From fec9000cf787f563852f3d329f51fb5f71548ac4 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 27 Jun 2023 14:08:08 +0200 Subject: [PATCH 37/84] check for trim_last_base --- clarity_epp/export/illumina.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index 109ed93..fd183c4 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -185,7 +185,7 @@ def get_project(projects, urgent=False): # Setup custom settings custom_settings = '' - if conversion_tool == 'bcl2fastq': + if conversion_tool == 'bcl2fastq' and trim_last_base: custom_settings = ( 'Read1EndWithCycle,{read_1_value}\n' 'Read2EndWithCycle,{read_2_value}\n' From 04e37cc84422058e21a392db901622d33e5e39ca Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 27 Jun 2023 21:22:24 +0200 Subject: [PATCH 38/84] Enable FindAdaptersWithIndels --- templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv | 1 + templates/NovaSeq_BCLCONVERT_Samplesheet.csv | 1 + 2 files changed, 2 insertions(+) diff --git a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv index d0a9ca3..73d2674 100644 --- a/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Reverse_Complement_Samplesheet.csv @@ -13,6 +13,7 @@ ${PROCESS.UDF.Read 2 Cycles} [Settings] AdapterRead1,${PROCESS.UDF.Adapter} AdapterRead2,${PROCESS.UDF.Adapter Read 2} +FindAdaptersWithIndels,true [Data]
diff --git a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv index 7dfb88f..96c0804 100644 --- a/templates/NovaSeq_BCLCONVERT_Samplesheet.csv +++ b/templates/NovaSeq_BCLCONVERT_Samplesheet.csv @@ -14,6 +14,7 @@ ${PROCESS.UDF.Read 2 Cycles} [Settings] AdapterRead1,${PROCESS.UDF.Adapter} AdapterRead2,${PROCESS.UDF.Adapter Read 2} +FindAdaptersWithIndels,true [Data]
From 2e016dc373748f3bf946092f590dcc53b3a55795 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 6 Jul 2023 15:17:21 +0200 Subject: [PATCH 39/84] Check for udf_data['Dx Gerelateerde onderzoeken'] --- clarity_epp/upload/samples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 0f73e80..1a69912 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -173,7 +173,7 @@ def from_helix(lims, email_settings, input_file): udf_data['Dx Familienummer'] = udf_data['Dx Familienummer'].split('/')[-1].strip(' ') # Set NICU status for related samples using Dx Gerelateerde onderzoeken - if udf_data['Dx NICU Spoed']: + if udf_data['Dx NICU Spoed'] and udf_data['Dx Gerelateerde onderzoeken']: for related_research in udf_data['Dx Gerelateerde onderzoeken'].split(';'): for related_sample in lims.get_samples(udf={'Dx Onderzoeknummer': related_research}): related_sample.udf['Dx NICU Spoed'] = udf_data['Dx NICU Spoed'] From 84f59158c7dece7505ee5eaf32d1c301250efb13 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 7 Jul 2023 11:04:54 +0200 Subject: [PATCH 40/84] Round volume --- clarity_epp/export/manual_pipetting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index f496e89..d882e27 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -709,7 +709,7 @@ def samplesheet_pool_magnis_pools(lims, process_id, output_file): sample_count += 1 output_file.write( - '{pool}\t{container}\t{sample_count}\t{volume}\n'.format( + '{pool}\t{container}\t{sample_count}\t{volume:.2f}\n'.format( pool=input_artifact.name, container=input_artifact.container.name, sample_count=sample_count, From 2c1e20f941aab3c2f5a122e1fd535f011957eda0 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 7 Jul 2023 13:29:42 +0200 Subject: [PATCH 41/84] Skip related nicu check for parents --- clarity_epp/upload/samples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 1a69912..cbf1898 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -173,7 +173,7 @@ def from_helix(lims, email_settings, input_file): udf_data['Dx Familienummer'] = udf_data['Dx Familienummer'].split('/')[-1].strip(' ') # Set NICU status for related samples using Dx Gerelateerde onderzoeken - if udf_data['Dx NICU Spoed'] and udf_data['Dx Gerelateerde onderzoeken']: + if udf_data['Dx NICU Spoed'] and not udf_data['Dx Onderzoeksreden'] == 'Informativiteitstest': for related_research in udf_data['Dx Gerelateerde onderzoeken'].split(';'): for related_sample in lims.get_samples(udf={'Dx Onderzoeknummer': related_research}): related_sample.udf['Dx NICU Spoed'] = udf_data['Dx NICU Spoed'] From c08584e4b298b499a6db5179beb4b3dd01f4e508 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 7 Jul 2023 15:06:23 +0200 Subject: [PATCH 42/84] Only set norm. manueel for samples with conc --- clarity_epp/upload/samples.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index cbf1898..01b1ea1 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -128,11 +128,12 @@ def from_helix(lims, email_settings, input_file): else: udf_data['Dx Handmatig'] = False - # Set 'Dx norm. manueel' udf - if udf_data['Dx Concentratie (ng/ul)'] and udf_data['Dx Concentratie (ng/ul)'] < 29.3: - udf_data['Dx norm. manueel'] = True - else: - udf_data['Dx norm. manueel'] = False + # Set 'Dx norm. manueel' udf for samples with Dx Concentratie (ng/ul) + if udf_data['Dx Concentratie (ng/ul)']: + if udf_data['Dx Concentratie (ng/ul)'] < 29.3: + udf_data['Dx norm. manueel'] = True + else: + udf_data['Dx norm. manueel'] = False # Set 'Dx Familie status' udf if udf_data['Dx Onderzoeksreden'] == 'Bevestiging diagnose': From 9952346ccc3afc56a1837728c1e05245cbee71b7 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Mon, 10 Jul 2023 13:02:34 +0200 Subject: [PATCH 43/84] Add samplesheet normalization mix samples --- clarity_epp.py | 4 +- clarity_epp/export/manual_pipetting.py | 85 ++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/clarity_epp.py b/clarity_epp.py index d539e91..0985e03 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -92,6 +92,8 @@ def export_manual_pipetting(args): clarity_epp.export.manual_pipetting.samplesheet_pool_samples(lims, args.process_id, args.output_file) elif args.type == 'pool_magnis_pools': clarity_epp.export.manual_pipetting.samplesheet_pool_magnis_pools(lims, args.process_id, args.output_file) + elif args.type == 'normalization_mix': + clarity_epp.export.manual_pipetting.samplesheet_normalization_mix(lims, args.process_id, args.output_file) def export_ped_file(args): @@ -298,7 +300,7 @@ def placement_tecan(args): choices=[ 'purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization', 'capture', 'exonuclease', 'pcr_exonuclease', 'mip_multiplex_pool', 'mip_dilute_pool', 'pool_samples', - 'pool_magnis_pools' + 'pool_magnis_pools', 'normalization_mix' ], help='Samplesheet type' ) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 5a5fbd2..3b7ba3a 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -726,3 +726,88 @@ def samplesheet_pool_magnis_pools(lims, process_id, output_file): volume=sample_count * 1.25 ) ) + + +def samplesheet_normalization_mix(lims, process_id, output_file): + """"Create manual pipetting samplesheet for normalizing mix fraction samples.""" + process = Process(lims, id=process_id) + + output_file.write( + 'Monsternummer\tConcentratie (ng/ul)\tVolume sample (ul)\tVolume low TE (ul)\tContainer\tWell\n' + ) + + samples = {} + + # Find all QC process types + qc_process_types = clarity_epp.export.utils.get_process_types(lims, ['Dx Qubit QC', 'Dx Tecan']) + + # Find concentration in last QC process + for input_artifact in process.all_inputs(): + for input_sample in input_artifact.samples: + qc_processes = lims.get_processes(type=qc_process_types, inputartifactlimsid=input_artifact.id) + if qc_processes: + qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] + for qc_artifact in qc_process.outputs_per_input(input_artifact.id): + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + + else: + parent_process = input_artifact.parent_process + for parent_artifact in parent_process.all_inputs(): + qc_processes = lims.get_processes(type=qc_process_types, inputartifactlimsid=parent_artifact.id) + if qc_processes: + qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] + for qc_artifact in qc_process.outputs_per_input(parent_artifact.id): + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + + samples[input_sample.udf['Dx Monsternummer']] = {'conc': concentration} + + # Calculation of pipetting volumes + for input_artifact in process.all_inputs(): + dividend = 400 + minuend = 50 + sample_mix = False + + if len(input_artifact.samples) > 1: + sample_mix = True + + if sample_mix: + dividend = 200 + minuend = 25 + + for input_sample in input_artifact.samples: + sample_volume = dividend / samples[input_sample.udf['Dx Monsternummer']]['conc'] + low_te_volume = minuend - sample_volume + samples[input_sample.udf['Dx Monsternummer']]['sample_volume'] = sample_volume + samples[input_sample.udf['Dx Monsternummer']]['low_te_volume'] = low_te_volume + + # Compose output per sample in well + output = {} + for output_artifact in process.all_outputs(): + if output_artifact.type == 'Analyte': + for output_sample in output_artifact.samples: + monster = output_sample.udf['Dx Monsternummer'] + well = ''.join(output_artifact.location[1].split(':')) + output_data = ( + '{sample}\t{concentration:.2f}\t{sample_volume:.2f}\t{low_te_volume:.2f}\t' + '{container}\t{well}\n' + ).format( + sample=monster, + concentration=samples[monster]['conc'], + sample_volume=samples[monster]['sample_volume'], + low_te_volume=samples[monster]['low_te_volume'], + container=output_artifact.location[0].name, + well=well + ) + if well in output: + output[well][monster] = output_data + else: + output[well] = {monster: output_data} + + # Write output file per sample sorted for well + for well in clarity_epp.export.utils.sort_96_well_plate(output.keys()): + for sample in output[well]: + output_file.write('{output}\n').format(output=output[well][sample]) \ No newline at end of file From d56b3eed427532b45f1e02bd75335082602980af Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Mon, 10 Jul 2023 13:20:20 +0200 Subject: [PATCH 44/84] fix output --- clarity_epp/export/manual_pipetting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 3b7ba3a..21d4d1a 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -810,4 +810,4 @@ def samplesheet_normalization_mix(lims, process_id, output_file): # Write output file per sample sorted for well for well in clarity_epp.export.utils.sort_96_well_plate(output.keys()): for sample in output[well]: - output_file.write('{output}\n').format(output=output[well][sample]) \ No newline at end of file + output_file.write(output[well][sample]) \ No newline at end of file From 4d5e685218bbecdebcdf9e9e977fc10b6d87373c Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 10 Jul 2023 15:23:35 +0200 Subject: [PATCH 45/84] Update import warnings --- clarity_epp/upload/samples.py | 58 +++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 01b1ea1..9a47826 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -86,6 +86,7 @@ def from_helix(lims, email_settings, input_file): # Setup email subject = "Lims Helix Upload: {0}".format(project_name) message = "Project: {0}\n\nSamples:\n".format(project_name) + sample_messages = {} # Parse samples for line in input_file: @@ -191,21 +192,49 @@ def from_helix(lims, email_settings, input_file): # Set 'Dx Mengfractie' if udf_data['Dx Stoftest code'] == config.stoftestcode_wes_duplo: udf_data['Dx Mengfractie'] = True - for duplo_sample in lims.get_samples(udf={ + + # Find WES sample(s) + duplo_samples = lims.get_samples(udf={ 'Dx Persoons ID': udf_data['Dx Persoons ID'], 'Dx Onderzoeknummer': udf_data['Dx Onderzoeknummer'], 'Dx Stoftest code': config.stoftestcode_wes, - }): - duplo_sample.udf['Dx Mengfractie'] = True - duplo_sample.put() + }) + if duplo_samples: # Set duplo status for WES samples + for duplo_sample in duplo_samples: + duplo_sample.udf['Dx Mengfractie'] = True + duplo_sample.put() + # Check Dx Monsternummer + if duplo_sample.udf['Dx Monsternummer'] == udf_data['Dx Monsternummer']: + udf_data['Dx Import warning'] = '; '.join([ + 'WES en WES_duplo zelfde monster ({sample}).'.format(sample=duplo_sample.name), + udf_data['Dx Import warning'] + ]) + else: # Set import warning if no WES samples found + udf_data['Dx Import warning'] = '; '.join(['Alleen WES_duplo', udf_data['Dx Import warning']]) elif udf_data['Dx Stoftest code'] == config.stoftestcode_wes: - if lims.get_samples(udf={ + # Find WES_duplo sample(s) + duplo_samples = lims.get_samples(udf={ 'Dx Persoons ID': udf_data['Dx Persoons ID'], 'Dx Onderzoeknummer': udf_data['Dx Onderzoeknummer'], 'Dx Stoftest code': config.stoftestcode_wes_duplo, - }): + }) + if duplo_samples: # Set duplo status for WES sample udf_data['Dx Mengfractie'] = True + for duplo_sample in duplo_samples: + # Remove import warning from WES_duplo samples + if duplo_sample.udf['Dx Import warning'] and 'Alleen WES_duplo' in duplo_sample.udf['Dx Import warning']: + import_warning = duplo_sample.udf['Dx Import warning'].split(';') + import_warning.remove('Alleen WES_duplo') + duplo_sample.udf['Dx Import warning'] = '; '.join(import_warning) + duplo_sample.put() + + # Check Dx Monsternummer + if duplo_sample.udf['Dx Monsternummer'] == udf_data['Dx Monsternummer']: + udf_data['Dx Import warning'] = '; '.join([ + 'WES en WES_duplo zelfde monster ({sample}).'.format(sample=duplo_sample.name), + udf_data['Dx Import warning'] + ]) else: udf_data['Dx Mengfractie'] = False @@ -218,12 +247,14 @@ def from_helix(lims, email_settings, input_file): and sample.udf['Dx Foetus'] == udf_data['Dx Foetus'] ): udf_data['Dx Import warning'] = '; '.join([ - '{sample}: Monsternummer hetzelfde, Protocolomschrijving hetzelfde.'.format(sample=sample.name), + 'Let op herhaling of dubbele indicatie ({sample}).'.format(sample=sample.name), udf_data['Dx Import warning'] ]) - else: + elif not sample.udf['Dx Mengfractie']: udf_data['Dx Import warning'] = '; '.join([ - '{sample}: Monsternummer hetzelfde, Protocolomschrijving uniek.'.format(sample=sample.name), + 'Eerder onderzoek met protocolomschrijving {protocol} ({sample}).'.format( + protocol=sample.udf['Dx Protocolomschrijving'], sample=sample.name + ), udf_data['Dx Import warning'] ]) elif ( @@ -231,7 +262,7 @@ def from_helix(lims, email_settings, input_file): and sample.udf['Dx Foetus'] == udf_data['Dx Foetus'] ): udf_data['Dx Import warning'] = '; '.join([ - '{sample}: Monsternummer uniek, Protocolomschrijving hetzelfde.'.format(sample=sample.name), + 'Let op herhaling of dubbele indicatie ({sample}).'.format(sample=sample.name), udf_data['Dx Import warning'] ]) @@ -242,18 +273,19 @@ def from_helix(lims, email_settings, input_file): sample = Sample.create(lims, container=container, position='1:1', project=project, name=sample_name, udf=udf_data) lims.route_artifacts([sample.artifact], workflow_uri=workflow.uri) if udf_data['Dx Import warning']: - message += "{0}\tCreated and added to workflow: {1}.\tImport warning: {2}\n".format( + sample_messages[sample.name] = "{0}\tCreated and added to workflow: {1}.\tImport warning: {2}".format( sample.name, workflow.name, udf_data['Dx Import warning'] ) else: - message += "{0}\tCreated and added to workflow: {1}.\n".format(sample.name, workflow.name) + sample_messages[sample.name] = "{0}\tCreated and added to workflow: {1}.".format(sample.name, workflow.name) else: - message += "{0}\tERROR: Stoftest code {1} is not linked to a workflow.\n".format( + sample_messages[sample_name] += "{0}\tERROR: Stoftest code {1} is not linked to a workflow.".format( sample_name, udf_data['Dx Stoftest code'] ) # Send final email + message += '\n'.join(sample_messages.values()) send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message) From 03d41a60c3d42a31ea2e93c18e188053d801d0f9 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Mon, 10 Jul 2023 15:40:35 +0200 Subject: [PATCH 46/84] Add label file nunc mix samples --- clarity_epp.py | 7 ++++++- clarity_epp/export/labels.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/clarity_epp.py b/clarity_epp.py index 0985e03..65e3c21 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -59,6 +59,8 @@ def export_labels(args): clarity_epp.export.labels.container_sample(lims, args.process_id, args.output_file, args.description) elif args.type == 'storage_location': clarity_epp.export.labels.storage_location(lims, args.process_id, args.output_file) + elif args.type == 'nunc_mix_sample': + clarity_epp.export.labels.nunc_mix_sample(lims, args.process_id, args.output_file) def export_magnis(args): @@ -281,7 +283,10 @@ def placement_tecan(args): parser_export_illumina.set_defaults(func=export_illumina) parser_export_labels = subparser_export.add_parser('labels', help='Export container labels', parents=[output_parser]) - parser_export_labels.add_argument('type', choices=['container', 'container_sample', 'storage_location'], help='Label type') + parser_export_labels.add_argument( + 'type', + choices=['container', 'container_sample', 'storage_location', 'nunc_mix_sample'], + help='Label type') parser_export_labels.add_argument('process_id', help='Clarity lims process id') parser_export_labels.add_argument('-d', '--description', nargs='?', help='Container name description') parser_export_labels.set_defaults(func=export_labels) diff --git a/clarity_epp/export/labels.py b/clarity_epp/export/labels.py index f63ae8f..5b0c66c 100644 --- a/clarity_epp/export/labels.py +++ b/clarity_epp/export/labels.py @@ -40,3 +40,21 @@ def storage_location(lims, process_id, output_file): tray=storage_location[0][2:6], # Select 4 digits from: CB[1-9][1-9][1-9][1-9]KK pos=storage_location[1] )) + + +def nunc_mix_sample(lims, process_id, output_file): + """Generate (mix) sample nunc label file.""" + process = Process(lims, id=process_id) + + #Write empty header + output_file.write('\n') + + for artifact in process.analytes()[0]: + sample_mix = False + if len(artifact.samples) > 1: + sample_mix = True + + if sample_mix: + output_file.write(artifact.name) + else: + output_file.write(artifact.samples[0].udf['Dx Monsternummer']) \ No newline at end of file From d7e426da9fb66d5b12f8c6787f113ec0e3162acf Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Mon, 10 Jul 2023 15:54:59 +0200 Subject: [PATCH 47/84] Adjust line length --- clarity_epp/export/labels.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/clarity_epp/export/labels.py b/clarity_epp/export/labels.py index 5b0c66c..889bf27 100644 --- a/clarity_epp/export/labels.py +++ b/clarity_epp/export/labels.py @@ -9,7 +9,10 @@ def container(lims, process_id, output_file, description=''): for index, container in enumerate(sorted(process.output_containers(), key=lambda container: container.id, reverse=True)): if description: if ',' in description: - output_file.write('{description}\t{container}\r\n'.format(description=description.split(',')[index], container=container.name)) + output_file.write('{description}\t{container}\r\n'.format( + description=description.split(',')[index], + container=container.name + )) else: output_file.write('{description}\t{container}\r\n'.format(description=description, container=container.name)) else: @@ -22,7 +25,11 @@ def container_sample(lims, process_id, output_file, description=''): for container in process.output_containers(): for artifact in container.placements.values(): if description: - output_file.write('{description}\t{sample}\t{container}\r\n'.format(description=description, container=container.name, sample=artifact.name)) + output_file.write('{description}\t{sample}\t{container}\r\n'.format( + description=description, + container=container.name, + sample=artifact.name + )) else: output_file.write('{sample}\t{container}\r\n'.format(container=container.name, sample=artifact.name)) From e83cfe25849d54441ddfe82743e4866546234a43 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 11 Jul 2023 15:56:25 +0200 Subject: [PATCH 48/84] Fix indent --- clarity_epp/export/manual_pipetting.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 21d4d1a..6a4b5ee 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -792,15 +792,14 @@ def samplesheet_normalization_mix(lims, process_id, output_file): monster = output_sample.udf['Dx Monsternummer'] well = ''.join(output_artifact.location[1].split(':')) output_data = ( - '{sample}\t{concentration:.2f}\t{sample_volume:.2f}\t{low_te_volume:.2f}\t' - '{container}\t{well}\n' - ).format( - sample=monster, - concentration=samples[monster]['conc'], - sample_volume=samples[monster]['sample_volume'], - low_te_volume=samples[monster]['low_te_volume'], - container=output_artifact.location[0].name, - well=well + '{sample}\t{concentration:.2f}\t{sample_volume:.2f}\t{low_te_volume:.2f}\t{container}\t{well}\n'.format( + sample=monster, + concentration=samples[monster]['conc'], + sample_volume=samples[monster]['sample_volume'], + low_te_volume=samples[monster]['low_te_volume'], + container=output_artifact.location[0].name, + well=well + ) ) if well in output: output[well][monster] = output_data From 54a1dec2b5a8e1f931c1aec24e7fffe356aa5d84 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 11 Jul 2023 16:05:07 +0200 Subject: [PATCH 49/84] Adjust Tecan QC step name --- clarity_epp/export/manual_pipetting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 6a4b5ee..939a27a 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -739,7 +739,7 @@ def samplesheet_normalization_mix(lims, process_id, output_file): samples = {} # Find all QC process types - qc_process_types = clarity_epp.export.utils.get_process_types(lims, ['Dx Qubit QC', 'Dx Tecan']) + qc_process_types = clarity_epp.export.utils.get_process_types(lims, ['Dx Qubit QC', 'Dx Tecan Spark 10M QC']) # Find concentration in last QC process for input_artifact in process.all_inputs(): From 0de5e14531d50325bcb4ae44806de3d682154bea Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 11 Jul 2023 16:31:11 +0200 Subject: [PATCH 50/84] Add concentration from QC process --- clarity_epp/export/tecan.py | 82 +++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 5d7631e..8f2dff3 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -41,48 +41,88 @@ def samplesheet(lims, process_id, type, output_file): )) elif type == 'filling_out_purify': + # Samplesheet Tecan Fluent 480 'Dx Uitvullen en zuiveren' (mix) samples output_file.write( 'SourceTubeID;VolSample;VolWater;PositionIndex;MengID\n' ) + + # Find all QC process types + qc_process_types = clarity_epp.export.utils.get_process_types(lims, ['Dx Qubit QC', 'Dx Tecan Spark 10M QC']) + + samples = {} + # Find concentration in last QC process + for input_artifact in process.all_inputs(): + for input_sample in input_artifact.samples: + qc_processes = lims.get_processes(type=qc_process_types, inputartifactlimsid=input_artifact.id) + if qc_processes: + qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] + for qc_artifact in qc_process.outputs_per_input(input_artifact.id): + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + + else: + parent_process = input_artifact.parent_process + for parent_artifact in parent_process.all_inputs(): + qc_processes = lims.get_processes(type=qc_process_types, inputartifactlimsid=parent_artifact.id) + if qc_processes: + qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] + for qc_artifact in qc_process.outputs_per_input(parent_artifact.id): + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + else: + # No QC process found, use Helix concentration + concentration = input_sample.udf['Dx Concentratie (ng/ul)'] + + samples[input_sample.udf['Dx Monsternummer']] = {'conc': concentration} + for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): artifact = well_plate[well] sample_mix = False if len(artifact.samples) > 1: sample_mix = True - sample_volumes = {} - water_volumes = {} - mix_names = {} - messages = {} + + if sample_mix: + dividend = 880 + max_volume = 30 + else: + dividend = 1760 + max_volume = 60 + for sample in artifact.samples: - messages[sample] = "" - conc = sample.udf['Dx Concentratie (ng/ul)'] + monster = sample.udf['Dx Monsternummer'] + samples[monster]['message'] = '' if sample_mix: - dividend = 880 - max_volume = 30 - mix_names[sample] = artifact.name + samples[monster]['mix_names'] = input_artifact.name else: - dividend = 1760 - max_volume = 60 - mix_names[sample] = sample.udf['Dx Monsternummer'] - calc_sample = dividend / conc + samples[monster]['mix_names'] = monster + + # Calculation of pipetting volumes + calc_sample = dividend / samples[monster]['conc'] if calc_sample < 4: volume_sample = 4 elif calc_sample > max_volume: volume_sample = max_volume - messages[sample] = ("Conc. too low - volume= {calc_sample} ul".format(calc_sample=calc_sample)) + samples[monster]['message'] = ( + 'Conc. too low - volume= {calc_sample} ul'.format(calc_sample=calc_sample) + ) else: volume_sample = calc_sample - sample_volumes[sample] = volume_sample - water_volumes[sample] = max_volume - volume_sample + samples[monster]['sample_volume'] = volume_sample + volume_water = max_volume - volume_sample + samples[monster]['water_volume'] = volume_water + for sample in artifact.samples: + monster = sample.udf['Dx Monsternummer'] output_file.write('{sample};{volume_sample:.2f};{volume_water:.2f};{index};{name};{empty};{message}\n'.format( sample=sample.udf['Dx Fractienummer'], - volume_sample=sample_volumes[sample], - volume_water=water_volumes[sample], + volume_sample=samples[monster]['sample_volume'], + volume_water=samples[monster]['water_volume'], index=clarity_epp.export.utils.get_well_index(well, one_based=True), - name=mix_names[sample], - empty="", - message=messages[sample] + name=samples[monster]['mix_names'], + empty='', + message=samples[monster]['message'] )) elif type == 'normalise': From 48e63a228112a54e6babcb1f0d7479884c361e38 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 11 Jul 2023 18:09:26 +0200 Subject: [PATCH 51/84] Add check pipetting input/output nuncs --- clarity_epp.py | 9 +++++++++ clarity_epp/placement/__init__.py | 1 + clarity_epp/placement/pipetting.py | 27 +++++++++++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 clarity_epp/placement/pipetting.py diff --git a/clarity_epp.py b/clarity_epp.py index 65e3c21..dbf99f7 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -238,6 +238,11 @@ def placement_tecan(args): clarity_epp.placement.tecan.place_artifacts(lims, args.process_id) +def placement_pipetting(args): + """Check pipetted input and output nuncs.""" + clarity_epp.placement.pipetting.check_nunc_input_nunc_output(lims, args.process_id) + + if __name__ == "__main__": parser = argparse.ArgumentParser() subparser = parser.add_subparsers() @@ -450,5 +455,9 @@ def placement_tecan(args): parser_placement_tecan.add_argument('process_id', help='Clarity lims process id') parser_placement_tecan.set_defaults(func=placement_tecan) + parser_placement_pipetting = subparser_placement.add_parser('pipetting', help='Check pipetting input and output') + parser_placement_pipetting.add_argument('process_id', help='Clarity lims process id') + parser_placement_pipetting.set_defaults(func=placement_pipetting) + args = parser.parse_args() args.func(args) diff --git a/clarity_epp/placement/__init__.py b/clarity_epp/placement/__init__.py index c9d5394..2baf0f9 100644 --- a/clarity_epp/placement/__init__.py +++ b/clarity_epp/placement/__init__.py @@ -2,6 +2,7 @@ import clarity_epp.placement.artifact import clarity_epp.placement.barcode +import clarity_epp.placement.pipetting import clarity_epp.placement.plate import clarity_epp.placement.pool import clarity_epp.placement.step diff --git a/clarity_epp/placement/pipetting.py b/clarity_epp/placement/pipetting.py new file mode 100644 index 0000000..f25611d --- /dev/null +++ b/clarity_epp/placement/pipetting.py @@ -0,0 +1,27 @@ +"""Pipetting placement functions.""" + +from genologics.entities import Process + +def check_nunc_input_nunc_output(lims, process_id): + """Check nuncs.""" + process = Process(lims, id=process_id) + for output_artifact in process.all_outputs(): + if output_artifact.type == 'Analyte': + input_nunc_1 = '' + input_nunc_2 = '' + output_nunc = '' + input_combined = '' + if 'Dx Sample 1 norm' in output_artifact.udf: + input_nunc_1 = output_artifact.udf['Dx Sample 1 norm'] + if 'Dx Sample 2 norm' in output_artifact.udf: + input_nunc_2 = output_artifact.udf['Dx Sample 2 norm'] + if 'Dx Sample (output)' in output_artifact.udf: + output_nunc = output_artifact.udf['Dx Sample (output)'] + if input_nunc_1 and input_nunc_2: + input_combined = '{input_nunc_1}-{input_nunc_2}'.format(input_nunc_1=input_nunc_1, input_nunc_2=input_nunc_2) + elif input_nunc_1: + input_combined = input_nunc_1 + if input_combined and output_nunc: + if input_combined == output_nunc: + output_artifact.udf['Dx pipetteer check'] = True + output_artifact.put() From 32da5974f03f5400221f5e67f8ef692fb64ebf0f Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 12 Jul 2023 14:22:43 +0200 Subject: [PATCH 52/84] Fix udf check --- clarity_epp/upload/samples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 9a47826..ea8da16 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -223,7 +223,7 @@ def from_helix(lims, email_settings, input_file): udf_data['Dx Mengfractie'] = True for duplo_sample in duplo_samples: # Remove import warning from WES_duplo samples - if duplo_sample.udf['Dx Import warning'] and 'Alleen WES_duplo' in duplo_sample.udf['Dx Import warning']: + if 'Dx Import warning' in duplo_sample.udf and 'Alleen WES_duplo' in duplo_sample.udf['Dx Import warning']: import_warning = duplo_sample.udf['Dx Import warning'].split(';') import_warning.remove('Alleen WES_duplo') duplo_sample.udf['Dx Import warning'] = '; '.join(import_warning) From b65057ea61f305d363d61bbcfe0c803c1fa53045 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 13 Jul 2023 13:25:47 +0200 Subject: [PATCH 53/84] Update waarschuwing --- clarity_epp/upload/samples.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index ea8da16..b95c4eb 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -131,7 +131,7 @@ def from_helix(lims, email_settings, input_file): # Set 'Dx norm. manueel' udf for samples with Dx Concentratie (ng/ul) if udf_data['Dx Concentratie (ng/ul)']: - if udf_data['Dx Concentratie (ng/ul)'] < 29.3: + if udf_data['Dx Concentratie (ng/ul)'] <= 29.3: udf_data['Dx norm. manueel'] = True else: udf_data['Dx norm. manueel'] = False @@ -210,7 +210,7 @@ def from_helix(lims, email_settings, input_file): udf_data['Dx Import warning'] ]) else: # Set import warning if no WES samples found - udf_data['Dx Import warning'] = '; '.join(['Alleen WES_duplo', udf_data['Dx Import warning']]) + udf_data['Dx Import warning'] = '; '.join(['Alleen WES_duplo aangemeld.', udf_data['Dx Import warning']]) elif udf_data['Dx Stoftest code'] == config.stoftestcode_wes: # Find WES_duplo sample(s) @@ -223,9 +223,9 @@ def from_helix(lims, email_settings, input_file): udf_data['Dx Mengfractie'] = True for duplo_sample in duplo_samples: # Remove import warning from WES_duplo samples - if 'Dx Import warning' in duplo_sample.udf and 'Alleen WES_duplo' in duplo_sample.udf['Dx Import warning']: + if 'Dx Import warning' in duplo_sample.udf and 'Alleen WES_duplo aangemeld.' in duplo_sample.udf['Dx Import warning']: import_warning = duplo_sample.udf['Dx Import warning'].split(';') - import_warning.remove('Alleen WES_duplo') + import_warning.remove('Alleen WES_duplo aangemeld.') duplo_sample.udf['Dx Import warning'] = '; '.join(import_warning) duplo_sample.put() @@ -247,10 +247,10 @@ def from_helix(lims, email_settings, input_file): and sample.udf['Dx Foetus'] == udf_data['Dx Foetus'] ): udf_data['Dx Import warning'] = '; '.join([ - 'Let op herhaling of dubbele indicatie ({sample}).'.format(sample=sample.name), + 'Herhaling of dubbele indicatie, beide monsters ingeladen ({sample}).'.format(sample=sample.name), udf_data['Dx Import warning'] ]) - elif not sample.udf['Dx Mengfractie']: + elif 'Dx Mengfractie' not in sample.udf or not sample.udf['Dx Mengfractie']: udf_data['Dx Import warning'] = '; '.join([ 'Eerder onderzoek met protocolomschrijving {protocol} ({sample}).'.format( protocol=sample.udf['Dx Protocolomschrijving'], sample=sample.name @@ -262,7 +262,7 @@ def from_helix(lims, email_settings, input_file): and sample.udf['Dx Foetus'] == udf_data['Dx Foetus'] ): udf_data['Dx Import warning'] = '; '.join([ - 'Let op herhaling of dubbele indicatie ({sample}).'.format(sample=sample.name), + 'Herhaling of dubbele indicatie, beide monsters ingeladen ({sample}).'.format(sample=sample.name), udf_data['Dx Import warning'] ]) From 46705ef1ddb285c8e4b1afe65ee7b139c8954cf0 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Thu, 13 Jul 2023 16:54:21 +0200 Subject: [PATCH 54/84] Adjust calculation for use udfs --- clarity_epp/export/manual_pipetting.py | 38 +++++++++++++++++++------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 939a27a..b335e3c 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -767,22 +767,40 @@ def samplesheet_normalization_mix(lims, process_id, output_file): # Calculation of pipetting volumes for input_artifact in process.all_inputs(): - dividend = 400 - minuend = 50 + output_artifact = process.outputs_per_input(input_artifact.id, Analyte=True)[0] # assume one artifact per input + dividend = float(output_artifact.udf['Dx Input (ng)']) / len(input_artifact.samples) + minuend = float(output_artifact.udf['Dx Eindvolume (ul)']) / len(input_artifact.samples) sample_mix = False if len(input_artifact.samples) > 1: sample_mix = True - if sample_mix: - dividend = 200 - minuend = 25 - - for input_sample in input_artifact.samples: - sample_volume = dividend / samples[input_sample.udf['Dx Monsternummer']]['conc'] + input_sample_1 = input_artifact.samples[0] + if 'Dx sample vol. #1' in output_artifact.udf: + dividend = ( + samples[input_sample_1.udf['Dx Monsternummer']]['conc'] * float(output_artifact.udf['Dx sample vol. #1']) + ) + sample_volume = dividend / samples[input_sample_1.udf['Dx Monsternummer']]['conc'] + if sample_volume > minuend: + low_te_volume = 0 + else: low_te_volume = minuend - sample_volume - samples[input_sample.udf['Dx Monsternummer']]['sample_volume'] = sample_volume - samples[input_sample.udf['Dx Monsternummer']]['low_te_volume'] = low_te_volume + samples[input_sample_1.udf['Dx Monsternummer']]['sample_volume'] = sample_volume + samples[input_sample_1.udf['Dx Monsternummer']]['low_te_volume'] = low_te_volume + + if sample_mix: + input_sample_2 = input_artifact.samples[1] + if 'Dx sample vol. #2' in output_artifact.udf: + dividend = ( + samples[input_sample_2.udf['Dx Monsternummer']]['conc'] * float(output_artifact.udf['Dx sample vol. #2']) + ) + sample_volume = dividend / samples[input_sample_2.udf['Dx Monsternummer']]['conc'] + if sample_volume > minuend: + low_te_volume = 0 + else: + low_te_volume = minuend - sample_volume + samples[input_sample_2.udf['Dx Monsternummer']]['sample_volume'] = sample_volume + samples[input_sample_2.udf['Dx Monsternummer']]['low_te_volume'] = low_te_volume # Compose output per sample in well output = {} From fa4ebf620f6780d5b6e06b4ad59c5abc2d383541 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 18 Jul 2023 13:13:53 +0200 Subject: [PATCH 55/84] Update bioinf workflow --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index c08b058..1c457f5 100755 --- a/config.py +++ b/config.py @@ -82,7 +82,7 @@ # Post sequencing workflow sequencing_workflow = '1701' # DEV Dx Illumina Sequencing v1.2 -post_sequencing_workflow = '902' # DEV Dx Bioinformatica analyses v1.0 +post_sequencing_workflow = '1204' # DEV Dx Bioinformatica analyses v1.1 post_bioinf_workflow = '1803' # DEV Dx NGS WES onderzoeken afronden v2.0 # Research Onderzoeksindicatie From 560c6f8ebc7e3e114f5c3116826573aad45c35e8 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 18 Jul 2023 16:12:59 +0200 Subject: [PATCH 56/84] Add automation Tecan results mix sample --- clarity_epp.py | 4 +++- clarity_epp/upload/tecan.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/clarity_epp.py b/clarity_epp.py index dbf99f7..7b01105 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -155,6 +155,8 @@ def upload_tecan_results(args): clarity_epp.upload.tecan.results_qc(lims, args.process_id) elif args.type == 'purify_normalise': clarity_epp.upload.tecan.results_purify_normalise(lims, args.process_id) + elif args.type == 'purify_mix': + clarity_epp.upload.tecan.results_purify_mix(lims, args.process_id) def upload_tapestation_results(args): @@ -385,7 +387,7 @@ def placement_pipetting(args): parser_upload_tecan = subparser_upload.add_parser('tecan', help='Upload tecan results') parser_upload_tecan.add_argument('process_id', help='Clarity lims process id') - parser_upload_tecan.add_argument('type', choices=['qc', 'purify_normalise'], help='Tecan process type') + parser_upload_tecan.add_argument('type', choices=['qc', 'purify_normalise', 'purify_mix'], help='Tecan process type') parser_upload_tecan.set_defaults(func=upload_tecan_results) parser_upload_magnis = subparser_upload.add_parser('magnis', help='Upload magnis results') diff --git a/clarity_epp/upload/tecan.py b/clarity_epp/upload/tecan.py index 20a4598..8b7aece 100644 --- a/clarity_epp/upload/tecan.py +++ b/clarity_epp/upload/tecan.py @@ -137,3 +137,29 @@ def results_purify_normalise(lims, process_id): artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = tecan_result[sample.udf['Dx Fractienummer']]['conc'] artifact.udf['Dx QC status'] = tecan_result[sample.udf['Dx Fractienummer']]['norm'] artifact.put() + + +def results_purify_mix(lims, process_id): + """Upload tecan results to artifacts (mix samples).""" + process = Process(lims, id=process_id) + + # Find and parse Tecan Fluent 480 Output + tecan_result = {} + for result_file in process.result_files(): + if result_file.name == 'Tecan Fluent 480 Output': + file_data = lims.get_file_contents(result_file.files[0].id).split('\n') + header = file_data[0].rstrip().split(';') + for line in file_data[1:]: + if line.rstrip(): + data = line.rstrip().split(';') + tecan_result[data[header.index('SampleID')]] = { + 'conc': float(data[header.index('Concentratie(ng/ul)')]), + 'norm': txt_to_bool(data[header.index('Normalisatie')]) + } + break # File found exit loop + + # Set concentration values on artifacts + for artifact in process.analytes()[0]: + artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = tecan_result[artifact.name]['conc'] + artifact.udf['Dx QC status'] = tecan_result[artifact.name]['norm'] + artifact.put() From b737a798fd55762c1df9f5e4896b3018bc29fcb8 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 18 Jul 2023 16:20:06 +0200 Subject: [PATCH 57/84] Fix new line label file --- clarity_epp/export/labels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clarity_epp/export/labels.py b/clarity_epp/export/labels.py index 889bf27..2880b40 100644 --- a/clarity_epp/export/labels.py +++ b/clarity_epp/export/labels.py @@ -62,6 +62,6 @@ def nunc_mix_sample(lims, process_id, output_file): sample_mix = True if sample_mix: - output_file.write(artifact.name) + output_file.write('{sample}\n'.format(sample=artifact.name)) else: - output_file.write(artifact.samples[0].udf['Dx Monsternummer']) \ No newline at end of file + output_file.write('{sample}\n'.format(sample=artifact.samples[0].udf['Dx Monsternummer'])) \ No newline at end of file From 58cb838c4c44f28a3a186dc61c14b8bc9afbbac9 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 18 Jul 2023 16:44:55 +0200 Subject: [PATCH 58/84] Add check nunc-artifact --- clarity_epp/placement/pipetting.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/clarity_epp/placement/pipetting.py b/clarity_epp/placement/pipetting.py index f25611d..42ff3ff 100644 --- a/clarity_epp/placement/pipetting.py +++ b/clarity_epp/placement/pipetting.py @@ -11,6 +11,14 @@ def check_nunc_input_nunc_output(lims, process_id): input_nunc_2 = '' output_nunc = '' input_combined = '' + if len(output_artifact.samples) > 1: + fraction = ('{fraction1}-{fraction2}'.format( + fraction1=(output_artifact.samples[0].udf['Dx Fractienummer']), + fraction2=(output_artifact.samples[1].udf['Dx Fractienummer']) + ) + ) + else: + fraction = output_artifact.samples[0].udf['Dx Fractienummer'] if 'Dx Sample 1 norm' in output_artifact.udf: input_nunc_1 = output_artifact.udf['Dx Sample 1 norm'] if 'Dx Sample 2 norm' in output_artifact.udf: @@ -22,6 +30,6 @@ def check_nunc_input_nunc_output(lims, process_id): elif input_nunc_1: input_combined = input_nunc_1 if input_combined and output_nunc: - if input_combined == output_nunc: + if input_combined == output_nunc and output_nunc == fraction: output_artifact.udf['Dx pipetteer check'] = True output_artifact.put() From 1eba215b06b1854179845eff1e4c47ebab04af10 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 18 Jul 2023 16:54:48 +0200 Subject: [PATCH 59/84] Set False if not True --- clarity_epp/placement/pipetting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clarity_epp/placement/pipetting.py b/clarity_epp/placement/pipetting.py index 42ff3ff..a5f8d28 100644 --- a/clarity_epp/placement/pipetting.py +++ b/clarity_epp/placement/pipetting.py @@ -32,4 +32,6 @@ def check_nunc_input_nunc_output(lims, process_id): if input_combined and output_nunc: if input_combined == output_nunc and output_nunc == fraction: output_artifact.udf['Dx pipetteer check'] = True - output_artifact.put() + else: + output_artifact.udf['Dx pipetteer check'] = False + output_artifact.put() From 04e4ef786a14b0aad69da82e80d97ce0bfb87467 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 18 Jul 2023 17:08:23 +0200 Subject: [PATCH 60/84] Update workflow config --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 701125e..69e10f8 100755 --- a/config.py +++ b/config.py @@ -24,8 +24,8 @@ stoftestcode_research = 'NGS_023' stoftestcode_workflow = { - stoftestcode_wes: '1802', # DEV Dx Exoom Magnis v2.0 - stoftestcode_wes_duplo: '1802', # DEV Dx Exoom Magnis v2.0 + stoftestcode_wes: '1852', # DEV Dx Exoom Magnis v2.1 + stoftestcode_wes_duplo: '1852', # DEV Dx Exoom Magnis v2.1 stoftestcode_mip: '1651', # DEV Dx smMIP v1.2 } From db83dac8fce0985ee0b1be196188e783f2ea2002 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Fri, 21 Jul 2023 10:35:56 +0200 Subject: [PATCH 61/84] FIx Tecan Spark QC support --- clarity_epp/export/manual_pipetting.py | 14 ++++++++------ clarity_epp/export/tecan.py | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index b335e3c..26354dd 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -748,9 +748,10 @@ def samplesheet_normalization_mix(lims, process_id, output_file): if qc_processes: qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] for qc_artifact in qc_process.outputs_per_input(input_artifact.id): - for qc_sample in qc_artifact.samples: - if qc_sample.name == input_sample.name: - concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + if input_sample.name in qc_artifact.name: + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) else: parent_process = input_artifact.parent_process @@ -759,9 +760,10 @@ def samplesheet_normalization_mix(lims, process_id, output_file): if qc_processes: qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] for qc_artifact in qc_process.outputs_per_input(parent_artifact.id): - for qc_sample in qc_artifact.samples: - if qc_sample.name == input_sample.name: - concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + if input_sample.name in qc_artifact.name: + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) samples[input_sample.udf['Dx Monsternummer']] = {'conc': concentration} diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 8f2dff3..8cdca61 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -57,9 +57,10 @@ def samplesheet(lims, process_id, type, output_file): if qc_processes: qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] for qc_artifact in qc_process.outputs_per_input(input_artifact.id): - for qc_sample in qc_artifact.samples: - if qc_sample.name == input_sample.name: - concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + if input_sample.name in qc_artifact.name: + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) else: parent_process = input_artifact.parent_process @@ -68,9 +69,10 @@ def samplesheet(lims, process_id, type, output_file): if qc_processes: qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] for qc_artifact in qc_process.outputs_per_input(parent_artifact.id): - for qc_sample in qc_artifact.samples: - if qc_sample.name == input_sample.name: - concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + if input_sample.name in qc_artifact.name: + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) else: # No QC process found, use Helix concentration concentration = input_sample.udf['Dx Concentratie (ng/ul)'] From 78d137e66745cce3fb251c322875079dda06af37 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 25 Jul 2023 18:23:32 +0200 Subject: [PATCH 62/84] fixes --- clarity_epp/export/manual_pipetting.py | 32 ++++++++++++++++---------- clarity_epp/export/tecan.py | 25 ++++++++++---------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 26354dd..4efc492 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -756,14 +756,18 @@ def samplesheet_normalization_mix(lims, process_id, output_file): else: parent_process = input_artifact.parent_process for parent_artifact in parent_process.all_inputs(): - qc_processes = lims.get_processes(type=qc_process_types, inputartifactlimsid=parent_artifact.id) - if qc_processes: - qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] - for qc_artifact in qc_process.outputs_per_input(parent_artifact.id): - if input_sample.name in qc_artifact.name: - for qc_sample in qc_artifact.samples: - if qc_sample.name == input_sample.name: - concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + if parent_artifact.name == input_sample.name: + qc_processes = lims.get_processes(type=qc_process_types, inputartifactlimsid=parent_artifact.id) + if qc_processes: + qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] + for qc_artifact in qc_process.outputs_per_input(parent_artifact.id): + if input_sample.name in qc_artifact.name: + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + else: + # No QC process found, use Helix concentration + concentration = input_sample.udf['Dx Concentratie (ng/ul)'] samples[input_sample.udf['Dx Monsternummer']] = {'conc': concentration} @@ -779,10 +783,12 @@ def samplesheet_normalization_mix(lims, process_id, output_file): input_sample_1 = input_artifact.samples[0] if 'Dx sample vol. #1' in output_artifact.udf: - dividend = ( + dividend_sample_1 = ( samples[input_sample_1.udf['Dx Monsternummer']]['conc'] * float(output_artifact.udf['Dx sample vol. #1']) ) - sample_volume = dividend / samples[input_sample_1.udf['Dx Monsternummer']]['conc'] + else: + dividend_sample_1 = dividend + sample_volume = dividend_sample_1 / samples[input_sample_1.udf['Dx Monsternummer']]['conc'] if sample_volume > minuend: low_te_volume = 0 else: @@ -793,10 +799,12 @@ def samplesheet_normalization_mix(lims, process_id, output_file): if sample_mix: input_sample_2 = input_artifact.samples[1] if 'Dx sample vol. #2' in output_artifact.udf: - dividend = ( + dividend_sample_2 = ( samples[input_sample_2.udf['Dx Monsternummer']]['conc'] * float(output_artifact.udf['Dx sample vol. #2']) ) - sample_volume = dividend / samples[input_sample_2.udf['Dx Monsternummer']]['conc'] + else: + dividend_sample_2 = dividend + sample_volume = dividend_sample_2 / samples[input_sample_2.udf['Dx Monsternummer']]['conc'] if sample_volume > minuend: low_te_volume = 0 else: diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 8cdca61..60f8baf 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -65,17 +65,18 @@ def samplesheet(lims, process_id, type, output_file): else: parent_process = input_artifact.parent_process for parent_artifact in parent_process.all_inputs(): - qc_processes = lims.get_processes(type=qc_process_types, inputartifactlimsid=parent_artifact.id) - if qc_processes: - qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] - for qc_artifact in qc_process.outputs_per_input(parent_artifact.id): - if input_sample.name in qc_artifact.name: - for qc_sample in qc_artifact.samples: - if qc_sample.name == input_sample.name: - concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) - else: - # No QC process found, use Helix concentration - concentration = input_sample.udf['Dx Concentratie (ng/ul)'] + if parent_artifact.name == input_sample.name: + qc_processes = lims.get_processes(type=qc_process_types, inputartifactlimsid=parent_artifact.id) + if qc_processes: + qc_process = sorted(qc_processes, key=lambda process: int(process.id.split('-')[-1]))[-1] + for qc_artifact in qc_process.outputs_per_input(parent_artifact.id): + if input_sample.name in qc_artifact.name: + for qc_sample in qc_artifact.samples: + if qc_sample.name == input_sample.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + else: + # No QC process found, use Helix concentration + concentration = input_sample.udf['Dx Concentratie (ng/ul)'] samples[input_sample.udf['Dx Monsternummer']] = {'conc': concentration} @@ -96,7 +97,7 @@ def samplesheet(lims, process_id, type, output_file): monster = sample.udf['Dx Monsternummer'] samples[monster]['message'] = '' if sample_mix: - samples[monster]['mix_names'] = input_artifact.name + samples[monster]['mix_names'] = artifact.name else: samples[monster]['mix_names'] = monster From f615a9bcfffb44cc00235f54289d3f5364ac136e Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Wed, 16 Aug 2023 16:52:52 +0200 Subject: [PATCH 63/84] Reset 'Dx norm. manueel' udf in QC steps --- clarity_epp/qc/qubit.py | 8 ++++++++ clarity_epp/upload/tecan.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/clarity_epp/qc/qubit.py b/clarity_epp/qc/qubit.py index 4c5fb2e..c951cdf 100644 --- a/clarity_epp/qc/qubit.py +++ b/clarity_epp/qc/qubit.py @@ -27,6 +27,14 @@ def set_qc_flag(lims, process_id, cutoff=10): sample_measurements_average = sum(sample_measurements) / float(len(sample_measurements)) artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = sample_measurements_average + # Reset 'Dx norm. manueel' udf + if 'Dx Sample registratie zuivering' in process.all_inputs()[0].parent_process.type.name: + if sample_measurements_average <= 29.3: + artifact.samples[0].udf['Dx norm. manueel'] = True + else: + artifact.samples[0].udf['Dx norm. manueel'] = False + artifact.samples[0].put() + if concentration_range[0] <= sample_measurements_average <= concentration_range[1]: if len(sample_measurements) == 1: artifact.qc_flag = 'PASSED' diff --git a/clarity_epp/upload/tecan.py b/clarity_epp/upload/tecan.py index 8b7aece..4ee6897 100644 --- a/clarity_epp/upload/tecan.py +++ b/clarity_epp/upload/tecan.py @@ -77,6 +77,14 @@ def results_qc(lims, process_id): sample_concentration = ((sample_fluorescence - baseline_fluorescence) * regression_slope) / 2.0 artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = sample_concentration + # Reset 'Dx norm. manueel' udf + if 'Dx Sample registratie zuivering' in process.all_inputs()[0].parent_process.type.name: + if sample_concentration <= 29.3: + artifact.samples[0].udf['Dx norm. manueel'] = True + else: + artifact.samples[0].udf['Dx norm. manueel'] = False + artifact.samples[0].put() + # Set artifact Concentratie fluorescentie # Get artifact index == count if artifact_name not in artifact_count: From b7fe1ec5f1f04cf326538317cff7a7b8005ffcf7 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Wed, 16 Aug 2023 17:12:48 +0200 Subject: [PATCH 64/84] Fix single sample key error --- clarity_epp/upload/tecan.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/clarity_epp/upload/tecan.py b/clarity_epp/upload/tecan.py index 4ee6897..82784bf 100644 --- a/clarity_epp/upload/tecan.py +++ b/clarity_epp/upload/tecan.py @@ -168,6 +168,11 @@ def results_purify_mix(lims, process_id): # Set concentration values on artifacts for artifact in process.analytes()[0]: - artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = tecan_result[artifact.name]['conc'] - artifact.udf['Dx QC status'] = tecan_result[artifact.name]['norm'] + if len(artifact.samples) > 1: + artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = tecan_result[artifact.name]['conc'] + artifact.udf['Dx QC status'] = tecan_result[artifact.name]['norm'] + else: + sample = artifact.samples[0] + artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = tecan_result[sample.udf['Dx Monsternummer']]['conc'] + artifact.udf['Dx QC status'] = tecan_result[sample.udf['Dx Monsternummer']]['norm'] artifact.put() From 25258b1247307d32eb023b40cf83a01647c326d2 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Wed, 16 Aug 2023 18:11:22 +0200 Subject: [PATCH 65/84] Refactor for shortened mix sample names --- clarity_epp/__init__.py | 20 ++++++++++++++++++-- clarity_epp/export/labels.py | 16 ++++++++++++++-- clarity_epp/export/tecan.py | 16 ++++++++++------ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/clarity_epp/__init__.py b/clarity_epp/__init__.py index 2c10e95..4a427e5 100644 --- a/clarity_epp/__init__.py +++ b/clarity_epp/__init__.py @@ -4,6 +4,7 @@ from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase from email.mime.text import MIMEText +import re import smtplib import mimetypes @@ -47,8 +48,23 @@ def get_sample_artifacts_from_pool(lims, pool_artifact): return sample_artifacts - - +def get_mix_sample_barcode(artifact): + """Generate mix sample shortened barcode name.""" + sample_names = {} + for sample in artifact.samples: + if 'Dx Monsternummer' in sample.udf: + monster = sample.udf['Dx Monsternummer'] + if re.match(r'\d{4}D\d+', monster): + sample_names[sample] = monster[2:4], monster[5:] + elif monster.startswith('D'): + sample_names[sample] = monster + + barcode_name = '' + if sample_names: + for sample in artifact.samples: + barcode_name += ''.join(sample_names[sample]) + + return barcode_name def send_email(server, sender, receivers, subject, text, attachment=None): diff --git a/clarity_epp/export/labels.py b/clarity_epp/export/labels.py index 2880b40..24cdf4f 100644 --- a/clarity_epp/export/labels.py +++ b/clarity_epp/export/labels.py @@ -2,6 +2,8 @@ from genologics.entities import Process +from .. import get_mix_sample_barcode + def container(lims, process_id, output_file, description=''): """Generate container label file.""" @@ -57,11 +59,21 @@ def nunc_mix_sample(lims, process_id, output_file): output_file.write('\n') for artifact in process.analytes()[0]: + well = ''.join(artifact.location[1].split(':')) sample_mix = False if len(artifact.samples) > 1: sample_mix = True if sample_mix: - output_file.write('{sample}\n'.format(sample=artifact.name)) + barcode_name = get_mix_sample_barcode(artifact) + output_file.write('{sample};;;;;{container}:{well};;1\n'.format( + sample=barcode_name, + container=artifact.container.name, + well=well + )) else: - output_file.write('{sample}\n'.format(sample=artifact.samples[0].udf['Dx Monsternummer'])) \ No newline at end of file + output_file.write('{sample};;;;;{container}:{well};;1\n'.format( + sample=artifact.samples[0].udf['Dx Fractienummer'], + container=artifact.container.name, + well=well + )) \ No newline at end of file diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 60f8baf..96cac97 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -2,6 +2,7 @@ from genologics.entities import Process +from .. import get_mix_sample_barcode import clarity_epp.export.utils @@ -132,9 +133,12 @@ def samplesheet(lims, process_id, type, output_file): output_file.write('SourceTubeID;PositionID;PositionIndex\n') for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): artifact = well_plate[well] - for sample in artifact.samples: - output_file.write('{sample};{well};{index}\n'.format( - sample=sample.udf['Dx Fractienummer'], - well=well, - index=clarity_epp.export.utils.get_well_index(well, one_based=True) - )) \ No newline at end of file + if len(artifact.samples) > 1: + source_tube = get_mix_sample_barcode(artifact) + else: + source_tube = sample.udf['Dx Fractienummer'] + output_file.write('{sample};{well};{index}\n'.format( + sample=source_tube, + well=well, + index=clarity_epp.export.utils.get_well_index(well, one_based=True) + )) \ No newline at end of file From c35f87802da1a8da0ddb7cc829e8b9e0e8ba4f7b Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 17 Aug 2023 17:08:21 +0200 Subject: [PATCH 66/84] Add CREv4 (elidS34226467) --- clarity_epp/export/illumina.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index fd183c4..b2704ca 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -83,6 +83,10 @@ def get_project(projects, urgent=False): project_type = 'SSv7' families[family]['project_type'] = project_type families[family]['split_project_type'] = True + elif 'elidS34226467' in newest_protocol and not families[family]['NICU']: + project_type = 'CREv4' + families[family]['project_type'] = project_type + families[family]['split_project_type'] = True # Set urgent status if 'Dx Spoed' in list(sample.udf) and sample.udf['Dx Spoed']: From 6b02572033b38862eeeaee6b215b92e6d843ffb4 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Fri, 18 Aug 2023 12:57:10 +0200 Subject: [PATCH 67/84] Refactor for changed columns pipetting scheme --- clarity_epp/export/manual_pipetting.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 69f06f7..abdfeb2 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -3,6 +3,7 @@ from genologics.entities import Process +from .. import get_mix_sample_barcode import clarity_epp.export.utils @@ -740,7 +741,7 @@ def samplesheet_normalization_mix(lims, process_id, output_file): process = Process(lims, id=process_id) output_file.write( - 'Monsternummer\tConcentratie (ng/ul)\tVolume sample (ul)\tVolume low TE (ul)\tContainer\tWell\n' + 'Fractienummer\tConcentratie (ng/ul)\tVolume sample (ul)\tVolume low TE (ul)\tContainer_tube\n' ) samples = {} @@ -825,15 +826,18 @@ def samplesheet_normalization_mix(lims, process_id, output_file): if output_artifact.type == 'Analyte': for output_sample in output_artifact.samples: monster = output_sample.udf['Dx Monsternummer'] + if len(output_artifact.samples) > 1: + container = get_mix_sample_barcode(output_artifact) + else: + container = output_sample.udf['Dx Fractienummer'] well = ''.join(output_artifact.location[1].split(':')) output_data = ( - '{sample}\t{concentration:.2f}\t{sample_volume:.2f}\t{low_te_volume:.2f}\t{container}\t{well}\n'.format( - sample=monster, + '{sample}\t{concentration:.2f}\t{sample_volume:.2f}\t{low_te_volume:.2f}\t{container}\n'.format( + sample=output_sample.udf['Dx Fractienummer'], concentration=samples[monster]['conc'], sample_volume=samples[monster]['sample_volume'], low_te_volume=samples[monster]['low_te_volume'], - container=output_artifact.location[0].name, - well=well + container=container ) ) if well in output: From ed9d79eabcf75897bb83eb2b2c2fd499e07f91f1 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Fri, 18 Aug 2023 12:58:48 +0200 Subject: [PATCH 68/84] Refactor pipetting check --- clarity_epp/placement/pipetting.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/clarity_epp/placement/pipetting.py b/clarity_epp/placement/pipetting.py index a5f8d28..36094a4 100644 --- a/clarity_epp/placement/pipetting.py +++ b/clarity_epp/placement/pipetting.py @@ -2,6 +2,9 @@ from genologics.entities import Process +from .. import get_mix_sample_barcode + + def check_nunc_input_nunc_output(lims, process_id): """Check nuncs.""" process = Process(lims, id=process_id) @@ -10,13 +13,12 @@ def check_nunc_input_nunc_output(lims, process_id): input_nunc_1 = '' input_nunc_2 = '' output_nunc = '' - input_combined = '' + sample_mix = False if len(output_artifact.samples) > 1: - fraction = ('{fraction1}-{fraction2}'.format( - fraction1=(output_artifact.samples[0].udf['Dx Fractienummer']), - fraction2=(output_artifact.samples[1].udf['Dx Fractienummer']) - ) - ) + sample_mix = True + mix_name = get_mix_sample_barcode(output_artifact) + fraction1 = output_artifact.samples[0].udf['Dx Fractienummer'] + fraction2 = output_artifact.samples[1].udf['Dx Fractienummer'] else: fraction = output_artifact.samples[0].udf['Dx Fractienummer'] if 'Dx Sample 1 norm' in output_artifact.udf: @@ -25,13 +27,14 @@ def check_nunc_input_nunc_output(lims, process_id): input_nunc_2 = output_artifact.udf['Dx Sample 2 norm'] if 'Dx Sample (output)' in output_artifact.udf: output_nunc = output_artifact.udf['Dx Sample (output)'] - if input_nunc_1 and input_nunc_2: - input_combined = '{input_nunc_1}-{input_nunc_2}'.format(input_nunc_1=input_nunc_1, input_nunc_2=input_nunc_2) - elif input_nunc_1: - input_combined = input_nunc_1 - if input_combined and output_nunc: - if input_combined == output_nunc and output_nunc == fraction: + if sample_mix: + if input_nunc_1 == fraction1 and input_nunc_2 == fraction2 and output_nunc == mix_name: + output_artifact.udf['Dx pipetteer check'] = True + else: + output_artifact.udf['Dx pipetteer check'] = False + else: + if input_nunc_1 == fraction and input_nunc_2 == '' and output_nunc == fraction: output_artifact.udf['Dx pipetteer check'] = True else: output_artifact.udf['Dx pipetteer check'] = False - output_artifact.put() + output_artifact.put() From 67facf86b850f331fd5de35c3716390b7a009e27 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Fri, 18 Aug 2023 13:15:24 +0200 Subject: [PATCH 69/84] Switched samples also ok --- clarity_epp/placement/pipetting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clarity_epp/placement/pipetting.py b/clarity_epp/placement/pipetting.py index 36094a4..d4ef888 100644 --- a/clarity_epp/placement/pipetting.py +++ b/clarity_epp/placement/pipetting.py @@ -30,6 +30,8 @@ def check_nunc_input_nunc_output(lims, process_id): if sample_mix: if input_nunc_1 == fraction1 and input_nunc_2 == fraction2 and output_nunc == mix_name: output_artifact.udf['Dx pipetteer check'] = True + elif input_nunc_1 == fraction2 and input_nunc_2 == fraction1 and output_nunc == mix_name: + output_artifact.udf['Dx pipetteer check'] = True else: output_artifact.udf['Dx pipetteer check'] = False else: From 165bc5e40133f73a609bc349bd3fb9fd5665cb2b Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Tue, 22 Aug 2023 15:04:25 +0200 Subject: [PATCH 70/84] Fix resetting Dx norm. manueel --- clarity_epp/qc/qubit.py | 14 ++++++++------ clarity_epp/upload/tecan.py | 15 +++++++++------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/clarity_epp/qc/qubit.py b/clarity_epp/qc/qubit.py index c951cdf..4c2a88c 100644 --- a/clarity_epp/qc/qubit.py +++ b/clarity_epp/qc/qubit.py @@ -28,12 +28,14 @@ def set_qc_flag(lims, process_id, cutoff=10): artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = sample_measurements_average # Reset 'Dx norm. manueel' udf - if 'Dx Sample registratie zuivering' in process.all_inputs()[0].parent_process.type.name: - if sample_measurements_average <= 29.3: - artifact.samples[0].udf['Dx norm. manueel'] = True - else: - artifact.samples[0].udf['Dx norm. manueel'] = False - artifact.samples[0].put() + for analyte in process.analytes()[0]: + if analyte.name == artifact.name: + if 'Dx Sample registratie zuivering' in analyte.parent_process.type.name: + if sample_measurements_average <= 29.3: + artifact.samples[0].udf['Dx norm. manueel'] = True + else: + artifact.samples[0].udf['Dx norm. manueel'] = False + artifact.samples[0].put() if concentration_range[0] <= sample_measurements_average <= concentration_range[1]: if len(sample_measurements) == 1: diff --git a/clarity_epp/upload/tecan.py b/clarity_epp/upload/tecan.py index 82784bf..d0e33c0 100644 --- a/clarity_epp/upload/tecan.py +++ b/clarity_epp/upload/tecan.py @@ -78,12 +78,15 @@ def results_qc(lims, process_id): artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = sample_concentration # Reset 'Dx norm. manueel' udf - if 'Dx Sample registratie zuivering' in process.all_inputs()[0].parent_process.type.name: - if sample_concentration <= 29.3: - artifact.samples[0].udf['Dx norm. manueel'] = True - else: - artifact.samples[0].udf['Dx norm. manueel'] = False - artifact.samples[0].put() + if 'Dx Tecan std' not in artifact.name: + for analyte in process.analytes()[0]: + if analyte.name == artifact.name: + if 'Dx Sample registratie zuivering' in analyte.parent_process.type.name: + if sample_concentration <= 29.3: + artifact.samples[0].udf['Dx norm. manueel'] = True + else: + artifact.samples[0].udf['Dx norm. manueel'] = False + artifact.samples[0].put() # Set artifact Concentratie fluorescentie # Get artifact index == count From 58496e26b79857b1b0b8b89f27959ea8cfcdedb7 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Thu, 24 Aug 2023 11:05:02 +0200 Subject: [PATCH 71/84] Removed whitespace --- clarity_epp/__init__.py | 2 +- clarity_epp/export/labels.py | 2 +- clarity_epp/export/manual_pipetting.py | 14 +++++++------- clarity_epp/export/tecan.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/clarity_epp/__init__.py b/clarity_epp/__init__.py index 4a427e5..5bcd626 100644 --- a/clarity_epp/__init__.py +++ b/clarity_epp/__init__.py @@ -63,7 +63,7 @@ def get_mix_sample_barcode(artifact): if sample_names: for sample in artifact.samples: barcode_name += ''.join(sample_names[sample]) - + return barcode_name diff --git a/clarity_epp/export/labels.py b/clarity_epp/export/labels.py index 24cdf4f..3263c18 100644 --- a/clarity_epp/export/labels.py +++ b/clarity_epp/export/labels.py @@ -63,7 +63,7 @@ def nunc_mix_sample(lims, process_id, output_file): sample_mix = False if len(artifact.samples) > 1: sample_mix = True - + if sample_mix: barcode_name = get_mix_sample_barcode(artifact) output_file.write('{sample};;;;;{container}:{well};;1\n'.format( diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index abdfeb2..75572b6 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -739,7 +739,7 @@ def samplesheet_pool_magnis_pools(lims, process_id, output_file): def samplesheet_normalization_mix(lims, process_id, output_file): """"Create manual pipetting samplesheet for normalizing mix fraction samples.""" process = Process(lims, id=process_id) - + output_file.write( 'Fractienummer\tConcentratie (ng/ul)\tVolume sample (ul)\tVolume low TE (ul)\tContainer_tube\n' ) @@ -760,7 +760,7 @@ def samplesheet_normalization_mix(lims, process_id, output_file): for qc_sample in qc_artifact.samples: if qc_sample.name == input_sample.name: concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) - + else: parent_process = input_artifact.parent_process for parent_artifact in parent_process.all_inputs(): @@ -776,9 +776,9 @@ def samplesheet_normalization_mix(lims, process_id, output_file): else: # No QC process found, use Helix concentration concentration = input_sample.udf['Dx Concentratie (ng/ul)'] - + samples[input_sample.udf['Dx Monsternummer']] = {'conc': concentration} - + # Calculation of pipetting volumes for input_artifact in process.all_inputs(): output_artifact = process.outputs_per_input(input_artifact.id, Analyte=True)[0] # assume one artifact per input @@ -803,7 +803,7 @@ def samplesheet_normalization_mix(lims, process_id, output_file): low_te_volume = minuend - sample_volume samples[input_sample_1.udf['Dx Monsternummer']]['sample_volume'] = sample_volume samples[input_sample_1.udf['Dx Monsternummer']]['low_te_volume'] = low_te_volume - + if sample_mix: input_sample_2 = input_artifact.samples[1] if 'Dx sample vol. #2' in output_artifact.udf: @@ -819,7 +819,7 @@ def samplesheet_normalization_mix(lims, process_id, output_file): low_te_volume = minuend - sample_volume samples[input_sample_2.udf['Dx Monsternummer']]['sample_volume'] = sample_volume samples[input_sample_2.udf['Dx Monsternummer']]['low_te_volume'] = low_te_volume - + # Compose output per sample in well output = {} for output_artifact in process.all_outputs(): @@ -844,7 +844,7 @@ def samplesheet_normalization_mix(lims, process_id, output_file): output[well][monster] = output_data else: output[well] = {monster: output_data} - + # Write output file per sample sorted for well for well in clarity_epp.export.utils.sort_96_well_plate(output.keys()): for sample in output[well]: diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 96cac97..350bf11 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -86,7 +86,7 @@ def samplesheet(lims, process_id, type, output_file): sample_mix = False if len(artifact.samples) > 1: sample_mix = True - + if sample_mix: dividend = 880 max_volume = 30 @@ -101,7 +101,7 @@ def samplesheet(lims, process_id, type, output_file): samples[monster]['mix_names'] = artifact.name else: samples[monster]['mix_names'] = monster - + # Calculation of pipetting volumes calc_sample = dividend / samples[monster]['conc'] if calc_sample < 4: @@ -128,7 +128,7 @@ def samplesheet(lims, process_id, type, output_file): empty='', message=samples[monster]['message'] )) - + elif type == 'normalise': output_file.write('SourceTubeID;PositionID;PositionIndex\n') for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): From 8dd96c21d5f343afbaf2dde8d37ff4ff3794946e Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Thu, 24 Aug 2023 13:54:43 +0200 Subject: [PATCH 72/84] Fix for single samples --- clarity_epp/export/tecan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 350bf11..538882a 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -136,6 +136,7 @@ def samplesheet(lims, process_id, type, output_file): if len(artifact.samples) > 1: source_tube = get_mix_sample_barcode(artifact) else: + sample = artifact.samples[0] source_tube = sample.udf['Dx Fractienummer'] output_file.write('{sample};{well};{index}\n'.format( sample=source_tube, From 594bb26fba76e0c8bce0ce9678957fb9d3e9817e Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Thu, 24 Aug 2023 16:35:42 +0200 Subject: [PATCH 73/84] Only samples to add to plate in samplesheet --- clarity_epp/export/tecan.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 538882a..814fda4 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -131,15 +131,20 @@ def samplesheet(lims, process_id, type, output_file): elif type == 'normalise': output_file.write('SourceTubeID;PositionID;PositionIndex\n') + outputs = [] + for output in process.all_outputs(): + if output.name not in ['Dx labels nunc', 'Dx pipetteerschema manueel normaliseren', 'Dx Fluent480 samplesheet manueel normaliseren']: + outputs.append(output.name) for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): artifact = well_plate[well] - if len(artifact.samples) > 1: - source_tube = get_mix_sample_barcode(artifact) - else: - sample = artifact.samples[0] - source_tube = sample.udf['Dx Fractienummer'] - output_file.write('{sample};{well};{index}\n'.format( - sample=source_tube, - well=well, - index=clarity_epp.export.utils.get_well_index(well, one_based=True) - )) \ No newline at end of file + if artifact.name in outputs: + if len(artifact.samples) > 1: + source_tube = get_mix_sample_barcode(artifact) + else: + sample = artifact.samples[0] + source_tube = sample.udf['Dx Fractienummer'] + output_file.write('{sample};{well};{index}\n'.format( + sample=source_tube, + well=well, + index=clarity_epp.export.utils.get_well_index(well, one_based=True) + )) \ No newline at end of file From 019c51d4087eac30e76ec82d9e70d07a386d8704 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Fri, 25 Aug 2023 13:40:53 +0200 Subject: [PATCH 74/84] Fix resetting Dx norm. manueel --- clarity_epp/qc/qubit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/qc/qubit.py b/clarity_epp/qc/qubit.py index 4c2a88c..fcc07b6 100644 --- a/clarity_epp/qc/qubit.py +++ b/clarity_epp/qc/qubit.py @@ -29,7 +29,7 @@ def set_qc_flag(lims, process_id, cutoff=10): # Reset 'Dx norm. manueel' udf for analyte in process.analytes()[0]: - if analyte.name == artifact.name: + if analyte.name == sample: if 'Dx Sample registratie zuivering' in analyte.parent_process.type.name: if sample_measurements_average <= 29.3: artifact.samples[0].udf['Dx norm. manueel'] = True From d05e48090a427db23c584d3a355f56af320022ce Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 25 Aug 2023 14:18:52 +0200 Subject: [PATCH 75/84] Update meetw --- config.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 7551f0b..967ab42 100755 --- a/config.py +++ b/config.py @@ -32,20 +32,24 @@ # Export meetw protocol steps WES meetw_zui_wes_processes = [ 'Dx Sample registratie zuivering v1.1', + 'Dx Sample registratie zuivering v1.2', 'Dx Hamilton uitvullen v1.1', 'Dx Hamilton zuiveren v1.1', 'Dx Zuiveren gDNA manueel v1.1', 'Dx manueel gezuiverd placement v1.2', 'Dx gDNA Normalisatie Caliper v1.1', 'Dx Uitvullen en zuiveren (Fluent 480) v1.0', + 'Dx Uitvullen en zuiveren (Fluent 480) v1.1', 'Dx Normaliseren (Fluent 480) v1.0', - 'Dx gDNA handmatige normalisatie WES v1.0' + 'Dx gDNA handmatige normalisatie WES v1.0', + 'Dx gDNA handmatige normalisatie WES v1.1' ] meetw_sampleprep_wes_processes = [ 'Dx Fragmenteren v1.0', - 'Dx Library Prep & Target Enrichment Magnis v1.0' - ] + 'Dx Library Prep & Target Enrichment Magnis v1.0', + 'Dx Library Prep & Target Enrichment Magnis v1.1', +] meetw_seq_wes_processes = [ 'Dx Multiplexen Enrichment pools Magnis v1.0', From e03718543abf2e038271893b5507db3e717f42c4 Mon Sep 17 00:00:00 2001 From: Linda Walraven-van Oijen Date: Fri, 25 Aug 2023 15:34:22 +0200 Subject: [PATCH 76/84] Add automation for setting udf manual normalise --- clarity_epp.py | 10 ++++++---- clarity_epp/placement/artifact.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/clarity_epp.py b/clarity_epp.py index 2945953..5427656 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -203,12 +203,14 @@ def placement_automatic(args): clarity_epp.placement.plate.copy_layout(lims, args.process_id) -def placement_artifact_set_name(args): - """Change artifact name to sequence name.""" +def placement_artifact_set(args): + """Change artifact name or udf.""" if args.type == 'sequence_name': clarity_epp.placement.artifact.set_sequence_name(lims, args.process_id) elif args.type == 'run_id': clarity_epp.placement.artifact.set_runid_name(lims, args.process_id) + elif args.type == 'norm_udf': + clarity_epp.placement.artifact.set_norm_manual_udf(lims, args.process_id) def placement_route_artifact(args): @@ -430,9 +432,9 @@ def placement_pipetting(args): parser_placement_automatic.set_defaults(func=placement_automatic) parser_placement_artifact = subparser_placement.add_parser('artifact', help='Change artifact name to sequence name') - parser_placement_artifact.add_argument('type', choices=['sequence_name', 'run_id'], help='Check type') + parser_placement_artifact.add_argument('type', choices=['sequence_name', 'run_id', 'norm_udf'], help='Check type') parser_placement_artifact.add_argument('process_id', help='Clarity lims process id') - parser_placement_artifact.set_defaults(func=placement_artifact_set_name) + parser_placement_artifact.set_defaults(func=placement_artifact_set) parser_placement_route_artifact = subparser_placement.add_parser('route_artifact', help='Route artifact to a workflow') parser_placement_route_artifact.add_argument('process_id', help='Clarity lims process id') diff --git a/clarity_epp/placement/artifact.py b/clarity_epp/placement/artifact.py index a070342..aeead75 100644 --- a/clarity_epp/placement/artifact.py +++ b/clarity_epp/placement/artifact.py @@ -52,3 +52,15 @@ def route_to_workflow(lims, process_id, workflow): elif workflow == 'sequencing': lims.route_artifacts(artifacts_completed, workflow_uri=Workflow(lims, id=config.sequencing_workflow).uri) + + +def set_norm_manual_udf(lims, process_id): + """Combine mix sample udfs 'Dx norm. manueel'.""" + process = Process(lims, id=process_id) + + for artifact in process.all_outputs(): + artifact.udf['Dx norm. manueel'] = False + for sample in artifact.samples: + if sample.udf['Dx norm. manueel'] == True: + artifact.udf['Dx norm. manueel'] = True + artifact.put() \ No newline at end of file From 805c65b109ac1725d82dcfcef25c2366279bf112 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 19 Sep 2023 10:46:16 +0200 Subject: [PATCH 77/84] fix bug --- clarity_epp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/__init__.py b/clarity_epp/__init__.py index 5bcd626..88590fc 100644 --- a/clarity_epp/__init__.py +++ b/clarity_epp/__init__.py @@ -21,7 +21,7 @@ def get_sequence_name(artifact): if sample_numbers: sequence_name = '-'.join(sample_numbers) else: # non Dx sample - sequence_name = artifact.sample[0].name + sequence_name = artifact.samples[0].name return sequence_name From a882f1e07cfa23a754f1aaa42810429de9d0141d Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 19 Sep 2023 11:55:45 +0200 Subject: [PATCH 78/84] Sort sample numbers --- clarity_epp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/__init__.py b/clarity_epp/__init__.py index 88590fc..e5ba3f7 100644 --- a/clarity_epp/__init__.py +++ b/clarity_epp/__init__.py @@ -19,7 +19,7 @@ def get_sequence_name(artifact): sample_numbers.append(sample.udf['Dx Monsternummer']) if sample_numbers: - sequence_name = '-'.join(sample_numbers) + sequence_name = '-'.join(sorted(sample_numbers)) else: # non Dx sample sequence_name = artifact.samples[0].name From 4c1217ec210680f869d72627078772ced72bd108 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 19 Sep 2023 11:59:03 +0200 Subject: [PATCH 79/84] Loop over samples --- clarity_epp/export/labels.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/clarity_epp/export/labels.py b/clarity_epp/export/labels.py index 3263c18..946a2a4 100644 --- a/clarity_epp/export/labels.py +++ b/clarity_epp/export/labels.py @@ -12,7 +12,7 @@ def container(lims, process_id, output_file, description=''): if description: if ',' in description: output_file.write('{description}\t{container}\r\n'.format( - description=description.split(',')[index], + description=description.split(',')[index], container=container.name )) else: @@ -28,8 +28,8 @@ def container_sample(lims, process_id, output_file, description=''): for artifact in container.placements.values(): if description: output_file.write('{description}\t{sample}\t{container}\r\n'.format( - description=description, - container=container.name, + description=description, + container=container.name, sample=artifact.name )) else: @@ -44,11 +44,12 @@ def storage_location(lims, process_id, output_file): output_file.write('Bakje\tpos\n') for artifact in process.analytes()[0]: - storage_location = artifact.samples[0].udf['Dx Opslaglocatie'].split() - output_file.write('{tray}\t{pos}\n'.format( - tray=storage_location[0][2:6], # Select 4 digits from: CB[1-9][1-9][1-9][1-9]KK - pos=storage_location[1] - )) + for sample in artifact.samples: + storage_location = sample.udf['Dx Opslaglocatie'].split() + output_file.write('{tray}\t{pos}\n'.format( + tray=storage_location[0][2:6], # Select 4 digits from: CB[1-9][1-9][1-9][1-9]KK + pos=storage_location[1] + )) def nunc_mix_sample(lims, process_id, output_file): From 9ea720d26fd0d09dcdf7997406cb2b1348be22cc Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 21 Sep 2023 10:20:50 +0200 Subject: [PATCH 80/84] Add Carriage return needed for brady printers --- clarity_epp/export/labels.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/clarity_epp/export/labels.py b/clarity_epp/export/labels.py index 946a2a4..28c5320 100644 --- a/clarity_epp/export/labels.py +++ b/clarity_epp/export/labels.py @@ -41,12 +41,12 @@ def storage_location(lims, process_id, output_file): process = Process(lims, id=process_id) # Write header - output_file.write('Bakje\tpos\n') + output_file.write('Bakje\tpos\r\n') for artifact in process.analytes()[0]: for sample in artifact.samples: storage_location = sample.udf['Dx Opslaglocatie'].split() - output_file.write('{tray}\t{pos}\n'.format( + output_file.write('{tray}\t{pos}\r\n'.format( tray=storage_location[0][2:6], # Select 4 digits from: CB[1-9][1-9][1-9][1-9]KK pos=storage_location[1] )) @@ -56,8 +56,8 @@ def nunc_mix_sample(lims, process_id, output_file): """Generate (mix) sample nunc label file.""" process = Process(lims, id=process_id) - #Write empty header - output_file.write('\n') + # Write empty header + output_file.write('\r\n') for artifact in process.analytes()[0]: well = ''.join(artifact.location[1].split(':')) @@ -67,13 +67,13 @@ def nunc_mix_sample(lims, process_id, output_file): if sample_mix: barcode_name = get_mix_sample_barcode(artifact) - output_file.write('{sample};;;;;{container}:{well};;1\n'.format( + output_file.write('{sample};;;;;{container}:{well};;1\r\n'.format( sample=barcode_name, container=artifact.container.name, well=well )) else: - output_file.write('{sample};;;;;{container}:{well};;1\n'.format( + output_file.write('{sample};;;;;{container}:{well};;1\r\n'.format( sample=artifact.samples[0].udf['Dx Fractienummer'], container=artifact.container.name, well=well From 64e2e7433e0be718510307d67bd3ef4a599b148c Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 21 Sep 2023 16:01:24 +0200 Subject: [PATCH 81/84] Clarify function --- clarity_epp/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clarity_epp/__init__.py b/clarity_epp/__init__.py index e5ba3f7..9c033f3 100644 --- a/clarity_epp/__init__.py +++ b/clarity_epp/__init__.py @@ -12,15 +12,15 @@ def get_sequence_name(artifact): - """Generate sequence name.""" + """Generate sequence name, for combined or single samples.""" sample_numbers = [] for sample in artifact.samples: - if 'Dx Monsternummer' in sample.udf: + if 'Dx Monsternummer' in sample.udf: # Use monsternummer for Dx samples sample_numbers.append(sample.udf['Dx Monsternummer']) - if sample_numbers: + if sample_numbers: # Merge monsternummer for Dx samples sequence_name = '-'.join(sorted(sample_numbers)) - else: # non Dx sample + else: # Use sample name for non Dx samples sequence_name = artifact.samples[0].name return sequence_name From aa3fa930037de443bea3a26eb5618f81d7ba3c67 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 21 Sep 2023 16:13:21 +0200 Subject: [PATCH 82/84] Add ',' --- config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 967ab42..55d4581 100755 --- a/config.py +++ b/config.py @@ -42,7 +42,7 @@ 'Dx Uitvullen en zuiveren (Fluent 480) v1.1', 'Dx Normaliseren (Fluent 480) v1.0', 'Dx gDNA handmatige normalisatie WES v1.0', - 'Dx gDNA handmatige normalisatie WES v1.1' + 'Dx gDNA handmatige normalisatie WES v1.1', ] meetw_sampleprep_wes_processes = [ @@ -57,7 +57,7 @@ 'Dx Library pool denatureren en laden (NovaSeq) v1.3', 'AUTOMATED - NovaSeq Run (NovaSeq 6000 v3.1)', 'Dx QC controle Lab sequencen v1.1', - 'Dx NovaSeq QC controle Lab sequencen v1.3' + 'Dx NovaSeq QC controle Lab sequencen v1.3', ] # Export meetw protocol steps MIP @@ -69,7 +69,7 @@ 'Dx Capture v1.0', 'Dx Exonuclease behandeling v1.0', 'Dx PCR na exonuclease behandeling v1.0', - 'Dx smMIP multiplexen & BBSS sequence pool v1.0' + 'Dx smMIP multiplexen & BBSS sequence pool v1.0', ] meetw_seq_mip_processes = [ From a37765fa29fddc56987107dc2f388322f7b59a77 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 26 Sep 2023 10:14:05 +0200 Subject: [PATCH 83/84] Only lookup meetw and vcf files for WES samples --- clarity_epp/export/workflow.py | 114 ++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 53 deletions(-) diff --git a/clarity_epp/export/workflow.py b/clarity_epp/export/workflow.py index 840c00b..e238e1e 100644 --- a/clarity_epp/export/workflow.py +++ b/clarity_epp/export/workflow.py @@ -31,61 +31,69 @@ def helix_magnis(lims, process_id, output_file): for artifact in process.all_inputs(): for sample in artifact.samples: - if 'Dx Werklijstnummer' in sample.udf: # Only check samples with a 'Werklijstnummer' - sample_artifacts = lims.get_artifacts(samplelimsid=sample.id, type='Analyte') - # Filter artifacts without parent_process - sample_artifacts = [sample_artifact for sample_artifact in sample_artifacts if sample_artifact.parent_process] - # Sort artifact by parent process id - sample_artifacts = sorted( - sample_artifacts, - key=lambda artifact: int(artifact.parent_process.id.split('-')[-1]) - ) - - sample_all_processes = {} - # reset after 'Dx Sample registratie zuivering' process - # this is a new import from helix, should not be counted as a repeat - sample_filter_processes = {} - - for sample_artifact in sample_artifacts: - if 'Dx Sample registratie zuivering' in sample_artifact.parent_process.type.name: - sample_filter_processes = {} # reset after new helix import - process_id = sample_artifact.parent_process.id - process_name = sample_artifact.parent_process.type.name - - if process_name in sample_all_processes: - sample_all_processes[process_name].add(process_id) - else: - sample_all_processes[process_name] = set([process_id]) - - if process_name in sample_filter_processes: - sample_filter_processes[process_name].add(process_id) - else: - sample_filter_processes[process_name] = set([process_id]) + # Only output samples with a 'Werklijstnummer' + if 'Dx Werklijstnummer' in sample.udf: + # Setup empty vars + meetw_zui, meetw_zui_herh, meetw_sampleprep, meetw_sampleprep_herh, meetw_seq, meetw_seq_herh = [''] * 6 + sequence_name, gatk_vcf, exomedepth_vcf = [''] * 3 + + # Only lookup meetw and vcf files for WES samples + if sample.udf['Dx Stoftest code'] == config.stoftestcode_wes: + sample_artifacts = lims.get_artifacts(samplelimsid=sample.id, type='Analyte') + # Filter artifacts without parent_process + sample_artifacts = [sample_artifact for sample_artifact in sample_artifacts if sample_artifact.parent_process] + # Sort artifact by parent process id + sample_artifacts = sorted( + sample_artifacts, + key=lambda artifact: int(artifact.parent_process.id.split('-')[-1]) + ) - # Determine meetw - repeat_cutoff = len(sample.udf['Dx Werklijstnummer'].split(';')) * 2 - meetw_zui, meetw_zui_herh = determin_meetw( - config.meetw_zui_wes_processes, sample_all_processes, repeat_cutoff - ) - meetw_sampleprep, meetw_sampleprep_herh = determin_meetw( - config.meetw_sampleprep_wes_processes, sample_filter_processes, 2 - ) - meetw_seq, meetw_seq_herh = determin_meetw( - config.meetw_seq_wes_processes, sample_filter_processes, 2 - ) + sample_all_processes = {} + # reset after 'Dx Sample registratie zuivering' process + # this is a new import from helix, should not be counted as a repeat + sample_filter_processes = {} + + for sample_artifact in sample_artifacts: + if 'Dx Sample registratie zuivering' in sample_artifact.parent_process.type.name: + sample_filter_processes = {} # reset after new helix import + process_id = sample_artifact.parent_process.id + process_name = sample_artifact.parent_process.type.name + + if process_name in sample_all_processes: + sample_all_processes[process_name].add(process_id) + else: + sample_all_processes[process_name] = set([process_id]) + + if process_name in sample_filter_processes: + sample_filter_processes[process_name].add(process_id) + else: + sample_filter_processes[process_name] = set([process_id]) + + # Determine meetw + repeat_cutoff = len(sample.udf['Dx Werklijstnummer'].split(';')) * 2 + meetw_zui, meetw_zui_herh = determin_meetw( + config.meetw_zui_wes_processes, sample_all_processes, repeat_cutoff + ) + meetw_sampleprep, meetw_sampleprep_herh = determin_meetw( + config.meetw_sampleprep_wes_processes, sample_filter_processes, 2 + ) + meetw_seq, meetw_seq_herh = determin_meetw( + config.meetw_seq_wes_processes, sample_filter_processes, 2 + ) - # Determine vcf files - gatk_vcf = '' - exomedepth_vcf = '' - if 'Dx GATK vcf' in artifact.udf: - gatk_vcf = artifact.udf['Dx GATK vcf'] - elif 'Dx GATK vcf' in artifact.input_artifact_list()[0].udf: # Look one more step back. - gatk_vcf = artifact.input_artifact_list()[0].udf['Dx GATK vcf'] + # Determine vcf files + sequence_name = get_sequence_name(artifact) + gatk_vcf = '' + exomedepth_vcf = '' + if 'Dx GATK vcf' in artifact.udf: + gatk_vcf = artifact.udf['Dx GATK vcf'] + elif 'Dx GATK vcf' in artifact.input_artifact_list()[0].udf: # Look one more step back. + gatk_vcf = artifact.input_artifact_list()[0].udf['Dx GATK vcf'] - if 'Dx ExomeDepth vcf' in artifact.udf: - exomedepth_vcf = artifact.udf['Dx ExomeDepth vcf'] - elif 'Dx ExomeDepth vcf' in artifact.input_artifact_list()[0].udf: # Look one more step back. - exomedepth_vcf = artifact.input_artifact_list()[0].udf['Dx ExomeDepth vcf'] + if 'Dx ExomeDepth vcf' in artifact.udf: + exomedepth_vcf = artifact.udf['Dx ExomeDepth vcf'] + elif 'Dx ExomeDepth vcf' in artifact.input_artifact_list()[0].udf: # Look one more step back. + exomedepth_vcf = artifact.input_artifact_list()[0].udf['Dx ExomeDepth vcf'] output_file.write(( "{meet_id}\t{werklijst}\t{onderzoeksnummer}\t{monsternummer}\t{meetw_zui}\t{meetw_zui_herh}\t" @@ -100,7 +108,7 @@ def helix_magnis(lims, process_id, output_file): meetw_sampleprep=meetw_sampleprep, meetw_sampleprep_herh=meetw_sampleprep_herh, meetw_seq=meetw_seq, meetw_seq_herh=meetw_seq_herh, meetw_bfx='J', - sample_name=get_sequence_name(artifact), + sample_name=sequence_name, vcf_file=gatk_vcf, cnv_vcf_file=exomedepth_vcf, ) From 0bdb960458550714ab8c1a3293838ce8c892c977 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 26 Sep 2023 11:17:22 +0200 Subject: [PATCH 84/84] Pep8 --- clarity_epp/export/workflow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clarity_epp/export/workflow.py b/clarity_epp/export/workflow.py index e238e1e..b7c62f7 100644 --- a/clarity_epp/export/workflow.py +++ b/clarity_epp/export/workflow.py @@ -41,7 +41,9 @@ def helix_magnis(lims, process_id, output_file): if sample.udf['Dx Stoftest code'] == config.stoftestcode_wes: sample_artifacts = lims.get_artifacts(samplelimsid=sample.id, type='Analyte') # Filter artifacts without parent_process - sample_artifacts = [sample_artifact for sample_artifact in sample_artifacts if sample_artifact.parent_process] + sample_artifacts = [ + sample_artifact for sample_artifact in sample_artifacts if sample_artifact.parent_process + ] # Sort artifact by parent process id sample_artifacts = sorted( sample_artifacts,