From 56d1bfcd6237f74077189c64620c7f9fa1fc83f9 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 15 Dec 2020 13:16:05 +0100 Subject: [PATCH 01/53] Remove 'zuiveren' from hamilton parent process if --- clarity_epp/export/caliper.py | 47 +++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/clarity_epp/export/caliper.py b/clarity_epp/export/caliper.py index beece0c..8586779 100755 --- a/clarity_epp/export/caliper.py +++ b/clarity_epp/export/caliper.py @@ -4,29 +4,34 @@ import utils + def samplesheet_normalise(lims, process_id, output_file): """Create Caliper samplesheet for normalising 96 well plate.""" - output_file.write('Monsternummer\tPlate_Id_input\tWell\tPlate_Id_output\tPipetteervolume DNA (ul)\tPipetteervolume H2O (ul)\n') + output_file.write( + 'Monsternummer\tPlate_Id_input\tWell\tPlate_Id_output\tPipetteervolume DNA (ul)\tPipetteervolume H2O (ul)\n' + ) process = Process(lims, id=process_id) parent_processes = [] parent_process_barcode_manual = 'None' parent_process_barcode_hamilton = 'None' + for p in process.parent_processes(): - if 'Dx manueel gezuiverd placement' in p.type.name: + if p.type.name.startswith('Dx manueel gezuiverd placement'): for pp in p.parent_processes(): parent_processes.append(pp) parent_process_barcode_manual = p.output_containers()[0].name - if 'Dx Hamilton zuiveren' in p.type.name: + elif p.type.name.startswith('Dx Hamilton'): parent_processes.append(p) parent_process_barcode_hamilton = p.output_containers()[0].name - if 'Dx Zuiveren gDNA manueel' in p.type.name: + elif p.type.name.startswith('Dx Zuiveren gDNA manueel'): parent_processes.append(p) + if parent_process_barcode_hamilton != 'None': parent_process_barcode = parent_process_barcode_hamilton else: parent_process_barcode = parent_process_barcode_manual - parent_processes = list(set(parent_processes)) - process_types = [] + + # Get all Qubit and Tecan Spark QC types types = [] process_types = lims.get_process_types() for pt in process_types: @@ -34,16 +39,18 @@ def samplesheet_normalise(lims, process_id, output_file): types.append(pt.name) elif 'Dx Tecan Spark 10M QC' in pt.name: types.append(pt.name) + + # Get all unique input artifact ids + parent_processes = list(set(parent_processes)) input_artifact_ids = [] for p in parent_processes: for analyte in p.all_outputs(): input_artifact_ids.append(analyte.id) input_artifact_ids = list(set(input_artifact_ids)) - qc_processes = lims.get_processes( - type=[types], - inputartifactlimsid=input_artifact_ids - ) - qc_processes = list(set(qc_processes)) + + # Get unique QC processes for input artifacts + qc_processes = list(set(lims.get_processes(type=[types], inputartifactlimsid=input_artifact_ids))) + samples_measurements_qubit = {} sample_concentration = {} samples_measurements_tecan = {} @@ -146,11 +153,13 @@ def samplesheet_normalise(lims, process_id, output_file): volume_H2O[placement] = output_ul-int(round(float(output_ng)/conc[placement])) for well in utils.sort_96_well_plate(monsternummer.keys()): - output_file.write('{monsternummer}\t{plate_id_input}\t{position}\t{plate_id_output}\t{volume_DNA}\t{volume_H2O}\n'.format( - monsternummer=monsternummer[well], - plate_id_input=parent_process_barcode, - position=well, - plate_id_output=output_plate_barcode, - volume_DNA=volume_DNA[well], - volume_H2O=volume_H2O[well] - )) + output_file.write( + '{monsternummer}\t{plate_id_input}\t{position}\t{plate_id_output}\t{volume_DNA}\t{volume_H2O}\n'.format( + monsternummer=monsternummer[well], + plate_id_input=parent_process_barcode, + position=well, + plate_id_output=output_plate_barcode, + volume_DNA=volume_DNA[well], + volume_H2O=volume_H2O[well] + ) + ) From 6d7aae7fc65c0ef40285c03373ef8de7ca8e2e15 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 16 Dec 2020 16:07:46 +0100 Subject: [PATCH 02/53] Fix for output with samples not in current process --- clarity_epp/export/caliper.py | 113 ++++++++++++++++------------------ 1 file changed, 53 insertions(+), 60 deletions(-) diff --git a/clarity_epp/export/caliper.py b/clarity_epp/export/caliper.py index 8586779..3ef4a5f 100755 --- a/clarity_epp/export/caliper.py +++ b/clarity_epp/export/caliper.py @@ -11,6 +11,7 @@ def samplesheet_normalise(lims, process_id, output_file): 'Monsternummer\tPlate_Id_input\tWell\tPlate_Id_output\tPipetteervolume DNA (ul)\tPipetteervolume H2O (ul)\n' ) process = Process(lims, id=process_id) + process_samples = [artifact.name for artifact in process.analytes()[0]] parent_processes = [] parent_process_barcode_manual = 'None' parent_process_barcode_hamilton = 'None' @@ -32,13 +33,12 @@ def samplesheet_normalise(lims, process_id, output_file): parent_process_barcode = parent_process_barcode_manual # Get all Qubit and Tecan Spark QC types - types = [] - process_types = lims.get_process_types() - for pt in process_types: - if 'Dx Qubit QC' in pt.name: - types.append(pt.name) - elif 'Dx Tecan Spark 10M QC' in pt.name: - types.append(pt.name) + qc_process_types = [] + for process_type in lims.get_process_types(): + if 'Dx Qubit QC' in process_type.name: + qc_process_types.append(process_type.name) + elif 'Dx Tecan Spark 10M QC' in process_type.name: + qc_process_types.append(process_type.name) # Get all unique input artifact ids parent_processes = list(set(parent_processes)) @@ -49,7 +49,7 @@ def samplesheet_normalise(lims, process_id, output_file): input_artifact_ids = list(set(input_artifact_ids)) # Get unique QC processes for input artifacts - qc_processes = list(set(lims.get_processes(type=[types], inputartifactlimsid=input_artifact_ids))) + qc_processes = list(set(lims.get_processes(type=[qc_process_types], inputartifactlimsid=input_artifact_ids))) samples_measurements_qubit = {} sample_concentration = {} @@ -74,48 +74,40 @@ def samplesheet_normalise(lims, process_id, output_file): output_ul = process.udf['Eindvolume (ul) genormaliseerd gDNA'] output_plate_barcode = process.output_containers()[0].name - for p in qc_processes: - if 'Dx Qubit QC' in p.type.name: - for a in p.all_outputs(): - if 'Tecan' not in a.name and 'check' not in a.name and 'Label' not in a.name: - if 'Dx Conc. goedgekeurde meting (ng/ul)' in a.udf: - machine = 'Qubit' - sample = a.samples[0].name - measurement = a.udf['Dx Conc. goedgekeurde meting (ng/ul)'] - if sample in samples_measurements_qubit: - samples_measurements_qubit[sample].append(measurement) - else: - samples_measurements_qubit[sample] = [measurement] - else: - sample = a.samples[0].name - if sample not in sample_concentration: - sample_concentration[sample] = 'geen' - elif 'Dx Tecan Spark 10M QC' in p.type.name: - for a in p.all_outputs(): - if 'Tecan' not in a.name and 'check' not in a.name and 'Label' not in a.name: - if 'Dx Conc. goedgekeurde meting (ng/ul)' in a.udf: - machine = 'Tecan' - sample = a.samples[0].name - measurement = a.udf['Dx Conc. goedgekeurde meting (ng/ul)'] - if sample in samples_measurements_tecan: - samples_measurements_tecan[sample].append(measurement) - else: - samples_measurements_tecan[sample] = [measurement] - else: - sample = a.samples[0].name - if sample not in sample_concentration: - sample_concentration[sample] = 'geen' - for p in qc_processes: - for a in p.all_outputs(): - if 'Tecan' not in a.name and 'check' not in a.name and 'Label' not in a.name: - if 'Dx Tecan Spark 10M QC' in p.type.name: - if 'Dx Conc. goedgekeurde meting (ng/ul)' in a.udf: - machine = 'Tecan' - sample = a.samples[0].name - elif 'Dx Qubit QC' in p.type.name: - if 'Dx Conc. goedgekeurde meting (ng/ul)' in a.udf: - machine = 'Qubit' - sample = a.samples[0].name + for qc_process in qc_processes: + if 'Dx Qubit QC' in qc_process.type.name: + for artifact in qc_process.all_outputs(): + sample = artifact.samples[0].name + if sample in process_samples and not any(keyword in artifact.name for keyword in ['Tecan', 'check', 'Label']): + if 'Dx Conc. goedgekeurde meting (ng/ul)' in artifact.udf: + measurement = artifact.udf['Dx Conc. goedgekeurde meting (ng/ul)'] + if sample not in samples_measurements_qubit: + samples_measurements_qubit[sample] = [] + samples_measurements_qubit[sample].append(measurement) + elif sample not in sample_concentration: + sample_concentration[sample] = 'geen' + + elif 'Dx Tecan Spark 10M QC' in qc_process.type.name: + for artifact in qc_process.all_outputs(): + sample = artifact.samples[0].name + if sample in process_samples and not any(keyword in artifact.name for keyword in ['Tecan', 'check', 'Label']): + if 'Dx Conc. goedgekeurde meting (ng/ul)' in artifact.udf: + measurement = artifact.udf['Dx Conc. goedgekeurde meting (ng/ul)'] + if sample not in samples_measurements_tecan: + samples_measurements_tecan[sample] = [] + samples_measurements_tecan[sample].append(measurement) + elif sample not in sample_concentration: + sample_concentration[sample] = 'geen' + + for qc_process in qc_processes: + for artifact in qc_process.all_outputs(): + sample = artifact.samples[0].name + if not any(keyword in artifact.name for keyword in ['Tecan', 'check', 'Label']): + if 'Dx Tecan Spark 10M QC' in qc_process.type.name and 'Dx Conc. goedgekeurde meting (ng/ul)' in artifact.udf: + machine = 'Tecan' + elif 'Dx Qubit QC' in qc_process.type.name and 'Dx Conc. goedgekeurde meting (ng/ul)' in artifact.udf: + machine = 'Qubit' + if sample not in sample_concentration or machine == 'Qubit': if sample in samples_measurements_tecan or sample in samples_measurements_qubit: if machine == 'Tecan': @@ -141,16 +133,17 @@ def samplesheet_normalise(lims, process_id, output_file): for placement, artifact in process.output_containers()[0].placements.iteritems(): sample = artifact.samples[0].name - placement = ''.join(placement.split(':')) - monsternummer[placement] = sample - conc_measured[placement] = sample_concentration[sample] - if conc_measured[placement] != 'geen': - if output_ng/conc_measured[placement] > 50: - conc[placement] = output_ng/50 - else: - conc[placement] = conc_measured[placement] - volume_DNA[placement] = int(round(float(output_ng)/conc[placement])) - volume_H2O[placement] = output_ul-int(round(float(output_ng)/conc[placement])) + if sample in process_samples: + placement = ''.join(placement.split(':')) + monsternummer[placement] = sample + conc_measured[placement] = sample_concentration[sample] + if conc_measured[placement] != 'geen': + if output_ng/conc_measured[placement] > 50: + conc[placement] = output_ng/50 + else: + conc[placement] = conc_measured[placement] + volume_DNA[placement] = int(round(float(output_ng)/conc[placement])) + volume_H2O[placement] = output_ul-int(round(float(output_ng)/conc[placement])) for well in utils.sort_96_well_plate(monsternummer.keys()): output_file.write( From c5ec2f2223ac718bdb1dd5a3cd870beaed748ecb Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 26 Jan 2021 15:51:01 +0100 Subject: [PATCH 03/53] Add workflow update scripts --- scripts/move_samples_to_new_workflow.py | 36 ++++++++++++++++ scripts/update_workflow.sh | 55 +++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 scripts/move_samples_to_new_workflow.py create mode 100755 scripts/update_workflow.sh diff --git a/scripts/move_samples_to_new_workflow.py b/scripts/move_samples_to_new_workflow.py new file mode 100644 index 0000000..e86f35c --- /dev/null +++ b/scripts/move_samples_to_new_workflow.py @@ -0,0 +1,36 @@ +from genologics.lims import Lims +from genologics.entities import Workflow, Step, Queue, Stage + +import config + +# Setup lims connection +baseuri = 'https://usf-lims.umcutrecht.nl' +username = 'lims_user' +password = 'lims_user_password' +lims = Lims(baseuri, username, password) + +old_queue = Queue(lims, id='') # Example: Dx Sample registratie zuivering v1.0 = 1142 +old_stage = Stage(lims, uri='') # Example: Dx Sample registratie zuivering v1.0 = https://usf-lims.umcutrecht.nl/api/v2/configuration/workflows/751/stages/2119 +new_workflow = Workflow(lims, id='') # Example: Dx Exoom KAPA v1.6 = 901 + +artifacts = [] +print '# Old queue:', len(old_queue.artifacts) + +for artifact in old_queue.artifacts: + if artifact.name not in ['']: # Add sample that should not be moved + print 'Move:', artifact.name + artifacts.append(artifact) + else: + print 'Keep:', artifact.name + +print '# Move to new workflow:', len(artifacts) + +# Move things to new workflow, uncomment to enable. +# lims.route_artifacts(artifacts, stage_uri=old_stage.uri, unassign=True) +# lims.route_artifacts(artifacts, workflow_uri=new_workflow.uri) + +old_queue = Queue(lims, id='') # Example: Dx Sample registratie zuivering v1.0 = 1142 +print '# Old queue:', len(old_queue.artifacts) + +new_queue = Queue(lims, id='') # Example: Dx Sample registratie zuivering v1.1 = 1342 +print '# New queue:', len(new_queue.artifacts) \ No newline at end of file diff --git a/scripts/update_workflow.sh b/scripts/update_workflow.sh new file mode 100755 index 0000000..787ea2a --- /dev/null +++ b/scripts/update_workflow.sh @@ -0,0 +1,55 @@ +# + +# Step 1a - Export workflow +java -jar config-slicer.jar \ + -o custom \ + -s \ + -u \ + -p \ + -m _workflowManifest.txt \ + -w '' + +# Step 1b - Export workflow +java -jar config-slicer.jar \ + -o export \ + -s \ + -u \ + -p \ + -m _workflowManifest.txt \ + -k _workflowManifest.xml + +# Step 2 - Compare workflow +java -jar config-slicer.jar \ + -o validate \ + -s \ + -u \ + -p \ + -k _workflowManifest.xml \ + > validate_before.out + +# Step 3a - Import workflow +java -jar config-slicer.jar \ + -o import \ + -s \ + -u \ + -p \ + -k _workflowManifest.xml \ + > import.out + +# Step 3b - Import workflow +# Needs promt don't redirect output! +java -jar config-slicer.jar \ + -o importAndOverwrite \ + -s \ + -u \ + -p \ + -k _workflowManifest.xml + +# Step 4 - Compare workflow +java -jar config-slicer.jar \ + -o validate \ + -s \ + -u \ + -p \ + -k _workflowManifest.xml \ + > validate_after.out From 025fd2de528996c6d86c37048bf81e8eb474d08a Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 27 Jan 2021 11:54:02 +0100 Subject: [PATCH 04/53] Update --- scripts/move_samples_to_new_workflow.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/move_samples_to_new_workflow.py b/scripts/move_samples_to_new_workflow.py index e86f35c..aee3e36 100644 --- a/scripts/move_samples_to_new_workflow.py +++ b/scripts/move_samples_to_new_workflow.py @@ -9,15 +9,16 @@ password = 'lims_user_password' lims = Lims(baseuri, username, password) -old_queue = Queue(lims, id='') # Example: Dx Sample registratie zuivering v1.0 = 1142 +old_queue = Queue(lims, id='') # Example: Dx Sample registratie zuivering v1.0 = 1142 (same as step id) old_stage = Stage(lims, uri='') # Example: Dx Sample registratie zuivering v1.0 = https://usf-lims.umcutrecht.nl/api/v2/configuration/workflows/751/stages/2119 new_workflow = Workflow(lims, id='') # Example: Dx Exoom KAPA v1.6 = 901 +skip_artifacts = [''] # Add samples that should not be moved to new workflow artifacts = [] print '# Old queue:', len(old_queue.artifacts) for artifact in old_queue.artifacts: - if artifact.name not in ['']: # Add sample that should not be moved + if artifact.name not in skip_artifacts: print 'Move:', artifact.name artifacts.append(artifact) else: @@ -26,11 +27,11 @@ print '# Move to new workflow:', len(artifacts) # Move things to new workflow, uncomment to enable. -# lims.route_artifacts(artifacts, stage_uri=old_stage.uri, unassign=True) -# lims.route_artifacts(artifacts, workflow_uri=new_workflow.uri) +# lims.route_artifacts(artifacts, stage_uri=old_stage.uri, unassign=True) # remove from old stage +# lims.route_artifacts(artifacts, workflow_uri=new_workflow.uri) # add to new workflow -old_queue = Queue(lims, id='') # Example: Dx Sample registratie zuivering v1.0 = 1142 +old_queue = Queue(lims, id='') # Example: Dx Sample registratie zuivering v1.0 = 1142 (same as step id) print '# Old queue:', len(old_queue.artifacts) -new_queue = Queue(lims, id='') # Example: Dx Sample registratie zuivering v1.1 = 1342 -print '# New queue:', len(new_queue.artifacts) \ No newline at end of file +new_queue = Queue(lims, id='') # Example: Dx Sample registratie zuivering v1.1 = 1342 (same as step id) +print '# New queue:', len(new_queue.artifacts) From 0a4f1d43a10824de4472ddd2538eaab48f12f1d6 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 27 Jan 2021 13:43:15 +0100 Subject: [PATCH 05/53] Clarify parameter --- scripts/update_workflow.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/update_workflow.sh b/scripts/update_workflow.sh index 787ea2a..3f21c2d 100755 --- a/scripts/update_workflow.sh +++ b/scripts/update_workflow.sh @@ -6,8 +6,9 @@ java -jar config-slicer.jar \ -s \ -u \ -p \ - -m _workflowManifest.txt \ - -w '' + -w '' \ + -m _workflowManifest.txt + # Step 1b - Export workflow java -jar config-slicer.jar \ @@ -15,8 +16,8 @@ java -jar config-slicer.jar \ -s \ -u \ -p \ - -m _workflowManifest.txt \ - -k _workflowManifest.xml + -m _workflowManifest.txt \ + -k _workflowManifest.xml # Step 2 - Compare workflow java -jar config-slicer.jar \ @@ -24,7 +25,7 @@ java -jar config-slicer.jar \ -s \ -u \ -p \ - -k _workflowManifest.xml \ + -k _workflowManifest.xml \ > validate_before.out # Step 3a - Import workflow @@ -33,7 +34,7 @@ java -jar config-slicer.jar \ -s \ -u \ -p \ - -k _workflowManifest.xml \ + -k _workflowManifest.xml \ > import.out # Step 3b - Import workflow @@ -43,7 +44,7 @@ java -jar config-slicer.jar \ -s \ -u \ -p \ - -k _workflowManifest.xml + -k _workflowManifest.xml # Step 4 - Compare workflow java -jar config-slicer.jar \ From e15dfaa7a4c6e0acabf1450bcc189a15ee982b13 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 27 Jan 2021 14:26:50 +0100 Subject: [PATCH 06/53] Ignore tmp folder --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 47369c1..6b28362 100755 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,6 @@ ENV/ # Mac OS .DS_Store + +# temp scripts +tmp \ No newline at end of file From 01f34f9ec6c15e656aec0ea4748caee17aa4a204 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 2 Feb 2021 15:04:13 +0100 Subject: [PATCH 07/53] First commit management review --- scripts/management_review.py | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 scripts/management_review.py diff --git a/scripts/management_review.py b/scripts/management_review.py new file mode 100644 index 0000000..330c2bd --- /dev/null +++ b/scripts/management_review.py @@ -0,0 +1,100 @@ +from collections import OrderedDict +import re + +from genologics.lims import Lims +from genologics.entities import Artifact, SampleHistory, Protocol + +# Setup lims connection +baseuri = 'https://usf-lims.umcutrecht.nl' +username = 'lims_user' +password = 'lims_user_password' +lims = Lims(baseuri, username, password) + +# Get DX projects and filter on year based on project name +dx_projects = [project for project in lims.get_projects(udf={'Application': 'DX'}) if project.name.startswith('Dx WL20')] +sample_count = 0 + +# Expected actions +action_list = ['total', 'complete', 'nextstep', 'remove', 'rework', 'repeat', 'completerepeat', 'store', 'leave', 'None'] + +# Expected processes +processes = [ + 'Dx Sample registratie zuivering', 'Dx Hamilton uitvullen', 'Dx Hamilton zuiveren', + 'Dx Zuiveren gDNA manueel', 'Dx manueel gezuiverd placement', 'Dx gDNA Normalisatie Caliper', 'Dx Fragmenteren & BBSS', + 'Dx LibraryPrep Caliper KAPA', 'Dx Library Prep amplificatie & clean up KAPA', 'Dx Multiplexen library prep', + 'Dx Enrichment DNA fragments', 'Dx Post Enrichment clean up', 'Dx Aliquot Post Enrichment (clean)', + 'Dx Post Enrichment PCR & clean up', 'Dx Aliquot Post Enrichment PCR (clean)', + 'Dx Library pool verdunnen', 'Dx Multiplexen library pool', 'Dx Multiplexen sequence pool', + 'Dx Library pool denatureren en laden (NovaSeq)', 'Dx Automated NovaSeq Run (standaard)', + 'Dx Library pool denatureren en laden (Nextseq)', 'Dx NextSeq Run', + 'Dx Library pool denatureren en laden (NovaSeq) Dx QC controle Lab sequencen', + 'Dx Library pool denatureren en laden (Nextseq) Dx QC controle Lab sequencen', + 'Dx NGS labwerk afronden', 'Dx Bioinformatica analyses', 'Dx NGS onderzoeken afronden', +] + +# Append combined process name for qc processes +qc_processes = [ + 'Dx Qubit QC', 'Dx Tecan Spark 10M QC', 'Dx Bioanalyzer QC', + 'Dx Tapestation 2200 QC', 'Dx Tapestation 4200 QC', 'Dx Aggregate QC' +] +processes_before_qc = [ + 'Dx Hamilton zuiveren', 'Dx Zuiveren gDNA manueel', 'Dx Fragmenteren & BBSS', 'Dx LibraryPrep Caliper KAPA', + 'Dx Library Prep amplificatie & clean up KAPA', 'Dx Post Enrichment PCR & clean up', 'Dx Multiplexen library pool' +] +for process in processes_before_qc: + for qc_process in qc_processes: + processes.append("{0} {1}".format(process, qc_process)) +processes.extend(qc_processes) + +# Setup count dictonary +process_action_counts = OrderedDict() +for process in processes: + process_action_counts[process] = {} + for action in action_list: + process_action_counts[process][action] = 0 + +for project in dx_projects: + for sample in lims.get_samples(projectlimsid=project.id): + sample_count += 1 + for artifact in lims.get_artifacts(samplelimsid=sample.id, resolve=True, type='Analyte'): + for process in lims.get_processes(inputartifactlimsid=artifact.id): + + # Parse process name, remov version. Add to count dictonary if not yet defined. + process_name = re.sub(r' v\d\.\d', '', process.type.name) + if 'QC' in process_name and artifact.parent_process: + process_name = "{0} {1}".format(re.sub(r' v\d\.\d', '', artifact.parent_process.type.name), process_name) + + # Find output artifacts + output_artifacts = [] + if process.all_outputs(): # outputs_per_input returns TypeError if there is no output + output_artifacts = [artifact.id for artifact in process.outputs_per_input(artifact.id, Analyte=True)] + + # For 'Bioinformatica analyses' and 'NGS onderzoeken afronden' we need to demux to get correct artifact + if process_name in ['Dx Bioinformatica analyses', 'Dx NGS onderzoeken afronden']: + artifact_demux = lims.get('/demux?'.join(artifact.uri.split('?'))) + for node in artifact_demux.getiterator('artifact'): + if node.find('samples') and len(node.find('samples').findall('sample')) == 1: + demux_artifact = Artifact(lims, uri=node.attrib['uri']) + if sample == demux_artifact.samples[0]: # 1 sample per artifact. + output_artifacts = [demux_artifact.id] + break + + # Some processes do not create an output artifcat (mainly QC) + if not output_artifacts: + output_artifacts = [artifact.id] + + # Get action for artifact + for action in process.step.actions.get_next_actions(): + if action['artifact'].id in output_artifacts: + process_action_counts[process_name]['total'] += 1 + process_action_counts[process_name][str(action['action'])] += 1 + +print 'Sample counts' +print 'Total Sample count: {0}'.format(str(sample_count)) +print 'Process\t{action_list}'.format(action_list='\t'.join(action_list)) +for process in process_action_counts: + if process_action_counts[process]['total']: + print '{process}\t{action_list}'.format( + process=process, + action_list='\t'.join([str(process_action_counts[process][action]) if action in process_action_counts[process] else '0' for action in action_list]) + ) From af56017eacd1f553926e6514528e666722c19ef8 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 4 Feb 2021 11:13:38 +0100 Subject: [PATCH 08/53] Update removed samples, solves #28 --- clarity_epp/export/sample.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/clarity_epp/export/sample.py b/clarity_epp/export/sample.py index d9448d8..3ca39c4 100644 --- a/clarity_epp/export/sample.py +++ b/clarity_epp/export/sample.py @@ -21,7 +21,7 @@ def removed_samples(lims, output_file): removed_process_id = None removed_date = None - for artifact in sorted(lims.get_artifacts(type="Analyte", sample_name=sample.name), key=utils.sort_artifact_list): + for artifact in sorted(lims.get_artifacts(type="Analyte", sample_name=sample.name, resolve=True), key=utils.sort_artifact_list): for stage, status, name in artifact.workflow_stages_and_statuses: if status == 'REMOVED': # Store removed from workflow sample_removed = True @@ -83,7 +83,7 @@ def removed_samples(lims, output_file): removed_date = None else: # Work done on other sample after removal from workflow - for artifact in lims.get_artifacts(type="Analyte", sample_name=person_sample.name): + for artifact in lims.get_artifacts(type="Analyte", sample_name=person_sample.name, resolve=True): for stage, status, name in artifact.workflow_stages_and_statuses: if status == 'COMPLETE': processes = list(set(lims.get_processes(inputartifactlimsid=artifact.id, type=name))) @@ -105,8 +105,11 @@ def removed_samples(lims, output_file): sample_removed_status = '' if 'Dx sample status verwijderd' in sample.udf: sample_removed_status = sample.udf['Dx sample status verwijderd'] - - if sample_removed_status != 'NIB afgerond': + + elapsed_time = datetime.datetime.now() - removed_date + + # Filter samples on sample_removed_status and elapsed month (6 => 6*31 = 186 days) since removal. + if (not sample_removed_status or sample_removed_status == 'Nieuw materiaal aangevraagd') and elapsed_time.days < 186: output_file.write('{removed_date}\t{name}\t{project}\t{werklijst}\t{onderzoek}\t{indicatie}\t{stage}\t{removed_status}\n'.format( removed_date=removed_date.strftime('%Y-%m-%d'), name=sample.name, From 63eee8f683fe18158ce1708c712ef2cb77083c49 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 4 Feb 2021 16:17:08 +0100 Subject: [PATCH 09/53] Samplesheet manual normalization --- clarity_epp.py | 6 ++-- clarity_epp/export/manual_pipetting.py | 44 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/clarity_epp.py b/clarity_epp.py index 50fe8b3..c60f58c 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -69,6 +69,8 @@ def export_manual_pipetting(args): clarity_epp.export.manual_pipetting.samplesheet_multiplex_library_pool(lims, args.process_id, args.output_file) elif args.type == 'multiplex_sequence_pool': clarity_epp.export.manual_pipetting.samplesheet_multiplex_sequence_pool(lims, args.process_id, args.output_file) + elif args.type == 'normalization': + clarity_epp.export.manual_pipetting.samplesheet_normalization(lims, args.process_id, args.output_file) def export_ped_file(args): @@ -219,8 +221,8 @@ def placement_complete_step(args): parser_export_labels.add_argument('-d', '--description', nargs='?', help='Container name description') parser_export_labels.set_defaults(func=export_labels) - parser_export_manual_pipetting = subparser_export.add_parser('manual', help='Create manual pipetting _exports', parents=[output_parser]) - parser_export_manual_pipetting.add_argument('type', choices=['purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool'], help='Samplesheet type') + parser_export_manual_pipetting = subparser_export.add_parser('manual', help='Create manual pipetting exports', parents=[output_parser]) + parser_export_manual_pipetting.add_argument('type', choices=['purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization'], help='Samplesheet type') parser_export_manual_pipetting.add_argument('process_id', help='Clarity lims process id') parser_export_manual_pipetting.set_defaults(func=export_manual_pipetting) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index ded4091..84a7f3e 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -324,3 +324,47 @@ def samplesheet_multiplex_sequence_pool(lims, process_id, output_file): tris_HCL_uL = 150 - total_load_uL output_file.write('{0}\t{1:.2f}\n'.format('Tris-HCL', tris_HCL_uL)) + + +def samplesheet_normalization(lims, process_id, output_file): + """Create manual pipetting samplesheet for normalizing samples.""" + output_file.write('Sample\tConcentration (ng/ul)\tEindvolume (ul)\tVerdunningsfactor\tVolume sample (ul)\tVolume water (ul)\n') + process = Process(lims, id=process_id) + + # Find all QC process types + qc_process_types = [] + for process_type in lims.get_process_types(): + if 'Dx Qubit QC' in process_type.name: + qc_process_types.append(process_type.name) + elif 'Dx Tecan Spark 10M QC' in process_type.name: + qc_process_types.append(process_type.name) + + for container in process.output_containers(): + artifact = container.placements['1:1'] # asume tubes + input_artifact = artifact.input_artifact_list()[0] # asume one input artifact + sample = artifact.samples[0] # asume one sample per tube + + # Find last qc process for artifact + qc_process = sorted( + lims.get_processes(type=qc_process_types, inputartifactlimsid=input_artifact.id), + key=lambda process: int(process.id.split('-')[-1]) + )[-1] + + # Find concentration measurement + for qc_artifact in qc_process.outputs_per_input(input_artifact.id): + if 'Dx Concentratie fluorescentie (ng/ul)' in qc_artifact.udf: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + + final_volume = artifact.udf['Dx Eindvolume (ul)'] + dilution_factor = concentration / 20 + sample_volume = final_volume / dilution_factor + water_volume = final_volume - sample_volume + + output_file.write('{sample}\t{concentration}\t{final_volume}\t{dilution_factor:.1f}\t{sample_volume:.1f}\t{water_volume:.1f}\n'.format( + sample=sample.name, + concentration=concentration, + final_volume=final_volume, + dilution_factor=dilution_factor, + sample_volume=sample_volume, + water_volume=water_volume + )) \ No newline at end of file From 28adbb47de67e7e3216e3a1c3d247bf2e8222416 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 4 Feb 2021 16:25:15 +0100 Subject: [PATCH 10/53] Round concentration. --- clarity_epp/export/manual_pipetting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 84a7f3e..60084d3 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -360,7 +360,7 @@ def samplesheet_normalization(lims, process_id, output_file): sample_volume = final_volume / dilution_factor water_volume = final_volume - sample_volume - output_file.write('{sample}\t{concentration}\t{final_volume}\t{dilution_factor:.1f}\t{sample_volume:.1f}\t{water_volume:.1f}\n'.format( + output_file.write('{sample}\t{concentration:.1f}\t{final_volume}\t{dilution_factor:.1f}\t{sample_volume:.1f}\t{water_volume:.1f}\n'.format( sample=sample.name, concentration=concentration, final_volume=final_volume, From 65a8ac58eafa201add76a0c31b961d6b2602fa67 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 8 Feb 2021 09:26:01 +0100 Subject: [PATCH 11/53] Fix for plate placements --- clarity_epp/export/manual_pipetting.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 60084d3..7e8f378 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -339,10 +339,9 @@ def samplesheet_normalization(lims, process_id, output_file): elif 'Dx Tecan Spark 10M QC' in process_type.name: qc_process_types.append(process_type.name) - for container in process.output_containers(): - artifact = container.placements['1:1'] # asume tubes - input_artifact = artifact.input_artifact_list()[0] # asume one input artifact - sample = artifact.samples[0] # asume one sample per tube + for input_artifact in process.all_inputs(resolve=True): + artifact = process.outputs_per_input(input_artifact.id, Analyte=True)[0] # assume one artifact per input + sample = input_artifact.samples[0] # asume one sample per input artifact # Find last qc process for artifact qc_process = sorted( @@ -367,4 +366,4 @@ def samplesheet_normalization(lims, process_id, output_file): dilution_factor=dilution_factor, sample_volume=sample_volume, water_volume=water_volume - )) \ No newline at end of file + )) From 2a2ba0878cc16cf24eefe463733ebe3491480ec1 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 16 Feb 2021 14:21:59 +0100 Subject: [PATCH 12/53] Skip research samples and fix output_artifacts. --- scripts/management_review.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/management_review.py b/scripts/management_review.py index 330c2bd..90420ef 100644 --- a/scripts/management_review.py +++ b/scripts/management_review.py @@ -55,6 +55,8 @@ for project in dx_projects: for sample in lims.get_samples(projectlimsid=project.id): + if sample.udf['Dx Onderzoeksreden'] == 'Research': # skip research + continue sample_count += 1 for artifact in lims.get_artifacts(samplelimsid=sample.id, resolve=True, type='Analyte'): for process in lims.get_processes(inputartifactlimsid=artifact.id): @@ -67,7 +69,7 @@ # Find output artifacts output_artifacts = [] if process.all_outputs(): # outputs_per_input returns TypeError if there is no output - output_artifacts = [artifact.id for artifact in process.outputs_per_input(artifact.id, Analyte=True)] + output_artifacts = [output_artifact.id for output_artifact in process.outputs_per_input(artifact.id, Analyte=True)] # For 'Bioinformatica analyses' and 'NGS onderzoeken afronden' we need to demux to get correct artifact if process_name in ['Dx Bioinformatica analyses', 'Dx NGS onderzoeken afronden']: @@ -89,7 +91,6 @@ process_action_counts[process_name]['total'] += 1 process_action_counts[process_name][str(action['action'])] += 1 -print 'Sample counts' print 'Total Sample count: {0}'.format(str(sample_count)) print 'Process\t{action_list}'.format(action_list='\t'.join(action_list)) for process in process_action_counts: From 26345062c246624a3802c5bcd9042d3ab6119622 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 16 Feb 2021 15:13:22 +0100 Subject: [PATCH 13/53] Create capture samplesheet --- clarity_epp.py | 4 +++- clarity_epp/export/manual_pipetting.py | 32 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/clarity_epp.py b/clarity_epp.py index c60f58c..f87e3bc 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -71,6 +71,8 @@ def export_manual_pipetting(args): clarity_epp.export.manual_pipetting.samplesheet_multiplex_sequence_pool(lims, args.process_id, args.output_file) elif args.type == 'normalization': clarity_epp.export.manual_pipetting.samplesheet_normalization(lims, args.process_id, args.output_file) + elif args.type == 'capture': + clarity_epp.export.manual_pipetting.samplesheet_capture(lims, args.process_id, args.output_file) def export_ped_file(args): @@ -222,7 +224,7 @@ def placement_complete_step(args): parser_export_labels.set_defaults(func=export_labels) parser_export_manual_pipetting = subparser_export.add_parser('manual', help='Create manual pipetting exports', parents=[output_parser]) - parser_export_manual_pipetting.add_argument('type', choices=['purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization'], help='Samplesheet type') + parser_export_manual_pipetting.add_argument('type', choices=['purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization', 'capture'], help='Samplesheet type') parser_export_manual_pipetting.add_argument('process_id', help='Clarity lims process id') parser_export_manual_pipetting.set_defaults(func=export_manual_pipetting) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 7e8f378..d49fd4b 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -367,3 +367,35 @@ def samplesheet_normalization(lims, process_id, output_file): sample_volume=sample_volume, water_volume=water_volume )) + + +def samplesheet_capture(lims, process_id, output_file): + """Create manual pipetting samplesheet for capture protocol.""" + process = Process(lims, id=process_id) + sample_count = len(process.analytes()[0]) + + # Hardcode for now all input paramters -> maybe use an UDF? + data = [ + ['Ampligase Buffer 10X', 2.5], + ['MIP pool werkoplossing', 0.04], + ['*dNTP 0.25mM', 0.03], + ['Hemo Klentaq 10U/ul', 0.32], + ['Ampligase 100U/ul', 0.01], + ['water', 17.10], + ] + + # Caculate for sample count + for i, item in enumerate(data): + data[i].append(sample_count * item[1] * 1.1) + + # Calculate final volume + data.append([ + 'ul MM in elke well', + sum([item[1] for item in data]), + sum([item[2] for item in data]), + ]) + + # Write samplesheet + output_file.write('Mastermix\t1\t{0}\n'.format(sample_count)) + for item in data: + output_file.write('{0}\t{1}\t{2}\n'.format(item[0], item[1], item[2])) From 630a0ae086630c9f375c66f06fc2fbe6b35f9afb Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 17 Feb 2021 14:22:09 +0100 Subject: [PATCH 14/53] Use more precise numbers --- clarity_epp/export/manual_pipetting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index d49fd4b..553ff75 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -377,11 +377,11 @@ def samplesheet_capture(lims, process_id, output_file): # Hardcode for now all input paramters -> maybe use an UDF? data = [ ['Ampligase Buffer 10X', 2.5], - ['MIP pool werkoplossing', 0.04], - ['*dNTP 0.25mM', 0.03], + ['MIP pool werkoplossing', 0.0410325795234137], + ['*dNTP 0.25mM', 0.032], ['Hemo Klentaq 10U/ul', 0.32], ['Ampligase 100U/ul', 0.01], - ['water', 17.10], + ['water', 17.0969674204766], ] # Caculate for sample count @@ -398,4 +398,4 @@ def samplesheet_capture(lims, process_id, output_file): # Write samplesheet output_file.write('Mastermix\t1\t{0}\n'.format(sample_count)) for item in data: - output_file.write('{0}\t{1}\t{2}\n'.format(item[0], item[1], item[2])) + output_file.write('{0}\t{1:.2f}\t{2:.2f}\n'.format(item[0], item[1], item[2])) From 4cea0eb8c7a28734b2e913c06af6416506a19dd7 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 19 Feb 2021 11:34:28 +0100 Subject: [PATCH 15/53] Use process udf instead of hardcoded values. --- clarity_epp/export/manual_pipetting.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 553ff75..38a1647 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -376,12 +376,12 @@ def samplesheet_capture(lims, process_id, output_file): # Hardcode for now all input paramters -> maybe use an UDF? data = [ - ['Ampligase Buffer 10X', 2.5], - ['MIP pool werkoplossing', 0.0410325795234137], - ['*dNTP 0.25mM', 0.032], - ['Hemo Klentaq 10U/ul', 0.32], - ['Ampligase 100U/ul', 0.01], - ['water', 17.0969674204766], + ['Ampligase Buffer 10X', process.udf['Ampligase Buffer 10X']], + ['MIP pool werkoplossing', process.udf['MIP pool werkoplossing']], + ['*dNTP 0.25mM', process.udf['*dNTP 0.25mM']], + ['Hemo Klentaq 10U/ul', process.udf['Hemo Klentaq 10U/ul']], + ['Ampligase 100U/ul', process.udf['Ampligase 100U/ul']], + ['Water', process.udf['Water']], ] # Caculate for sample count From 0a38a275062d868750c53f9f95afe470878ffb0e Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 3 Mar 2021 12:26:52 +0100 Subject: [PATCH 16/53] Add exonuclease and pcr_exonuclease samplesheets --- clarity_epp.py | 6 ++- clarity_epp/export/manual_pipetting.py | 61 +++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/clarity_epp.py b/clarity_epp.py index f87e3bc..741ee19 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -73,6 +73,10 @@ def export_manual_pipetting(args): clarity_epp.export.manual_pipetting.samplesheet_normalization(lims, args.process_id, args.output_file) elif args.type == 'capture': clarity_epp.export.manual_pipetting.samplesheet_capture(lims, args.process_id, args.output_file) + elif args.type == 'exonuclease': + clarity_epp.export.manual_pipetting.sammplesheet_exonuclease(lims, args.process_id, args.output_file) + elif args.type == 'pcr_exonuclease': + clarity_epp.export.manual_pipetting.sammplesheet_pcr_exonuclease(lims, args.process_id, args.output_file) def export_ped_file(args): @@ -224,7 +228,7 @@ def placement_complete_step(args): parser_export_labels.set_defaults(func=export_labels) parser_export_manual_pipetting = subparser_export.add_parser('manual', help='Create manual pipetting exports', parents=[output_parser]) - parser_export_manual_pipetting.add_argument('type', choices=['purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization', 'capture'], help='Samplesheet type') + parser_export_manual_pipetting.add_argument('type', choices=['purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization', 'capture', 'exonuclease', 'pcr_exonuclease'], help='Samplesheet type') parser_export_manual_pipetting.add_argument('process_id', help='Clarity lims process id') parser_export_manual_pipetting.set_defaults(func=export_manual_pipetting) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 38a1647..3e06166 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -374,7 +374,7 @@ def samplesheet_capture(lims, process_id, output_file): process = Process(lims, id=process_id) sample_count = len(process.analytes()[0]) - # Hardcode for now all input paramters -> maybe use an UDF? + # All input paramters data = [ ['Ampligase Buffer 10X', process.udf['Ampligase Buffer 10X']], ['MIP pool werkoplossing', process.udf['MIP pool werkoplossing']], @@ -399,3 +399,62 @@ def samplesheet_capture(lims, process_id, output_file): output_file.write('Mastermix\t1\t{0}\n'.format(sample_count)) for item in data: output_file.write('{0}\t{1:.2f}\t{2:.2f}\n'.format(item[0], item[1], item[2])) + + +def sammplesheet_exonuclease(lims, process_id, output_file): + """Create manual pipetting samplesheet for Exonuclease protocol""" + process = Process(lims, id=process_id) + sample_count = len(process.analytes()[0]) + + # All input paramters + data = [ + ['EXO I', process.udf['EXO I']], + ['EXO III', process.udf['EXO III']], + ['Ampligase buffer 10X', process.udf['Ampligase buffer 10X']], + ['H2O', process.udf['H2O']], + ] + + # Caculate for sample count + for i, item in enumerate(data): + data[i].append(sample_count * item[1] * 1.25) + + # Calculate total + data.append([ + 'TOTAL (incl. 25% overmaat)', + sum([item[1] for item in data]), + sum([item[2] for item in data]), + ]) + + # Write samplesheet + output_file.write('\tMaster Mix (ul)\t{0}\n'.format(sample_count)) + for item in data: + output_file.write('{0}\t{1:.2f}\t{2:.2f}\n'.format(item[0], item[1], item[2])) + + +def sammplesheet_pcr_exonuclease(lims, process_id, output_file): + """Create manual pipetting samplesheet for PCR after Exonuclease protocol""" + process = Process(lims, id=process_id) + sample_count = len(process.analytes()[0]) + + # All input paramters + data = [ + ['2X iProof', process.udf['2X iProof']], + ['Illumina forward primer(100µM) MIP_OLD_BB_FOR', process.udf['Illumina forward primer(100µM) MIP_OLD_BB_FOR']], + ['H2O', process.udf['H2O']], + ] + + # Caculate for sample count + for i, item in enumerate(data): + data[i].append(sample_count * item[1] * 1.1) + + # Calculate total + data.append([ + 'TOTAL (incl. 10% overmaat)', + sum([item[1] for item in data]), + sum([item[2] for item in data]), + ]) + + # Write samplesheet + output_file.write('\tMaster Mix (ul)\t{0}\n'.format(sample_count)) + for item in data: + output_file.write('{0}\t{1:.2f}\t{2:.2f}\n'.format(item[0], item[1], item[2])) From d0a3dc7a86a31f795179c7856166b4dc0c5c0fc7 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 3 Mar 2021 12:43:08 +0100 Subject: [PATCH 17/53] Replace micro char with u. --- clarity_epp/export/manual_pipetting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 3e06166..33a8a95 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -439,7 +439,7 @@ def sammplesheet_pcr_exonuclease(lims, process_id, output_file): # All input paramters data = [ ['2X iProof', process.udf['2X iProof']], - ['Illumina forward primer(100µM) MIP_OLD_BB_FOR', process.udf['Illumina forward primer(100µM) MIP_OLD_BB_FOR']], + ['Illumina forward primer(100uM) MIP_OLD_BB_FOR', process.udf['Illumina forward primer(100uM) MIP_OLD_BB_FOR']], ['H2O', process.udf['H2O']], ] From 3d6d3f5b4626425a996dc79e40a231717907929c Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 17 Mar 2021 17:31:28 +0100 Subject: [PATCH 18/53] Create error if Concentratie UDF does not exists --- clarity_epp/export/manual_pipetting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 33a8a95..880226b 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -351,8 +351,7 @@ def samplesheet_normalization(lims, process_id, output_file): # Find concentration measurement for qc_artifact in qc_process.outputs_per_input(input_artifact.id): - if 'Dx Concentratie fluorescentie (ng/ul)' in qc_artifact.udf: - concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) final_volume = artifact.udf['Dx Eindvolume (ul)'] dilution_factor = concentration / 20 From ced048c53288c179e4dcbb3922433ee890992877 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 18 Mar 2021 15:02:04 +0100 Subject: [PATCH 19/53] Add concentration parsing to tapestation --- clarity_epp/upload/tapestation.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/clarity_epp/upload/tapestation.py b/clarity_epp/upload/tapestation.py index 77a8376..4af5e48 100644 --- a/clarity_epp/upload/tapestation.py +++ b/clarity_epp/upload/tapestation.py @@ -6,7 +6,8 @@ def results(lims, process_id): """Upload tapestation results to artifacts.""" process = Process(lims, id=process_id) - sample_measurements = {} + sample_size_measurements = {} + sample_concentration_measurements = {} # Parse File for output in process.all_outputs(unique=True): @@ -15,10 +16,12 @@ def results(lims, process_id): for line in lims.get_file_contents(tapestation_result_file.id).split('\n'): if line.startswith('FileName'): header = line.split(',') - if 'Size [bp]' in header: - size_index = header.index('Size [bp]') # Tapestation compact peak table - else: - size_index = header.index('Average Size [bp]') # Tapestation compact region table + if 'Size [bp]' in header: # Tapestation compact peak table + size_index = header.index('Size [bp]') + concentration_index = None + else: # Tapestation compact region table + size_index = header.index('Average Size [bp]') + concentration_index = header.index('Conc. [pg/\xb5l]') # micro sign sample_index = header.index('Sample Description') elif line: @@ -27,12 +30,17 @@ def results(lims, process_id): if sample != 'Ladder': if data[size_index]: size = int(data[size_index]) - sample_measurements[sample] = size + sample_size_measurements[sample] = size + if concentration_index and data[concentration_index]: + concentration = float(data[concentration_index]) / 1000 # pg/ul to ng/ul + sample_concentration_measurements[sample] = concentration # Set UDF for artifact in process.all_outputs(): if artifact.name not in ['TapeStation Output', 'TapeStation Samplesheet', 'TapeStation Sampleplots PDF']: sample_name = artifact.name.split('_')[0] - if sample_name in sample_measurements: - artifact.udf['Dx Fragmentlengte (bp)'] = sample_measurements[sample_name] - artifact.put() + if sample_name in sample_size_measurements: + artifact.udf['Dx Fragmentlengte (bp)'] = sample_size_measurements[sample_name] + if sample_name in sample_concentration_measurements: + artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = sample_size_measurements[sample_name] + artifact.put() From c3698e2d4ac4e4c81e7e3ffcc45b036a37a7b8d7 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 24 Mar 2021 12:31:41 +0100 Subject: [PATCH 20/53] Add u to fix micro sign --- clarity_epp/upload/tapestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/upload/tapestation.py b/clarity_epp/upload/tapestation.py index 4af5e48..4a1f593 100644 --- a/clarity_epp/upload/tapestation.py +++ b/clarity_epp/upload/tapestation.py @@ -21,7 +21,7 @@ def results(lims, process_id): concentration_index = None else: # Tapestation compact region table size_index = header.index('Average Size [bp]') - concentration_index = header.index('Conc. [pg/\xb5l]') # micro sign + concentration_index = header.index(u'Conc. [pg/\xb5l]') # micro sign sample_index = header.index('Sample Description') elif line: From c9ecd8a1afac925b91bf5832fcf7a1f39bdcec66 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 26 Mar 2021 14:48:40 +0100 Subject: [PATCH 21/53] Fix sample_concentration_measurements --- clarity_epp/upload/tapestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/upload/tapestation.py b/clarity_epp/upload/tapestation.py index 4a1f593..2cb3a20 100644 --- a/clarity_epp/upload/tapestation.py +++ b/clarity_epp/upload/tapestation.py @@ -42,5 +42,5 @@ def results(lims, process_id): if sample_name in sample_size_measurements: artifact.udf['Dx Fragmentlengte (bp)'] = sample_size_measurements[sample_name] if sample_name in sample_concentration_measurements: - artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = sample_size_measurements[sample_name] + artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = sample_concentration_measurements[sample_name] artifact.put() From 01a0499918d05780605db22d4ff67c83edd2c8e5 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 7 Apr 2021 15:31:27 +0200 Subject: [PATCH 22/53] Add samplesheet_mip_multiplex_pool --- clarity_epp.py | 11 ++++- clarity_epp/export/manual_pipetting.py | 65 ++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/clarity_epp.py b/clarity_epp.py index 741ee19..3b08d92 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -77,6 +77,8 @@ def export_manual_pipetting(args): clarity_epp.export.manual_pipetting.sammplesheet_exonuclease(lims, args.process_id, args.output_file) elif args.type == 'pcr_exonuclease': clarity_epp.export.manual_pipetting.sammplesheet_pcr_exonuclease(lims, args.process_id, args.output_file) + elif args.type == 'mip_multiplex_pool': + clarity_epp.export.manual_pipetting.samplesheet_mip_multiplex_pool(lims, args.process_id, args.output_file) def export_ped_file(args): @@ -228,7 +230,14 @@ def placement_complete_step(args): parser_export_labels.set_defaults(func=export_labels) parser_export_manual_pipetting = subparser_export.add_parser('manual', help='Create manual pipetting exports', parents=[output_parser]) - parser_export_manual_pipetting.add_argument('type', choices=['purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization', 'capture', 'exonuclease', 'pcr_exonuclease'], help='Samplesheet type') + parser_export_manual_pipetting.add_argument( + 'type', + choices=[ + 'purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization', + 'capture', 'exonuclease', 'pcr_exonuclease', 'mip_multiplex_pool' + ], + help='Samplesheet type' + ) parser_export_manual_pipetting.add_argument('process_id', help='Clarity lims process id') parser_export_manual_pipetting.set_defaults(func=export_manual_pipetting) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 880226b..20ae3c4 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -457,3 +457,68 @@ def sammplesheet_pcr_exonuclease(lims, process_id, output_file): output_file.write('\tMaster Mix (ul)\t{0}\n'.format(sample_count)) for item in data: output_file.write('{0}\t{1:.2f}\t{2:.2f}\n'.format(item[0], item[1], item[2])) + + +def samplesheet_mip_multiplex_pool(lims, process_id, output_file): + """Create manual pipetting samplesheet for smMIP multiplexing""" + process = Process(lims, id=process_id) + input_artifacts = [] + + # Find all Dx Tapestation 2200/4200 QC process types + qc_process_types = [] + for process_type in lims.get_process_types(): + if 'Dx Tapestation 2200 QC' in process_type.name: + qc_process_types.append(process_type.name) + elif 'Dx Tapestation 4200 QC' in process_type.name: + qc_process_types.append(process_type.name) + + # Write header + output_file.write('{sample}\t{volume}\t{plate_id}\t{well_id}\t{concentration}\t{manual}\n'.format( + sample='Sample', + volume='Volume', + plate_id='Plaat_id', + well_id='Well_id', + concentration='Concentratie', + manual='Handmatig', + )) + + for input_artifact in process.all_inputs(resolve=True): + # Find last qc process for artifact + qc_process = sorted( + lims.get_processes(type=qc_process_types, inputartifactlimsid=input_artifact.id), + key=lambda process: int(process.id.split('-')[-1]) + )[-1] + + # Find concentration measurement + for qc_artifact in qc_process.outputs_per_input(input_artifact.id): + if qc_artifact.name == input_artifact.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + + input_artifacts.append({ + 'name': input_artifact.name, + 'concentration': concentration, + 'plate_id': input_artifact.location[0].id, + 'well_id': input_artifact.location[1], + 'manual': input_artifact.samples[0].udf['Dx Handmatig'] + }) + + # Calculate avg concentration for all non manual samples + concentrations = [input_artifact['concentration'] for input_artifact in input_artifacts if not input_artifact['manual']] + avg_concentration = sum(concentrations) / len(concentrations) + + for input_artifact in input_artifacts: + if input_artifact['concentration'] < avg_concentration * 0.5: + volume = 20 + elif input_artifact['concentration'] > avg_concentration * 1.5: + volume = 2 + else: + volume = 5 + + output_file.write('{sample}\t{volume}\t{plate_id}\t{well_id}\t{concentration}\t{manual}\n'.format( + sample=input_artifact['name'], + volume=volume, + plate_id=input_artifact['plate_id'], + well_id=input_artifact['well_id'], + concentration=input_artifact['concentration'], + manual=input_artifact['manual'], + )) From 6452719ad7f44aac2d3c3a03cbb23ab2e6e8c8c3 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 7 Apr 2021 15:35:20 +0200 Subject: [PATCH 23/53] Pep line length --- clarity_epp.py | 37 +++++++++++++++++++------- clarity_epp/export/manual_pipetting.py | 21 ++++++++++----- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/clarity_epp.py b/clarity_epp.py index 3b08d92..116faf0 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -193,13 +193,18 @@ def placement_complete_step(args): subparser = parser.add_subparsers() output_parser = argparse.ArgumentParser(add_help=False) - output_parser.add_argument('-o', '--output_file', nargs='?', type=argparse.FileType('w'), default=sys.stdout, help='Output file path (default=stdout)') + output_parser.add_argument( + '-o', '--output_file', nargs='?', type=argparse.FileType('w'), default=sys.stdout, + help='Output file path (default=stdout)' + ) # export parser_export = subparser.add_parser('export', help='Export from lims.') subparser_export = parser_export.add_subparsers() - parser_export_bioanalyzer = subparser_export.add_parser('bioanalyzer', help='Create bioanalyzer samplesheets', parents=[output_parser]) + parser_export_bioanalyzer = subparser_export.add_parser( + 'bioanalyzer', help='Create bioanalyzer samplesheets', parents=[output_parser] + ) parser_export_bioanalyzer.add_argument('process_id', help='Clarity lims process id') parser_export_bioanalyzer.set_defaults(func=export_bioanalyzer) @@ -213,12 +218,16 @@ def placement_complete_step(args): parser_export_email.add_argument('process_id', help='Clarity lims process id') parser_export_email.set_defaults(func=export_email) - parser_export_hamilton = subparser_export.add_parser('hamilton', help='Create hamilton samplesheets', parents=[output_parser]) + parser_export_hamilton = subparser_export.add_parser( + 'hamilton', help='Create hamilton samplesheets', parents=[output_parser] + ) parser_export_hamilton.add_argument('type', choices=['filling_out', 'purify'], help='Samplesheet type') parser_export_hamilton.add_argument('process_id', help='Clarity lims process id') parser_export_hamilton.set_defaults(func=export_hamilton) - parser_export_illumina = subparser_export.add_parser('illumina', help='Export updated illumina samplesheet.', parents=[output_parser]) + parser_export_illumina = subparser_export.add_parser( + 'illumina', help='Export updated illumina samplesheet.', parents=[output_parser] + ) parser_export_illumina.add_argument('process_id', help='Clarity lims process id') parser_export_illumina.add_argument('artifact_id', help='Clarity lims samplesheet artifact id') parser_export_illumina.set_defaults(func=export_illumina) @@ -229,7 +238,9 @@ def placement_complete_step(args): parser_export_labels.add_argument('-d', '--description', nargs='?', help='Container name description') parser_export_labels.set_defaults(func=export_labels) - parser_export_manual_pipetting = subparser_export.add_parser('manual', help='Create manual pipetting exports', parents=[output_parser]) + parser_export_manual_pipetting = subparser_export.add_parser( + 'manual', help='Create manual pipetting exports', parents=[output_parser] + ) parser_export_manual_pipetting.add_argument( 'type', choices=[ @@ -249,10 +260,14 @@ def placement_complete_step(args): parser_export_ped.add_argument('process_id', help='Clarity lims process id') parser_export_ped.set_defaults(func=export_ped_file) - parser_export_removed_samples = subparser_export.add_parser('removed_samples', help='Export removed sampels table.', parents=[output_parser]) + parser_export_removed_samples = subparser_export.add_parser( + 'removed_samples', help='Export removed sampels table.', parents=[output_parser] + ) parser_export_removed_samples.set_defaults(func=export_removed_samples) - parser_export_tapestation = subparser_export.add_parser('tapestation', help='Create tapestation samplesheets', parents=[output_parser]) + parser_export_tapestation = subparser_export.add_parser( + 'tapestation', help='Create tapestation samplesheets', parents=[output_parser] + ) parser_export_tapestation.add_argument('process_id', help='Clarity lims process id') parser_export_tapestation.set_defaults(func=export_tapestation) @@ -260,7 +275,9 @@ def placement_complete_step(args): parser_export_tecan.add_argument('process_id', help='Clarity lims process id') parser_export_tecan.set_defaults(func=export_tecan) - parser_export_workflow = subparser_export.add_parser('workflow', help='Export workflow result file.', parents=[output_parser]) + parser_export_workflow = subparser_export.add_parser( + 'workflow', help='Export workflow result file.', parents=[output_parser] + ) parser_export_workflow.add_argument('type', choices=['all', 'lab', 'data_analysis'], help='Workflow type') parser_export_workflow.add_argument('process_id', help='Clarity lims process id') parser_export_workflow.set_defaults(func=export_workflow) @@ -323,7 +340,9 @@ def placement_complete_step(args): parser_placement_barcode.add_argument('process_id', help='Clarity lims process id') parser_placement_barcode.set_defaults(func=placement_barcode) - parser_placement_complete_step = subparser_placement.add_parser('complete_step', help='Complete step Dx Mark protocol complete.') + parser_placement_complete_step = subparser_placement.add_parser( + 'complete_step', help='Complete step Dx Mark protocol complete.' + ) parser_placement_complete_step.add_argument('process_id', help='Clarity lims process id') parser_placement_complete_step.set_defaults(func=placement_complete_step) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 20ae3c4..fb4e807 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -123,11 +123,18 @@ def samplesheet_multiplex_library_pool(lims, process_id, output_file): well_id = {} pools_not_3 = [] order = [ - 'A1', 'B1', 'C1', 'D1', 'E1', 'F1', 'G1', 'H1', 'A2', 'B2', 'C2', 'D2', 'E2', 'F2', 'G2', 'H2', 'A3', 'B3', 'C3', 'D3', 'E3', 'F3', 'G3', - 'H3', 'A4', 'B4', 'C4', 'D4', 'E4', 'F4', 'G4', 'H4', 'A5', 'B5', 'C5', 'D5', 'E5', 'F5', 'G5', 'H5', 'A6', 'B6', 'C6', 'D6', 'E6', 'F6', - 'G6', 'H6', 'A7', 'B7', 'C7', 'D7', 'E7', 'F7', 'G7', 'H7', 'A8', 'B8', 'C8', 'D8', 'E8', 'F8', 'G8', 'H8', 'A9', 'B9', 'C9', 'D9', 'E9', - 'F9', 'G9', 'H9', 'A10', 'B10', 'C10', 'D10', 'E10', 'F10', 'G10', 'H10', 'A11', 'B11', 'C11', 'D11', 'E11', 'F11', 'G11', 'H11', 'A12', - 'B12', 'C12', 'D12', 'E12', 'F12', 'G12', 'H12' + 'A1', 'B1', 'C1', 'D1', 'E1', 'F1', 'G1', 'H1', + 'A2', 'B2', 'C2', 'D2', 'E2', 'F2', 'G2', 'H2', + 'A3', 'B3', 'C3', 'D3', 'E3', 'F3', 'G3', 'H3', + 'A4', 'B4', 'C4', 'D4', 'E4', 'F4', 'G4', 'H4', + 'A5', 'B5', 'C5', 'D5', 'E5', 'F5', 'G5', 'H5', + 'A6', 'B6', 'C6', 'D6', 'E6', 'F6', 'G6', 'H6', + 'A7', 'B7', 'C7', 'D7', 'E7', 'F7', 'G7', 'H7', + 'A8', 'B8', 'C8', 'D8', 'E8', 'F8', 'G8', 'H8', + 'A9', 'B9', 'C9', 'D9', 'E9', 'F9', 'G9', 'H9', + 'A10', 'B10', 'C10', 'D10', 'E10', 'F10', 'G10', 'H10', + 'A11', 'B11', 'C11', 'D11', 'E11', 'F11', 'G11', 'H11', + 'A12', 'B12', 'C12', 'D12', 'E12', 'F12', 'G12', 'H12' ] order = dict(zip(order, range(len(order)))) well_order = {} @@ -328,7 +335,9 @@ def samplesheet_multiplex_sequence_pool(lims, process_id, output_file): def samplesheet_normalization(lims, process_id, output_file): """Create manual pipetting samplesheet for normalizing samples.""" - output_file.write('Sample\tConcentration (ng/ul)\tEindvolume (ul)\tVerdunningsfactor\tVolume sample (ul)\tVolume water (ul)\n') + output_file.write( + 'Sample\tConcentration (ng/ul)\tEindvolume (ul)\tVerdunningsfactor\tVolume sample (ul)\tVolume water (ul)\n' + ) process = Process(lims, id=process_id) # Find all QC process types From 572cef2ed6658b77fc757ccd14c87052bc511d99 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 16 Apr 2021 12:11:35 +0200 Subject: [PATCH 24/53] Check artifact and qc_artifact name --- clarity_epp/export/manual_pipetting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index fb4e807..250ed8f 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -360,7 +360,8 @@ def samplesheet_normalization(lims, process_id, output_file): # Find concentration measurement for qc_artifact in qc_process.outputs_per_input(input_artifact.id): - concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + if qc_artifact.name == artifact.name: + concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) final_volume = artifact.udf['Dx Eindvolume (ul)'] dilution_factor = concentration / 20 From 8319785cc5f4410a1578506f38e03b255a06fac6 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 16 Apr 2021 13:26:17 +0200 Subject: [PATCH 25/53] Fix tecan scripts for new sample names. --- clarity_epp/export/tecan.py | 2 +- clarity_epp/upload/tecan.py | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 74f6509..4ae320e 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -13,7 +13,7 @@ def samplesheet(lims, process_id, output_file): for placement, artifact in process.output_containers()[0].placements.iteritems(): placement = ''.join(placement.split(':')) - well_plate[placement] = artifact.name + well_plate[placement] = artifact.name.split('_')[0] for well in utils.sort_96_well_plate(well_plate.keys()): output_file.write('{well}\t{sample}\n'.format( diff --git a/clarity_epp/upload/tecan.py b/clarity_epp/upload/tecan.py index efe15bc..4d41340 100644 --- a/clarity_epp/upload/tecan.py +++ b/clarity_epp/upload/tecan.py @@ -65,35 +65,36 @@ def results(lims, process_id): for artifact in process.all_outputs(): if artifact.name not in ['Tecan Spark Output', 'Tecan Spark Samplesheet', 'check gemiddelde concentratie', 'Label plaat']: + artifact_name = artifact.name.split('_')[0] # Set Average Concentratie fluorescentie - sample_fluorescence = sum(sample_measurements[artifact.name]) / float(len(sample_measurements[artifact.name])) + sample_fluorescence = sum(sample_measurements[artifact_name]) / float(len(sample_measurements[artifact_name])) sample_concentration = ((sample_fluorescence - baseline_fluorescence) * regression_slope) / 2.0 artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = sample_concentration # Set artifact Concentratie fluorescentie # Get artifact index == count - if artifact.name not in artifact_count: - artifact_count[artifact.name] = 0 + if artifact_name not in artifact_count: + artifact_count[artifact_name] = 0 else: - artifact_count[artifact.name] += 1 + artifact_count[artifact_name] += 1 - artifact_fluorescence = sample_measurements[artifact.name][artifact_count[artifact.name]] + artifact_fluorescence = sample_measurements[artifact_name][artifact_count[artifact_name]] artifact_concentration = ((artifact_fluorescence - baseline_fluorescence) * regression_slope) / 2.0 artifact.udf['Dx Conc. goedgekeurde meting (ng/ul)'] = artifact_concentration # Set QC flags - if artifact.name.startswith('Dx Tecan std'): + if artifact_name.startswith('Dx Tecan std'): artifact.qc_flag = 'PASSED' - std_number = int(artifact.name.split(' ')[3]) + std_number = int(artifact_name.split(' ')[3]) artifact.udf['Dx Conc. goedgekeurde meting (ng/ul)'] = ng_values[std_number - 1] artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = ng_values[std_number - 1] else: # Calculate measurement deviation from average. if concentration_range[0] <= sample_concentration <= concentration_range[1]: - if len(sample_measurements[artifact.name]) == 1: + if len(sample_measurements[artifact_name]) == 1: artifact.qc_flag = 'PASSED' - elif len(sample_measurements[artifact.name]) == 2: - artifact_fluorescence_difference = abs(sample_measurements[artifact.name][0] - sample_measurements[artifact.name][1]) + elif len(sample_measurements[artifact_name]) == 2: + artifact_fluorescence_difference = abs(sample_measurements[artifact_name][0] - sample_measurements[artifact_name][1]) artifact_fluorescence_deviation = artifact_fluorescence_difference / sample_fluorescence if artifact_fluorescence_deviation <= 0.1: artifact.qc_flag = 'PASSED' From 2f107b80b4485104e00e780aec6c5fc56064d2d4 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 29 Apr 2021 14:35:26 +0200 Subject: [PATCH 26/53] Update manual_pipetting.py Add 10% to Master Mix sum. --- clarity_epp/export/manual_pipetting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 250ed8f..50d6de9 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -459,7 +459,7 @@ def sammplesheet_pcr_exonuclease(lims, process_id, output_file): # Calculate total data.append([ 'TOTAL (incl. 10% overmaat)', - sum([item[1] for item in data]), + sum([item[1] for item in data]) * 1.1, sum([item[2] for item in data]), ]) From 26b7748531c7aa51eab6113b0fd96a24e4345864 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 26 May 2021 16:05:20 +0200 Subject: [PATCH 27/53] python3 --- clarity_epp/export/__init__.py | 26 +++++++++++++------------- clarity_epp/export/caliper.py | 2 +- clarity_epp/export/hamilton.py | 2 +- clarity_epp/export/illumina.py | 2 +- clarity_epp/export/sample.py | 2 +- clarity_epp/export/tapestation.py | 2 +- clarity_epp/export/tecan.py | 2 +- clarity_epp/placement/__init__.py | 10 +++++----- clarity_epp/qc/__init__.py | 6 +++--- clarity_epp/upload/__init__.py | 8 ++++---- clarity_epp/upload/samples.py | 2 +- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/clarity_epp/export/__init__.py b/clarity_epp/export/__init__.py index ab8818f..35aa207 100755 --- a/clarity_epp/export/__init__.py +++ b/clarity_epp/export/__init__.py @@ -1,15 +1,15 @@ """Export functions.""" -import bioanalyzer -import caliper -import email -import hamilton -import illumina -import labels -import manual_pipetting -import merge -import ped -import sample -import tapestation -import tecan -import workflow +import clarity_epp.export.bioanalyzer +import clarity_epp.export.caliper +import clarity_epp.export.email +import clarity_epp.export.hamilton +import clarity_epp.export.illumina +import clarity_epp.export.labels +import clarity_epp.export.manual_pipetting +import clarity_epp.export.merge +import clarity_epp.export.ped +import clarity_epp.export.sample +import clarity_epp.export.tapestation +import clarity_epp.export.tecan +import clarity_epp.export.workflow diff --git a/clarity_epp/export/caliper.py b/clarity_epp/export/caliper.py index 3ef4a5f..746473b 100755 --- a/clarity_epp/export/caliper.py +++ b/clarity_epp/export/caliper.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import utils +import clarity_epp.export.utils def samplesheet_normalise(lims, process_id, output_file): diff --git a/clarity_epp/export/hamilton.py b/clarity_epp/export/hamilton.py index c17f92b..c27730f 100755 --- a/clarity_epp/export/hamilton.py +++ b/clarity_epp/export/hamilton.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import utils +import clarity_epp.export.utils def samplesheet_filling_out(lims, process_id, output_file): diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index 237278d..e147a04 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -5,7 +5,7 @@ from genologics.entities import Process, Artifact from .. import get_sequence_name -import utils +import clarity_epp.export.utils import config diff --git a/clarity_epp/export/sample.py b/clarity_epp/export/sample.py index 3ca39c4..bf91f92 100644 --- a/clarity_epp/export/sample.py +++ b/clarity_epp/export/sample.py @@ -1,7 +1,7 @@ """Sample export functions.""" import datetime -import utils +import clarity_epp.export.utils def removed_samples(lims, output_file): diff --git a/clarity_epp/export/tapestation.py b/clarity_epp/export/tapestation.py index cd02131..37fa0c8 100644 --- a/clarity_epp/export/tapestation.py +++ b/clarity_epp/export/tapestation.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import utils +import clarity_epp.export.utils def samplesheet(lims, process_id, output_file): diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 4ae320e..05d55da 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import utils +import clarity_epp.export.utils def samplesheet(lims, process_id, output_file): diff --git a/clarity_epp/placement/__init__.py b/clarity_epp/placement/__init__.py index bdf3b13..6c6bff7 100644 --- a/clarity_epp/placement/__init__.py +++ b/clarity_epp/placement/__init__.py @@ -1,7 +1,7 @@ """Container placement functions.""" -import artifact -import barcode -import plate -import pool -import step +import clarity_epp.placement.artifact +import clarity_epp.placement.barcode +import clarity_epp.placement.plate +import clarity_epp.placement.pool +import clarity_epp.placement.step diff --git a/clarity_epp/qc/__init__.py b/clarity_epp/qc/__init__.py index fcd6e88..d585d0a 100644 --- a/clarity_epp/qc/__init__.py +++ b/clarity_epp/qc/__init__.py @@ -1,5 +1,5 @@ """Automatic QC functions.""" -import fragment_length -import illumina -import qubit +import clarity_epp.qc.fragment_length +import clarity_epp.qc.illumina +import clarity_epp.qc.qubit diff --git a/clarity_epp/upload/__init__.py b/clarity_epp/upload/__init__.py index 4f08d28..54d1801 100644 --- a/clarity_epp/upload/__init__.py +++ b/clarity_epp/upload/__init__.py @@ -1,6 +1,6 @@ """Upload functions.""" -import samples -import tecan -import tapestation -import bioanalyzer +import clarity_epp.upload.samples +import clarity_epp.upload.tecan +import clarity_epp.upload.tapestation +import clarity_epp.upload.bioanalyzer diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 79bc452..5463ab4 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -5,7 +5,7 @@ from genologics.entities import Sample, Project, Containertype, Container from .. import send_email -import utils +import clarity_epp.upload.utils def from_helix(lims, email_settings, input_file): From 48cdc4299e3000349616f6609235a4de2f9c1dfa Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 27 May 2021 13:42:40 +0200 Subject: [PATCH 28/53] Add samplesheet_mip_pool_dilution --- clarity_epp.py | 4 +++- clarity_epp/export/manual_pipetting.py | 32 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/clarity_epp.py b/clarity_epp.py index 116faf0..ca08fe2 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -79,6 +79,8 @@ def export_manual_pipetting(args): clarity_epp.export.manual_pipetting.sammplesheet_pcr_exonuclease(lims, args.process_id, args.output_file) elif args.type == 'mip_multiplex_pool': clarity_epp.export.manual_pipetting.samplesheet_mip_multiplex_pool(lims, args.process_id, args.output_file) + elif args.type == 'mip_dilute_pool': + clarity_epp.export.manual_pipetting.samplesheet_mip_pool_dilution(lims, args.process_id, args.output_file) def export_ped_file(args): @@ -245,7 +247,7 @@ def placement_complete_step(args): 'type', choices=[ 'purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization', - 'capture', 'exonuclease', 'pcr_exonuclease', 'mip_multiplex_pool' + 'capture', 'exonuclease', 'pcr_exonuclease', 'mip_multiplex_pool', 'mip_dilute_pool' ], help='Samplesheet type' ) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 50d6de9..e77464f 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -532,3 +532,35 @@ def samplesheet_mip_multiplex_pool(lims, process_id, output_file): concentration=input_artifact['concentration'], manual=input_artifact['manual'], )) + + +def samplesheet_mip_pool_dilution(lims, process_id, output_file): + """Create manual pipetting samplesheet for smMIP pool dilution""" + process = Process(lims, id=process_id) + + # Write header + output_file.write('{sample}\t{dna}\t{ul_sample}\t{ul_EB}\t{concentration}\t{fragment_length}\n'.format( + sample='Sample', + dna='nM DNA', + ul_sample='ul Sample', + ul_EB='ul EB buffer', + concentration='Concentratie (ng/ul)', + fragment_length='Fragment lengte (bp)', + )) + + for input_artifact in process.all_inputs(resolve=True): + concentration = float(input_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + fragment_length = float(input_artifact.udf['Dx Fragmentlengte (bp)']) + + dna = (concentration*(10**3/1)*(1/649)*(1/fragment_length))*1000 + ul_sample = 2/dna*10 + ul_EB = 10-ul_sample + + output_file.write('{sample}\t{dna:.2f}\t{ul_sample:.2f}\t{ul_EB:.2f}\t{concentration:.2f}\t{fragment_length}\n'.format( + sample=input_artifact.name, + dna=dna, + ul_sample=ul_sample, + ul_EB=ul_EB, + concentration=concentration, + fragment_length=int(fragment_length), + )) \ No newline at end of file From baab35befaf20ee46484b768461622cd73661462 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 27 May 2021 14:03:48 +0200 Subject: [PATCH 29/53] Fix floats --- clarity_epp/export/manual_pipetting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index e77464f..8663fde 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -552,7 +552,7 @@ def samplesheet_mip_pool_dilution(lims, process_id, output_file): concentration = float(input_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) fragment_length = float(input_artifact.udf['Dx Fragmentlengte (bp)']) - dna = (concentration*(10**3/1)*(1/649)*(1/fragment_length))*1000 + dna = (concentration*(10.0**3.0/1.0)*(1.0/649.0)*(1.0/fragment_length))*1000.0 ul_sample = 2/dna*10 ul_EB = 10-ul_sample From e5e3067c5e2775145a117eb9de426e607cadbed7 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 3 Jun 2021 13:50:32 +0200 Subject: [PATCH 30/53] Add 20 and 40 ul. --- clarity_epp/export/manual_pipetting.py | 34 ++++++++++++++------------ 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 8663fde..17d2921 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -539,28 +539,30 @@ def samplesheet_mip_pool_dilution(lims, process_id, output_file): process = Process(lims, id=process_id) # Write header - output_file.write('{sample}\t{dna}\t{ul_sample}\t{ul_EB}\t{concentration}\t{fragment_length}\n'.format( + output_file.write('{sample}\t{ul_sample_10}\t{ul_EB_10}\t{ul_sample_20}\t{ul_EB_20}\t{ul_sample_40}\t{ul_EB_40}\t\n'.format( sample='Sample', - dna='nM DNA', - ul_sample='ul Sample', - ul_EB='ul EB buffer', - concentration='Concentratie (ng/ul)', - fragment_length='Fragment lengte (bp)', + ul_sample_10='ul Sample (10 ul)', + ul_EB_10='ul EB buffer (10 ul)', + ul_sample_20='ul Sample (20 ul)', + ul_EB_20='ul EB buffer (20 ul)', + ul_sample_40='ul Sample (40 ul)', + ul_EB_40='ul EB buffer (40 ul)', )) for input_artifact in process.all_inputs(resolve=True): concentration = float(input_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) fragment_length = float(input_artifact.udf['Dx Fragmentlengte (bp)']) - dna = (concentration*(10.0**3.0/1.0)*(1.0/649.0)*(1.0/fragment_length))*1000.0 - ul_sample = 2/dna*10 - ul_EB = 10-ul_sample + dna = (concentration * (10.0**3.0 / 1.0) * (1.0 / 649.0) * (1.0 / fragment_length) ) * 1000.0 + ul_sample = 2 / dna * 10 + ul_EB = 10 - ul_sample - output_file.write('{sample}\t{dna:.2f}\t{ul_sample:.2f}\t{ul_EB:.2f}\t{concentration:.2f}\t{fragment_length}\n'.format( + output_file.write('{sample}\t{ul_sample_10:.2f}\t{ul_EB_10:.2f}\t{ul_sample_20:.2f}\t{ul_EB_20:.2f}\t{ul_sample_40:.2f}\t{ul_EB_40:.2f}\t\n'.format( sample=input_artifact.name, - dna=dna, - ul_sample=ul_sample, - ul_EB=ul_EB, - concentration=concentration, - fragment_length=int(fragment_length), - )) \ No newline at end of file + ul_sample_10=ul_sample, + ul_EB_10=ul_EB, + ul_sample_20=ul_sample * 2, + ul_EB_20=ul_EB * 2, + ul_sample_40=ul_sample * 4, + ul_EB_40=ul_EB * 4, + )) From 443717e8f5fb5e025d3dc40de831a08d2a2ccf7f Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 7 Jun 2021 10:28:41 +0200 Subject: [PATCH 31/53] Update imports --- clarity_epp/export/caliper.py | 2 +- clarity_epp/export/hamilton.py | 2 +- clarity_epp/export/illumina.py | 2 +- clarity_epp/export/sample.py | 2 +- clarity_epp/export/tapestation.py | 2 +- clarity_epp/export/tecan.py | 2 +- clarity_epp/placement/pool.py | 1 - clarity_epp/upload/samples.py | 2 +- clarity_epp/upload/utils.py | 3 --- 9 files changed, 7 insertions(+), 11 deletions(-) diff --git a/clarity_epp/export/caliper.py b/clarity_epp/export/caliper.py index 746473b..1f547c1 100755 --- a/clarity_epp/export/caliper.py +++ b/clarity_epp/export/caliper.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import clarity_epp.export.utils +import clarity_epp.export.utils as utils def samplesheet_normalise(lims, process_id, output_file): diff --git a/clarity_epp/export/hamilton.py b/clarity_epp/export/hamilton.py index c27730f..547183e 100755 --- a/clarity_epp/export/hamilton.py +++ b/clarity_epp/export/hamilton.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import clarity_epp.export.utils +import clarity_epp.export.utils as utils def samplesheet_filling_out(lims, process_id, output_file): diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index e147a04..b90f133 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -5,7 +5,7 @@ from genologics.entities import Process, Artifact from .. import get_sequence_name -import clarity_epp.export.utils +import clarity_epp.export.utils as utils import config diff --git a/clarity_epp/export/sample.py b/clarity_epp/export/sample.py index bf91f92..2b87b17 100644 --- a/clarity_epp/export/sample.py +++ b/clarity_epp/export/sample.py @@ -1,7 +1,7 @@ """Sample export functions.""" import datetime -import clarity_epp.export.utils +import clarity_epp.export.utils as utils def removed_samples(lims, output_file): diff --git a/clarity_epp/export/tapestation.py b/clarity_epp/export/tapestation.py index 37fa0c8..81384d8 100644 --- a/clarity_epp/export/tapestation.py +++ b/clarity_epp/export/tapestation.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import clarity_epp.export.utils +import clarity_epp.export.utils as utils def samplesheet(lims, process_id, output_file): diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 05d55da..2c88f06 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import clarity_epp.export.utils +import clarity_epp.export.utils as utils def samplesheet(lims, process_id, output_file): diff --git a/clarity_epp/placement/pool.py b/clarity_epp/placement/pool.py index f07ffe9..56098b9 100644 --- a/clarity_epp/placement/pool.py +++ b/clarity_epp/placement/pool.py @@ -2,7 +2,6 @@ from genologics.entities import Artifact, Process, Workflow -from .. import get_sequence_name import config diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index 5463ab4..b88807f 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -5,7 +5,7 @@ from genologics.entities import Sample, Project, Containertype, Container from .. import send_email -import clarity_epp.upload.utils +import clarity_epp.upload.utils as utils def from_helix(lims, email_settings, input_file): diff --git a/clarity_epp/upload/utils.py b/clarity_epp/upload/utils.py index d5012f2..6b2078b 100644 --- a/clarity_epp/upload/utils.py +++ b/clarity_epp/upload/utils.py @@ -1,7 +1,4 @@ """Utility functions.""" -from datetime import datetime -import re - from genologics.entities import Workflow import config From 8088c2bb5c6bfb3b86e20d8a911cc210ade58fca Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 7 Jun 2021 11:55:19 +0200 Subject: [PATCH 32/53] Add MIP projects. --- clarity_epp/export/illumina.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index b90f133..7f599fb 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -55,6 +55,15 @@ def get_project(projects, urgent=False): project_type = 'CREv2' families[family]['project_type'] = project_type families[family]['split_project_type'] = True + elif 'SNP fingerprint MIP' in sample.udf['Dx Protocolomschrijving'] and not families[family]['NICU']: + project_type = 'Fingerprint' + families[family]['project_type'] = project_type + families[family]['split_project_type'] = False + elif 'PID09.V7_smMIP' in sample.udf['Dx Protocolomschrijving'] and not families[family]['NICU']: + project_type = 'ERARE' + families[family]['project_type'] = project_type + families[family]['split_project_type'] = False + # Set urgent / merge status if 'Dx Spoed' in list(sample.udf) and sample.udf['Dx Spoed']: families[family]['urgent'] = True From c96585a579b08d5c03de0f6cdbab2d03b826ac0d Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 7 Jun 2021 12:16:37 +0200 Subject: [PATCH 33/53] Fix imports --- clarity_epp/export/caliper.py | 4 ++-- clarity_epp/export/hamilton.py | 6 +++--- clarity_epp/export/illumina.py | 4 ++-- clarity_epp/export/sample.py | 4 ++-- clarity_epp/export/tapestation.py | 4 ++-- clarity_epp/export/tecan.py | 4 ++-- clarity_epp/upload/samples.py | 12 ++++++------ 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/clarity_epp/export/caliper.py b/clarity_epp/export/caliper.py index 1f547c1..7d5c362 100755 --- a/clarity_epp/export/caliper.py +++ b/clarity_epp/export/caliper.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import clarity_epp.export.utils as utils +import clarity_epp.export.utils def samplesheet_normalise(lims, process_id, output_file): @@ -145,7 +145,7 @@ def samplesheet_normalise(lims, process_id, output_file): volume_DNA[placement] = int(round(float(output_ng)/conc[placement])) volume_H2O[placement] = output_ul-int(round(float(output_ng)/conc[placement])) - for well in utils.sort_96_well_plate(monsternummer.keys()): + for well in clarity_epp.export.utils.sort_96_well_plate(monsternummer.keys()): output_file.write( '{monsternummer}\t{plate_id_input}\t{position}\t{plate_id_output}\t{volume_DNA}\t{volume_H2O}\n'.format( monsternummer=monsternummer[well], diff --git a/clarity_epp/export/hamilton.py b/clarity_epp/export/hamilton.py index 547183e..d4c589b 100755 --- a/clarity_epp/export/hamilton.py +++ b/clarity_epp/export/hamilton.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import clarity_epp.export.utils as utils +import clarity_epp.export.utils def samplesheet_filling_out(lims, process_id, output_file): @@ -15,7 +15,7 @@ def samplesheet_filling_out(lims, process_id, output_file): placement = ''.join(placement.split(':')) well_plate[placement] = artifact.samples[0].udf['Dx Fractienummer'] - for well in utils.sort_96_well_plate(well_plate.keys()): + for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): output_file.write('{source_tube}\t{well}\n'.format( source_tube=well_plate[well], well=well @@ -33,7 +33,7 @@ def samplesheet_purify(lims, process_id, output_file): placement = ''.join(placement.split(':')) well_plate[placement] = artifact.samples[0].udf['Dx Fractienummer'] - for well in utils.sort_96_well_plate(well_plate.keys()): + for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): output_file.write('{sample}\t{sample_rack_barcode}\t{sample_rack_position}\t{sample_start_volume}\n'.format( sample=well_plate[well], sample_rack_barcode=parent_process_barcode, diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index 7f599fb..ac29205 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -5,7 +5,7 @@ from genologics.entities import Process, Artifact from .. import get_sequence_name -import clarity_epp.export.utils as utils +import clarity_epp.export.utils import config @@ -177,7 +177,7 @@ def get_project(projects, urgent=False): # Reverse complement index for NextSeq runs if nextseq_run: - data[index_index] = utils.reverse_complement(data[index_index]) + data[index_index] = clarity_epp.export.utils.reverse_complement(data[index_index]) output_file.write('{line}\n'.format(line=','.join(data))) else: # Leave other lines untouched. diff --git a/clarity_epp/export/sample.py b/clarity_epp/export/sample.py index 2b87b17..64db25a 100644 --- a/clarity_epp/export/sample.py +++ b/clarity_epp/export/sample.py @@ -1,7 +1,7 @@ """Sample export functions.""" import datetime -import clarity_epp.export.utils as utils +import clarity_epp.export.utils def removed_samples(lims, output_file): @@ -21,7 +21,7 @@ def removed_samples(lims, output_file): removed_process_id = None removed_date = None - for artifact in sorted(lims.get_artifacts(type="Analyte", sample_name=sample.name, resolve=True), key=utils.sort_artifact_list): + for artifact in sorted(lims.get_artifacts(type="Analyte", sample_name=sample.name, resolve=True), key=clarity_epp.export.utils.sort_artifact_list): for stage, status, name in artifact.workflow_stages_and_statuses: if status == 'REMOVED': # Store removed from workflow sample_removed = True diff --git a/clarity_epp/export/tapestation.py b/clarity_epp/export/tapestation.py index 81384d8..b7fcc32 100644 --- a/clarity_epp/export/tapestation.py +++ b/clarity_epp/export/tapestation.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import clarity_epp.export.utils as utils +import clarity_epp.export.utils def samplesheet(lims, process_id, output_file): @@ -14,7 +14,7 @@ def samplesheet(lims, process_id, output_file): placement = ''.join(placement.split(':')) well_plate[placement] = artifact.name.split('_')[0] - for well in utils.sort_96_well_plate(well_plate.keys()): + for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): output_file.write('{sample}\n'.format( sample=well_plate[well] )) diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index 2c88f06..b9440fc 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -2,7 +2,7 @@ from genologics.entities import Process -import clarity_epp.export.utils as utils +import clarity_epp.export.utils def samplesheet(lims, process_id, output_file): @@ -15,7 +15,7 @@ def samplesheet(lims, process_id, output_file): placement = ''.join(placement.split(':')) well_plate[placement] = artifact.name.split('_')[0] - for well in utils.sort_96_well_plate(well_plate.keys()): + for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): output_file.write('{well}\t{sample}\n'.format( well=well, sample=well_plate[well] diff --git a/clarity_epp/upload/samples.py b/clarity_epp/upload/samples.py index b88807f..1b722f1 100644 --- a/clarity_epp/upload/samples.py +++ b/clarity_epp/upload/samples.py @@ -5,7 +5,7 @@ from genologics.entities import Sample, Project, Containertype, Container from .. import send_email -import clarity_epp.upload.utils as utils +import clarity_epp.upload.utils def from_helix(lims, email_settings, input_file): @@ -86,15 +86,15 @@ def from_helix(lims, email_settings, input_file): for udf in udf_column: # Transform specific udf if udf in ['Dx Overleden', 'Dx Spoed', 'Dx NICU Spoed']: - udf_data[udf] = utils.char_to_bool(data[udf_column[udf]['index']]) + udf_data[udf] = clarity_epp.upload.utils.char_to_bool(data[udf_column[udf]['index']]) elif udf in ['Dx Geslacht', 'Dx Foetus geslacht']: - udf_data[udf] = utils.transform_sex(data[udf_column[udf]['index']]) + udf_data[udf] = clarity_epp.upload.utils.transform_sex(data[udf_column[udf]['index']]) elif udf == 'Dx Foetus': udf_data[udf] = bool(data[udf_column[udf]['index']].strip()) elif udf == 'Dx Concentratie (ng/ul)': udf_data[udf] = data[udf_column[udf]['index']].replace(',', '.') elif udf in ['Dx Monsternummer', 'Dx Fractienummer']: - udf_data[udf] = utils.transform_sample_name(data[udf_column[udf]['index']]) + udf_data[udf] = clarity_epp.upload.utils.transform_sample_name(data[udf_column[udf]['index']]) else: udf_data[udf] = data[udf_column[udf]['index']] @@ -160,7 +160,7 @@ def from_helix(lims, email_settings, input_file): sample.udf[udf] = udf_data[udf] # Add to new workflow - workflow = utils.stoftestcode_to_workflow(lims, udf_data['Dx Stoftest code']) + workflow = clarity_epp.upload.utils.stoftestcode_to_workflow(lims, udf_data['Dx Stoftest code']) if workflow: sample.put() lims.route_artifacts([sample.artifact], workflow_uri=workflow.uri) @@ -176,7 +176,7 @@ def from_helix(lims, email_settings, input_file): udf_data['Dx Import warning'] = ';'.join(['Onderzoek reeds uitgevoerd.', udf_data['Dx Import warning']]) # Add sample to workflow - workflow = utils.stoftestcode_to_workflow(lims, udf_data['Dx Stoftest code']) + workflow = clarity_epp.upload.utils.stoftestcode_to_workflow(lims, udf_data['Dx Stoftest code']) if workflow: container = Container.create(lims, type=container_type, name=udf_data['Dx Fractienummer']) sample = Sample.create(lims, container=container, position='1:1', project=project, name=sample_name, udf=udf_data) From 58eed398fadcc19527070414dc48e8957fefe54d Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 10 Jun 2021 12:48:52 +0200 Subject: [PATCH 34/53] Fix for nicu fingerprint samples --- clarity_epp/export/illumina.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index ac29205..7ec91a2 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -46,7 +46,15 @@ def get_project(projects, urgent=False): break else: # Dx clinic sample - if sample.udf['Dx NICU Spoed']: + if 'SNP fingerprint MIP' in sample.udf['Dx Protocolomschrijving'] and not families[family]['NICU']: + project_type = 'Fingerprint' + families[family]['project_type'] = project_type + families[family]['split_project_type'] = False + elif 'PID09.V7_smMIP' in sample.udf['Dx Protocolomschrijving'] and not families[family]['NICU']: + project_type = 'ERARE' + families[family]['project_type'] = project_type + families[family]['split_project_type'] = False + elif sample.udf['Dx NICU Spoed']: families[family]['NICU'] = True project_type = 'NICU_{0}'.format(sample.udf['Dx Familienummer']) families[family]['project_type'] = project_type @@ -55,14 +63,6 @@ def get_project(projects, urgent=False): project_type = 'CREv2' families[family]['project_type'] = project_type families[family]['split_project_type'] = True - elif 'SNP fingerprint MIP' in sample.udf['Dx Protocolomschrijving'] and not families[family]['NICU']: - project_type = 'Fingerprint' - families[family]['project_type'] = project_type - families[family]['split_project_type'] = False - elif 'PID09.V7_smMIP' in sample.udf['Dx Protocolomschrijving'] and not families[family]['NICU']: - project_type = 'ERARE' - families[family]['project_type'] = project_type - families[family]['split_project_type'] = False # Set urgent / merge status if 'Dx Spoed' in list(sample.udf) and sample.udf['Dx Spoed']: From 10c8774bfd5e825f141a0229c25b74991d28e8c4 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 10 Jun 2021 14:49:50 +0200 Subject: [PATCH 35/53] Add novaseq samplesheet templates --- ...L2FASTQ_Reverse_Complement_Samplesheet.csv | 63 +++++++++++++++++++ templates/NovaSeq_BCL2FASTQ_Samplesheet.csv | 56 +++++++++++++++++ templates/README | 2 + 3 files changed, 121 insertions(+) create mode 100644 templates/NovaSeq_BCL2FASTQ_Reverse_Complement_Samplesheet.csv create mode 100644 templates/NovaSeq_BCL2FASTQ_Samplesheet.csv create mode 100644 templates/README diff --git a/templates/NovaSeq_BCL2FASTQ_Reverse_Complement_Samplesheet.csv b/templates/NovaSeq_BCL2FASTQ_Reverse_Complement_Samplesheet.csv new file mode 100644 index 0000000..23ad34e --- /dev/null +++ b/templates/NovaSeq_BCL2FASTQ_Reverse_Complement_Samplesheet.csv @@ -0,0 +1,63 @@ +SORT.BY.${INPUT.CONTAINER.ROW}${INPUT.CONTAINER.COLUMN} +OUTPUT.FILE.NAME,${OUTPUT.CONTAINER.NAME}.csv +PROCESS.POOLED.ARTIFACTS +CONTROL.SAMPLE.DEFAULT.PROJECT.NAME,Controls +HIDE, ${PROCESS.UDF.Settings Header}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 1 Length}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 2 Length}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 1 Start From Cycle}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 2 Start From Cycle}, IF, NODATA + +[Header] +Investigator Name,${PROCESS.TECHNICIAN} +Experiment Name,${PROCESS.UDF.Experiment Name} +Date,${DATE} +Workflow,${PROCESS.UDF.Workflow} +[Reads] +${PROCESS.UDF.Read 1 Cycles} +${PROCESS.UDF.Read 2 Cycles} +${PROCESS.UDF.Settings Header} +Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} +Read2UMILength,${PROCESS.UDF.UMI - Read 2 Length} +Read1StartFromCycle,${PROCESS.UDF.UMI - Read 1 Start From Cycle} +Read2StartFromCycle,${PROCESS.UDF.UMI - Read 2 Start From Cycle} +[Data] + +
+Sample_ID,Sample_Name,Sample_Plate,Sample_Well,index,index2,Sample_Project,Description +
+ +${INPUT.LIMSID},${SAMPLE.NAME##NoSpecialCharacters},${INPUT.CONTAINER.NAME},${INPUT.CONTAINER.PLACEMENT},${INPUT.REAGENT.SEQUENCE##Single},${INPUT.REAGENT.SEQUENCE##Index2RC},${SAMPLE.PROJECT.NAME##NoSpecialCharacters}, + + +${INPUT.REAGENT.SEQUENCE##Single} +def index = token.indexOf("-") +if (index > 0) { + return token.substring(0,index) +} else { + return token +} + +${INPUT.REAGENT.SEQUENCE##Index2RC} +def index = token.indexOf("-") +if (index > 0) { + return token.split('-')[1].toUpperCase().reverse().collect { base -> + switch (base) { + case 'A': return 'T' + case 'C': return 'G' + case 'G': return 'C' + case 'T': return 'A' + default : throw new Exception("Input sequence '${sequence}' contains an invalid base '${base}'.") + } + }.join('') +} else { + return '' +} + +${SAMPLE.NAME##NoSpecialCharacters} +return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") + +${SAMPLE.PROJECT.NAME##NoSpecialCharacters} +return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") + + \ No newline at end of file diff --git a/templates/NovaSeq_BCL2FASTQ_Samplesheet.csv b/templates/NovaSeq_BCL2FASTQ_Samplesheet.csv new file mode 100644 index 0000000..ca4f999 --- /dev/null +++ b/templates/NovaSeq_BCL2FASTQ_Samplesheet.csv @@ -0,0 +1,56 @@ +SORT.BY.${INPUT.CONTAINER.ROW}${INPUT.CONTAINER.COLUMN} +OUTPUT.FILE.NAME,${OUTPUT.CONTAINER.NAME}.csv +PROCESS.POOLED.ARTIFACTS +CONTROL.SAMPLE.DEFAULT.PROJECT.NAME,Controls +HIDE, ${INPUT.REAGENT.SEQUENCE##Dual}, IF, NODATA +HIDE, ${PROCESS.UDF.Settings Header}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 1 Length}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 2 Length}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 1 Start From Cycle}, IF, NODATA +HIDE, ${PROCESS.UDF.UMI - Read 2 Start From Cycle}, IF, NODATA + +[Header] +Investigator Name,${PROCESS.TECHNICIAN} +Experiment Name,${PROCESS.UDF.Experiment Name} +Date,${DATE} +Workflow,${PROCESS.UDF.Workflow} +[Reads] +${PROCESS.UDF.Read 1 Cycles} +${PROCESS.UDF.Read 2 Cycles} +${PROCESS.UDF.Settings Header} +Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} +Read2UMILength,${PROCESS.UDF.UMI - Read 2 Length} +Read1StartFromCycle,${PROCESS.UDF.UMI - Read 1 Start From Cycle} +Read2StartFromCycle,${PROCESS.UDF.UMI - Read 2 Start From Cycle} +[Data] + +
+Sample_ID,Sample_Name,Sample_Plate,Sample_Well,index,index2,Sample_Project,Description +
+ +${INPUT.LIMSID},${SAMPLE.NAME##NoSpecialCharacters},${INPUT.CONTAINER.NAME},${INPUT.CONTAINER.PLACEMENT},${INPUT.REAGENT.SEQUENCE##Single},${INPUT.REAGENT.SEQUENCE##Dual},${SAMPLE.PROJECT.NAME##NoSpecialCharacters}, + + +${INPUT.REAGENT.SEQUENCE##Single} +def index = token.indexOf("-") +if (index > 0) { + return token.substring(0,index) +} else { + return token +} + +${INPUT.REAGENT.SEQUENCE##Dual} +def index = token.indexOf("-") +if (index >= 0) { + return token.substring(index + 1) +} else { + return null +} + +${SAMPLE.NAME##NoSpecialCharacters} +return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") + +${SAMPLE.PROJECT.NAME##NoSpecialCharacters} +return token.replaceAll(\"[^a-zA-Z0-9_]\", \"_\") + + \ No newline at end of file diff --git a/templates/README b/templates/README new file mode 100644 index 0000000..b4ae9ae --- /dev/null +++ b/templates/README @@ -0,0 +1,2 @@ +This folder contains templates to create Illumina samplesheets using clarity lims. +Original location on limsserver: `/opt/gls/clarity/extensions/conf/driverfiletemplates`. \ No newline at end of file From bbc76b164364b7f67612645ef695626fb67dfc48 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 10 Jun 2021 14:57:04 +0200 Subject: [PATCH 36/53] Update samplesheet template to match Dx samplesheet --- .../NovaSeq_BCL2FASTQ_Reverse_Complement_Samplesheet.csv | 6 +++--- templates/NovaSeq_BCL2FASTQ_Samplesheet.csv | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/NovaSeq_BCL2FASTQ_Reverse_Complement_Samplesheet.csv b/templates/NovaSeq_BCL2FASTQ_Reverse_Complement_Samplesheet.csv index 23ad34e..4064a78 100644 --- a/templates/NovaSeq_BCL2FASTQ_Reverse_Complement_Samplesheet.csv +++ b/templates/NovaSeq_BCL2FASTQ_Reverse_Complement_Samplesheet.csv @@ -2,7 +2,6 @@ SORT.BY.${INPUT.CONTAINER.ROW}${INPUT.CONTAINER.COLUMN} OUTPUT.FILE.NAME,${OUTPUT.CONTAINER.NAME}.csv PROCESS.POOLED.ARTIFACTS CONTROL.SAMPLE.DEFAULT.PROJECT.NAME,Controls -HIDE, ${PROCESS.UDF.Settings Header}, IF, NODATA HIDE, ${PROCESS.UDF.UMI - Read 1 Length}, IF, NODATA HIDE, ${PROCESS.UDF.UMI - Read 2 Length}, IF, NODATA HIDE, ${PROCESS.UDF.UMI - Read 1 Start From Cycle}, IF, NODATA @@ -12,11 +11,12 @@ HIDE, ${PROCESS.UDF.UMI - Read 2 Start From Cycle}, IF, NODATA Investigator Name,${PROCESS.TECHNICIAN} Experiment Name,${PROCESS.UDF.Experiment Name} Date,${DATE} -Workflow,${PROCESS.UDF.Workflow} [Reads] ${PROCESS.UDF.Read 1 Cycles} ${PROCESS.UDF.Read 2 Cycles} -${PROCESS.UDF.Settings Header} +[Settings] +Adapter,${PROCESS.UDF.Adapter} +AdapterRead2,${PROCESS.UDF.Adapter Read 2} Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} Read2UMILength,${PROCESS.UDF.UMI - Read 2 Length} Read1StartFromCycle,${PROCESS.UDF.UMI - Read 1 Start From Cycle} diff --git a/templates/NovaSeq_BCL2FASTQ_Samplesheet.csv b/templates/NovaSeq_BCL2FASTQ_Samplesheet.csv index ca4f999..d10a5f1 100644 --- a/templates/NovaSeq_BCL2FASTQ_Samplesheet.csv +++ b/templates/NovaSeq_BCL2FASTQ_Samplesheet.csv @@ -3,7 +3,6 @@ OUTPUT.FILE.NAME,${OUTPUT.CONTAINER.NAME}.csv PROCESS.POOLED.ARTIFACTS CONTROL.SAMPLE.DEFAULT.PROJECT.NAME,Controls HIDE, ${INPUT.REAGENT.SEQUENCE##Dual}, IF, NODATA -HIDE, ${PROCESS.UDF.Settings Header}, IF, NODATA HIDE, ${PROCESS.UDF.UMI - Read 1 Length}, IF, NODATA HIDE, ${PROCESS.UDF.UMI - Read 2 Length}, IF, NODATA HIDE, ${PROCESS.UDF.UMI - Read 1 Start From Cycle}, IF, NODATA @@ -13,11 +12,12 @@ HIDE, ${PROCESS.UDF.UMI - Read 2 Start From Cycle}, IF, NODATA Investigator Name,${PROCESS.TECHNICIAN} Experiment Name,${PROCESS.UDF.Experiment Name} Date,${DATE} -Workflow,${PROCESS.UDF.Workflow} [Reads] ${PROCESS.UDF.Read 1 Cycles} ${PROCESS.UDF.Read 2 Cycles} -${PROCESS.UDF.Settings Header} +[Settings] +Adapter,${PROCESS.UDF.Adapter} +AdapterRead2,${PROCESS.UDF.Adapter Read 2} Read1UMILength,${PROCESS.UDF.UMI - Read 1 Length} Read2UMILength,${PROCESS.UDF.UMI - Read 2 Length} Read1StartFromCycle,${PROCESS.UDF.UMI - Read 1 Start From Cycle} From 89fd6a4ac640926c614c7160bae53fe82fa239c0 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 10 Jun 2021 15:12:03 +0200 Subject: [PATCH 37/53] Python 3 compatibility --- clarity_epp/export/illumina.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index 7ec91a2..b60dde2 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -95,7 +95,7 @@ def get_project(projects, urgent=False): for project_type in project_types: project_types[project_type]['index'] = 0 if project_types[project_type]['split_project_type']: - for i in range(0, project_types[project_type]['sample_count']/9+1): + for i in range(0, int(project_types[project_type]['sample_count']/9+1)): project_types[project_type]['projects']['{0}_{1}'.format(project_type, i+1)] = 0 else: project_types[project_type]['projects'][project_type] = 0 @@ -181,4 +181,4 @@ def get_project(projects, urgent=False): output_file.write('{line}\n'.format(line=','.join(data))) else: # Leave other lines untouched. - output_file.write('{line}\n'.format(line=line)) + output_file.write('{line}\n'.format(line=line)) \ No newline at end of file From 192bc8cfa251b61d9b98d871381573a738fd7c71 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 10 Jun 2021 16:22:02 +0200 Subject: [PATCH 38/53] Fix sample names --- clarity_epp/export/illumina.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index b60dde2..bb548cc 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -102,19 +102,24 @@ def get_project(projects, urgent=False): # Set sample projects sample_projects = {} + sample_sequence_names = {} # Urgent families / samples, skip merge for family in [family for family in families.values() if family['urgent'] and not family['merge']]: family_project = get_project(project_types[family['project_type']]['projects'], urgent=True) for sample in family['samples']: - sample_projects[get_sequence_name(sample)] = family_project + sample_sequence_name = get_sequence_name(sample) + sample_sequence_names[sample.name] = sample_sequence_name + sample_projects[sample_sequence_name] = family_project project_types[family['project_type']]['projects'][family_project] += 1 # Merge families / samples for family in [family for family in families.values() if family['merge']]: family_project = get_project(project_types[family['project_type']]['projects']) for sample in family['samples']: - sample_projects[get_sequence_name(sample)] = family_project + sample_sequence_name = get_sequence_name(sample) + sample_sequence_names[sample.name] = sample_sequence_name + sample_projects[sample_sequence_name] = family_project project_types[family['project_type']]['projects'][family_project] += 1 # Non urgent and non merge families / samples @@ -122,7 +127,9 @@ def get_project(projects, urgent=False): for family in sorted(non_urgent_families, key=lambda fam: (len(fam['samples'])), reverse=True): family_project = get_project(project_types[family['project_type']]['projects']) for sample in family['samples']: - sample_projects[get_sequence_name(sample)] = family_project + sample_sequence_name = get_sequence_name(sample) + sample_sequence_names[sample.name] = sample_sequence_name + sample_projects[sample_sequence_name] = family_project project_types[family['project_type']]['projects'][family_project] += 1 # Check sequencer type -> NextSeq runs need to reverse complement 'index2' for dual barcodes and 'index' for single barcodes. @@ -166,6 +173,12 @@ def get_project(projects, urgent=False): elif sample_header: # Samples header seen, so continue with samples. data = line.rstrip().split(',') + # Fix sample name -> use sequence name + try: + data[sample_name_index] = sample_sequence_names[data[sample_name_index]] + except KeyError: + pass + # Set Sample_Project try: data[sample_project_index] = sample_projects[data[sample_name_index]] From 4b75934d884def96b246b0d9224f1f9390efb346 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 28 Jun 2021 17:01:19 +0200 Subject: [PATCH 39/53] Do not set Read1EndWithCycle for mip sampelsheets. --- clarity_epp/export/illumina.py | 47 ++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index bb548cc..e40b5fb 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -12,6 +12,7 @@ def update_samplesheet(lims, process_id, artifact_id, output_file): """Update illumina samplesheet.""" process = Process(lims, id=process_id) + trim_last_base = True # Used to set Read1EndWithCycle def get_project(projects, urgent=False): """Inner function to get a project name for samples.""" @@ -28,13 +29,22 @@ def get_project(projects, urgent=False): families = {} for artifact in process.all_inputs(): for sample in artifact.samples: - if 'Dx Familienummer' in list(sample.udf) and 'Dx NICU Spoed' in list(sample.udf) and 'Dx Protocolomschrijving' in list(sample.udf): + if ('Dx Familienummer' in list(sample.udf) + and 'Dx NICU Spoed' in list(sample.udf) + and 'Dx Protocolomschrijving' in list(sample.udf)): # Dx production sample family = sample.udf['Dx Familienummer'] # Create family if not exist if family not in families: - families[family] = {'samples': [], 'NICU': False, 'project_type': 'unknown_project', 'split_project_type': False, 'urgent': False, 'merge': False} + families[family] = { + 'samples': [], + 'NICU': False, + 'project_type': 'unknown_project', + 'split_project_type': False, + 'urgent': False, + 'merge': False + } # Update family information if sample.udf['Dx Onderzoeksreden'] == 'Research': # Dx research sample @@ -50,10 +60,12 @@ def get_project(projects, urgent=False): project_type = 'Fingerprint' families[family]['project_type'] = project_type families[family]['split_project_type'] = False + trim_last_base = False elif 'PID09.V7_smMIP' in sample.udf['Dx Protocolomschrijving'] and not families[family]['NICU']: project_type = 'ERARE' families[family]['project_type'] = project_type families[family]['split_project_type'] = False + trim_last_base = False elif sample.udf['Dx NICU Spoed']: families[family]['NICU'] = True project_type = 'NICU_{0}'.format(sample.udf['Dx Familienummer']) @@ -63,7 +75,7 @@ def get_project(projects, urgent=False): project_type = 'CREv2' families[family]['project_type'] = project_type families[family]['split_project_type'] = True - + # Set urgent / merge status if 'Dx Spoed' in list(sample.udf) and sample.udf['Dx Spoed']: families[family]['urgent'] = True @@ -76,9 +88,17 @@ def get_project(projects, urgent=False): family = 'GIAB' else: family = sample.project.name - family = re.sub('^dx[ _]*', '', family, flags=re.IGNORECASE) # Remove 'dx' (ignore case) and strip leading space or _ + # Remove 'dx' (ignore case) and strip leading space or _ + family = re.sub('^dx[ _]*', '', family, flags=re.IGNORECASE) if family not in families: - families[family] = {'samples': [], 'NICU': False, 'project_type': family, 'split_project_type': False, 'urgent': False, 'merge': False} + families[family] = { + 'samples': [], + 'NICU': False, + 'project_type': family, + 'split_project_type': False, + 'urgent': False, + 'merge': False + } # Add sample to family families[family]['samples'].append(sample) @@ -89,7 +109,11 @@ def get_project(projects, urgent=False): if family['project_type'] in project_types: project_types[family['project_type']]['sample_count'] += len(family['samples']) else: - project_types[family['project_type']] = {'sample_count': len(family['samples']), 'projects': {}, 'split_project_type': family['split_project_type']} + project_types[family['project_type']] = { + 'sample_count': len(family['samples']), + 'projects': {}, + 'split_project_type': family['split_project_type'] + } # Define projects per project_type for project_type in project_types: @@ -112,7 +136,7 @@ def get_project(projects, urgent=False): sample_sequence_names[sample.name] = sample_sequence_name sample_projects[sample_sequence_name] = family_project project_types[family['project_type']]['projects'][family_project] += 1 - + # Merge families / samples for family in [family for family in families.values() if family['merge']]: family_project = get_project(project_types[family['project_type']]['projects']) @@ -132,7 +156,8 @@ def get_project(projects, urgent=False): sample_projects[sample_sequence_name] = family_project project_types[family['project_type']]['projects'][family_project] += 1 - # Check sequencer type -> NextSeq runs need to reverse complement 'index2' for dual barcodes and 'index' for single barcodes. + # Check sequencer type + # NextSeq runs need to reverse complement 'index2' for dual barcodes and 'index' for single barcodes. if 'nextseq' in process.type.name.lower(): nextseq_run = True else: @@ -145,13 +170,13 @@ def get_project(projects, urgent=False): file_id = samplesheet_artifact.files[0].id for line in lims.get_file_contents(id=file_id).rstrip().split('\n'): - if line.startswith('[Settings]'): + if line.startswith('[Settings]') and trim_last_base: output_file.write('{line}\n'.format(line=line)) output_file.write('Read1EndWithCycle,{value}\n'.format(value=process.udf['Read 1 Cycles']-1)) output_file.write('Read2EndWithCycle,{value}\n'.format(value=process.udf['Read 2 Cycles']-1)) settings_section = True - elif line.startswith('[Data]') and not settings_section: + elif line.startswith('[Data]') and trim_last_base and not settings_section: output_file.write('[Settings]\n') output_file.write('Read1EndWithCycle,{value}\n'.format(value=process.udf['Read 1 Cycles']-1)) output_file.write('Read2EndWithCycle,{value}\n'.format(value=process.udf['Read 2 Cycles']-1)) @@ -194,4 +219,4 @@ def get_project(projects, urgent=False): output_file.write('{line}\n'.format(line=','.join(data))) else: # Leave other lines untouched. - output_file.write('{line}\n'.format(line=line)) \ No newline at end of file + output_file.write('{line}\n'.format(line=line)) From b3ecf78de9b5d799f242b3282a84d088b37801f8 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Fri, 2 Jul 2021 09:47:41 +0200 Subject: [PATCH 40/53] Change order sample / sequencing run --- clarity_epp/export/merge.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clarity_epp/export/merge.py b/clarity_epp/export/merge.py index ed6fe99..b921706 100644 --- a/clarity_epp/export/merge.py +++ b/clarity_epp/export/merge.py @@ -9,12 +9,12 @@ def create_file(lims, process_id, output_file): process = Process(lims, id=process_id) samples = process.analytes()[0][0].samples - output_file.write('Sample\tMerge 1 Sequencing Run\tMerge 1 Sample\tMerge 2 Sequencing Run\tMerge 2 Sample\n') + output_file.write('Sample\tMerge 1 Sample\tMerge 1 Sequencing Run\tMerge 2 Sample\tMerge 2 Sequencing Run\n') for sample in samples: sample_merge = [] if 'Dx Mergen' in sample.udf and sample.udf['Dx Mergen']: - for udf in ['Dx Merge 1 Runnaam', 'Dx Merge 1 Samplenaam', 'Dx Merge 2 Runnaam', 'Dx Merge 2 Samplenaam']: + for udf in ['Dx Merge 1 Samplenaam', 'Dx Merge 1 Runnaam', 'Dx Merge 2 Samplenaam', 'Dx Merge 2 Runnaam']: if udf in sample.udf: sample_merge.append(sample.udf[udf]) else: @@ -24,3 +24,4 @@ def create_file(lims, process_id, output_file): sample=get_sequence_name(sample), merge='\t'.join(sample_merge) )) +1 \ No newline at end of file From ed562df0561bec8488c27fbbc667bc03dae093c5 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 7 Jul 2021 13:25:09 +0200 Subject: [PATCH 41/53] Perform calculation if udf fields are available. --- clarity_epp/qc/illumina.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/clarity_epp/qc/illumina.py b/clarity_epp/qc/illumina.py index c616b66..b662b8e 100644 --- a/clarity_epp/qc/illumina.py +++ b/clarity_epp/qc/illumina.py @@ -7,12 +7,13 @@ def set_avg_q30(lims, process_id): process = Process(lims, id=process_id) artifact = process.analytes()[0][0] - r1_q30 = artifact.udf['% Bases >=Q30 R1'] - r1_yield = artifact.udf['Yield PF (Gb) R1'] - r2_q30 = artifact.udf['% Bases >=Q30 R2'] - r2_yield = artifact.udf['Yield PF (Gb) R2'] + if all([udf in artifact.udf for udf in ['% Bases >=Q30 R1', 'Yield PF (Gb) R1', '% Bases >=Q30 R2', 'Yield PF (Gb) R2']]): + r1_q30 = artifact.udf['% Bases >=Q30 R1'] + r1_yield = artifact.udf['Yield PF (Gb) R1'] + r2_q30 = artifact.udf['% Bases >=Q30 R2'] + r2_yield = artifact.udf['Yield PF (Gb) R2'] - average_q30 = (r1_q30*r1_yield + r2_q30*r2_yield) / (r1_yield+r2_yield) + average_q30 = (r1_q30*r1_yield + r2_q30*r2_yield) / (r1_yield+r2_yield) - artifact.udf['Dx Average % Bases >=Q30'] = average_q30 - artifact.put() + artifact.udf['Dx Average % Bases >=Q30'] = average_q30 + artifact.put() From 8a79cf623918fe04ea3fe75c91f1ca06624816d4 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 19 Jul 2021 11:39:33 +0200 Subject: [PATCH 42/53] Update to work with PID09 samples. --- clarity_epp/export/manual_pipetting.py | 30 ++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 17d2921..4e5fafb 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -334,9 +334,9 @@ def samplesheet_multiplex_sequence_pool(lims, process_id, output_file): def samplesheet_normalization(lims, process_id, output_file): - """Create manual pipetting samplesheet for normalizing samples.""" + """Create manual pipetting samplesheet for normalizing (MIP) samples.""" output_file.write( - 'Sample\tConcentration (ng/ul)\tEindvolume (ul)\tVerdunningsfactor\tVolume sample (ul)\tVolume water (ul)\n' + 'Sample\tConcentration (ng/ul)\tVolume sample (ul)\tVolume water (ul)\tOutput (ng)\tIndampen\n' ) process = Process(lims, id=process_id) @@ -360,21 +360,33 @@ def samplesheet_normalization(lims, process_id, output_file): # Find concentration measurement for qc_artifact in qc_process.outputs_per_input(input_artifact.id): - if qc_artifact.name == artifact.name: + if qc_artifact.name.split(' ')[0] == artifact.name: concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) + #concentration = float(qc_artifact.udf['Dx Conc. goedgekeurde meting (ng/ul)']) - final_volume = artifact.udf['Dx Eindvolume (ul)'] - dilution_factor = concentration / 20 - sample_volume = final_volume / dilution_factor + final_volume = float(artifact.udf['Dx Eindvolume (ul)']) + input_ng = float(artifact.udf['Dx Input (ng)']) + if 'Dx pipetteervolume (ul)' in artifact.udf: + input_ng = concentration * float(artifact.udf['Dx pipetteervolume (ul)']) + sample_volume = input_ng / concentration water_volume = final_volume - sample_volume + evaporate = 'N' - output_file.write('{sample}\t{concentration:.1f}\t{final_volume}\t{dilution_factor:.1f}\t{sample_volume:.1f}\t{water_volume:.1f}\n'.format( + if sample_volume < 0.5: + sample_volume = 0.5 + water_volume = final_volume - sample_volume + elif sample_volume > final_volume: + evaporate = 'J' + water_volume = 0 + + output_file.write('{sample}\t{concentration:.1f}\t{final_volume}\t{sample_volume:.1f}\t{water_volume:.1f}\t{output:.1f}\t{evaporate}\n'.format( sample=sample.name, concentration=concentration, final_volume=final_volume, - dilution_factor=dilution_factor, sample_volume=sample_volume, - water_volume=water_volume + water_volume=water_volume, + output=input_ng, + evaporate=evaporate )) From e15e3078ded9a5e8254aee5f3cd07caf3759d8ca Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 19 Jul 2021 12:08:45 +0200 Subject: [PATCH 43/53] Remove final_volume from output --- clarity_epp/export/manual_pipetting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index 4e5fafb..cce460a 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -379,10 +379,9 @@ def samplesheet_normalization(lims, process_id, output_file): evaporate = 'J' water_volume = 0 - output_file.write('{sample}\t{concentration:.1f}\t{final_volume}\t{sample_volume:.1f}\t{water_volume:.1f}\t{output:.1f}\t{evaporate}\n'.format( + output_file.write('{sample}\t{concentration:.1f}\t{sample_volume:.1f}\t{water_volume:.1f}\t{output:.1f}\t{evaporate}\n'.format( sample=sample.name, concentration=concentration, - final_volume=final_volume, sample_volume=sample_volume, water_volume=water_volume, output=input_ng, From 72ef09ddd6af60d9f6bf194185fb3fe196f9043f Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 21 Jul 2021 17:25:52 +0200 Subject: [PATCH 44/53] Remove dots for help msg --- clarity_epp.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/clarity_epp.py b/clarity_epp.py index ca08fe2..2008f27 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -80,7 +80,7 @@ def export_manual_pipetting(args): elif args.type == 'mip_multiplex_pool': clarity_epp.export.manual_pipetting.samplesheet_mip_multiplex_pool(lims, args.process_id, args.output_file) elif args.type == 'mip_dilute_pool': - clarity_epp.export.manual_pipetting.samplesheet_mip_pool_dilution(lims, args.process_id, args.output_file) + clarity_epp.export.manual_pipetting.samplesheet_mip_pool_dilution(lims, args.process_id, args.output_file) def export_ped_file(args): @@ -201,7 +201,7 @@ def placement_complete_step(args): ) # export - parser_export = subparser.add_parser('export', help='Export from lims.') + parser_export = subparser.add_parser('export', help='Export from lims') subparser_export = parser_export.add_subparsers() parser_export_bioanalyzer = subparser_export.add_parser( @@ -228,13 +228,13 @@ def placement_complete_step(args): parser_export_hamilton.set_defaults(func=export_hamilton) parser_export_illumina = subparser_export.add_parser( - 'illumina', help='Export updated illumina samplesheet.', parents=[output_parser] + 'illumina', help='Export updated illumina samplesheet', parents=[output_parser] ) parser_export_illumina.add_argument('process_id', help='Clarity lims process id') parser_export_illumina.add_argument('artifact_id', help='Clarity lims samplesheet artifact id') parser_export_illumina.set_defaults(func=export_illumina) - parser_export_labels = subparser_export.add_parser('labels', help='Export container labels.', parents=[output_parser]) + parser_export_labels = subparser_export.add_parser('labels', help='Export container labels', parents=[output_parser]) parser_export_labels.add_argument('type', choices=['container', 'container_sample', 'storage_location'], help='Label type') parser_export_labels.add_argument('process_id', help='Clarity lims process id') parser_export_labels.add_argument('-d', '--description', nargs='?', help='Container name description') @@ -254,16 +254,16 @@ def placement_complete_step(args): parser_export_manual_pipetting.add_argument('process_id', help='Clarity lims process id') parser_export_manual_pipetting.set_defaults(func=export_manual_pipetting) - parser_export_merge = subparser_export.add_parser('merge', help='Export merge file.', parents=[output_parser]) + parser_export_merge = subparser_export.add_parser('merge', help='Export merge file', parents=[output_parser]) parser_export_merge.add_argument('process_id', help='Clarity lims process id') parser_export_merge.set_defaults(func=export_merge_file) - parser_export_ped = subparser_export.add_parser('ped', help='Export ped file.', parents=[output_parser]) + parser_export_ped = subparser_export.add_parser('ped', help='Export ped file', parents=[output_parser]) parser_export_ped.add_argument('process_id', help='Clarity lims process id') parser_export_ped.set_defaults(func=export_ped_file) parser_export_removed_samples = subparser_export.add_parser( - 'removed_samples', help='Export removed sampels table.', parents=[output_parser] + 'removed_samples', help='Export removed sampels table', parents=[output_parser] ) parser_export_removed_samples.set_defaults(func=export_removed_samples) @@ -278,7 +278,7 @@ def placement_complete_step(args): parser_export_tecan.set_defaults(func=export_tecan) parser_export_workflow = subparser_export.add_parser( - 'workflow', help='Export workflow result file.', parents=[output_parser] + 'workflow', help='Export workflow result file', parents=[output_parser] ) parser_export_workflow.add_argument('type', choices=['all', 'lab', 'data_analysis'], help='Workflow type') parser_export_workflow.add_argument('process_id', help='Clarity lims process id') @@ -305,50 +305,50 @@ def placement_complete_step(args): parser_upload_tecan.set_defaults(func=upload_tecan_results) # QC - parser_qc = subparser.add_parser('qc', help='Set QC values/flags.') + parser_qc = subparser.add_parser('qc', help='Set QC values/flags') subparser_qc = parser_qc.add_subparsers() - parser_qc_fragment_length = subparser_qc.add_parser('fragment_length', help='Set fragment length qc flag.') + parser_qc_fragment_length = subparser_qc.add_parser('fragment_length', help='Set fragment length qc flag') parser_qc_fragment_length.add_argument('process_id', help='Clarity lims process id') parser_qc_fragment_length.set_defaults(func=qc_fragment_length) - parser_qc_illumina = subparser_qc.add_parser('illumina', help='Set average % Bases >=Q30.') + parser_qc_illumina = subparser_qc.add_parser('illumina', help='Set average % Bases >=Q30') parser_qc_illumina.add_argument('process_id', help='Clarity lims process id') parser_qc_illumina.set_defaults(func=qc_illumina) - parser_qc_qubit = subparser_qc.add_parser('qubit', help='Set qubit qc flag.') + parser_qc_qubit = subparser_qc.add_parser('qubit', help='Set qubit qc flag') parser_qc_qubit.add_argument('process_id', help='Clarity lims process id') parser_qc_qubit.set_defaults(func=qc_qubit) # placement - parser_placement = subparser.add_parser('placement', help='Container placement functions.') + parser_placement = subparser.add_parser('placement', help='Container placement functions') subparser_placement = parser_placement.add_subparsers() - parser_placement_automatic = subparser_placement.add_parser('copy', help='Copy container layout from previous step.') + parser_placement_automatic = subparser_placement.add_parser('copy', help='Copy container layout from previous step') parser_placement_automatic.add_argument('process_id', help='Clarity lims process id') parser_placement_automatic.set_defaults(func=placement_automatic) - parser_placement_artifact = subparser_placement.add_parser('artifact', help='Change artifact name to sequence name.') + parser_placement_artifact = subparser_placement.add_parser('artifact', help='Change artifact name to sequence name') parser_placement_artifact.add_argument('type', choices=['sequence_name', 'run_id'], help='Check type') parser_placement_artifact.add_argument('process_id', help='Clarity lims process id') parser_placement_artifact.set_defaults(func=placement_artifact_set_name) - parser_placement_route_artifact = subparser_placement.add_parser('route_artifact', help='Route artifact to a workflow.') + parser_placement_route_artifact = subparser_placement.add_parser('route_artifact', help='Route artifact to a workflow') parser_placement_route_artifact.add_argument('process_id', help='Clarity lims process id') parser_placement_route_artifact.set_defaults(func=placement_route_artifact) - parser_placement_barcode = subparser_placement.add_parser('barcode_check', help='Check barcode clarity_epp.placement.') + parser_placement_barcode = subparser_placement.add_parser('barcode_check', help='Check barcode clarity_epp.placement') parser_placement_barcode.add_argument('type', choices=['check_family'], help='Check type') parser_placement_barcode.add_argument('process_id', help='Clarity lims process id') parser_placement_barcode.set_defaults(func=placement_barcode) parser_placement_complete_step = subparser_placement.add_parser( - 'complete_step', help='Complete step Dx Mark protocol complete.' + 'complete_step', help='Complete step Dx Mark protocol complete' ) parser_placement_complete_step.add_argument('process_id', help='Clarity lims process id') parser_placement_complete_step.set_defaults(func=placement_complete_step) - parser_placement_unpooling = subparser_placement.add_parser('unpooling', help='Unpooling of sequencing pool.') + parser_placement_unpooling = subparser_placement.add_parser('unpooling', help='Unpooling of sequencing pool') parser_placement_unpooling.add_argument('process_id', help='Clarity lims process id') parser_placement_unpooling.set_defaults(func=placement_unpooling) From 78e5129b54f635abb8ce601446aea98330ed0671 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 21 Jul 2021 17:36:29 +0200 Subject: [PATCH 45/53] Remove dev comment --- clarity_epp/export/manual_pipetting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clarity_epp/export/manual_pipetting.py b/clarity_epp/export/manual_pipetting.py index cce460a..b1ba2b7 100755 --- a/clarity_epp/export/manual_pipetting.py +++ b/clarity_epp/export/manual_pipetting.py @@ -362,7 +362,6 @@ def samplesheet_normalization(lims, process_id, output_file): for qc_artifact in qc_process.outputs_per_input(input_artifact.id): if qc_artifact.name.split(' ')[0] == artifact.name: concentration = float(qc_artifact.udf['Dx Concentratie fluorescentie (ng/ul)']) - #concentration = float(qc_artifact.udf['Dx Conc. goedgekeurde meting (ng/ul)']) final_volume = float(artifact.udf['Dx Eindvolume (ul)']) input_ng = float(artifact.udf['Dx Input (ng)']) From 98705a22ed1339de2d45d39ba7759e1cf5e9c61d Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 21 Jul 2021 17:37:35 +0200 Subject: [PATCH 46/53] Remove typo --- clarity_epp/export/merge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clarity_epp/export/merge.py b/clarity_epp/export/merge.py index b921706..8f71555 100644 --- a/clarity_epp/export/merge.py +++ b/clarity_epp/export/merge.py @@ -24,4 +24,3 @@ def create_file(lims, process_id, output_file): sample=get_sequence_name(sample), merge='\t'.join(sample_merge) )) -1 \ No newline at end of file From d254d853066133b05298f670a6843fcf7d3c5274 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 21 Jul 2021 17:39:52 +0200 Subject: [PATCH 47/53] Remove unused import --- scripts/move_samples_to_new_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/move_samples_to_new_workflow.py b/scripts/move_samples_to_new_workflow.py index aee3e36..f0e76e6 100644 --- a/scripts/move_samples_to_new_workflow.py +++ b/scripts/move_samples_to_new_workflow.py @@ -1,5 +1,5 @@ from genologics.lims import Lims -from genologics.entities import Workflow, Step, Queue, Stage +from genologics.entities import Workflow, Queue, Stage import config From 71b60e66440ea2666cf28c8ab81f60cbbcc6879e Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 21 Jul 2021 17:47:44 +0200 Subject: [PATCH 48/53] Remove try except --- clarity_epp/export/illumina.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/clarity_epp/export/illumina.py b/clarity_epp/export/illumina.py index e40b5fb..59fa8e1 100644 --- a/clarity_epp/export/illumina.py +++ b/clarity_epp/export/illumina.py @@ -199,16 +199,12 @@ def get_project(projects, urgent=False): data = line.rstrip().split(',') # Fix sample name -> use sequence name - try: + if data[sample_name_index] in sample_sequence_names: data[sample_name_index] = sample_sequence_names[data[sample_name_index]] - except KeyError: - pass # Set Sample_Project - try: + if data[sample_name_index] in sample_projects: data[sample_project_index] = sample_projects[data[sample_name_index]] - except KeyError: - pass # Overwrite Sample_ID with Sample_name to get correct conversion output folder structure data[sample_id_index] = data[sample_name_index] From 8c155092070bb26f27e6958febc8db7e9f90363e Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Wed, 28 Jul 2021 15:28:10 +0200 Subject: [PATCH 49/53] Add export_sample_indications --- clarity_epp.py | 23 ++++++++++++++++++++--- clarity_epp/export/sample.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/clarity_epp.py b/clarity_epp.py index ca08fe2..60e3d77 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -80,7 +80,7 @@ def export_manual_pipetting(args): elif args.type == 'mip_multiplex_pool': clarity_epp.export.manual_pipetting.samplesheet_mip_multiplex_pool(lims, args.process_id, args.output_file) elif args.type == 'mip_dilute_pool': - clarity_epp.export.manual_pipetting.samplesheet_mip_pool_dilution(lims, args.process_id, args.output_file) + clarity_epp.export.manual_pipetting.samplesheet_mip_pool_dilution(lims, args.process_id, args.output_file) def export_ped_file(args): @@ -94,10 +94,17 @@ def export_merge_file(args): def export_removed_samples(args): - """Export removed sampels table.""" + """Export removed samples table.""" clarity_epp.export.sample.removed_samples(lims, args.output_file) +def export_sample_indications(args): + """Export sample indication table.""" + clarity_epp.export.sample.sample_indications( + lims, args.output_file, args.artifact_name, args.sequencing_run, args.sequencing_run_project + ) + + def export_tapestation(args): """Export samplesheets for Tapestation machine.""" clarity_epp.export.tapestation.samplesheet(lims, args.process_id, args.output_file) @@ -263,10 +270,20 @@ def placement_complete_step(args): parser_export_ped.set_defaults(func=export_ped_file) parser_export_removed_samples = subparser_export.add_parser( - 'removed_samples', help='Export removed sampels table.', parents=[output_parser] + 'removed_samples', help='Export removed samples table.', parents=[output_parser] ) parser_export_removed_samples.set_defaults(func=export_removed_samples) + parser_export_sample_indications = subparser_export.add_parser( + 'sample_indications', help='Export sample indication table.', parents=[output_parser] + ) + parser_export_sample_indications.add_argument('-a', '--artifact_name', nargs='?', help='Artifact name') + parser_export_sample_indications.add_argument('-r', '--sequencing_run', nargs='?', help='Sequencing run name') + parser_export_sample_indications.add_argument( + '-p', '--sequencing_run_project', nargs='?', help='Sequencing run project name' + ) + parser_export_sample_indications.set_defaults(func=export_sample_indications) + parser_export_tapestation = subparser_export.add_parser( 'tapestation', help='Create tapestation samplesheets', parents=[output_parser] ) diff --git a/clarity_epp/export/sample.py b/clarity_epp/export/sample.py index 64db25a..4df6f69 100644 --- a/clarity_epp/export/sample.py +++ b/clarity_epp/export/sample.py @@ -120,3 +120,32 @@ def removed_samples(lims, output_file): stage=removed_stage.name, removed_status=sample_removed_status )) + + +def sample_indications(lims, output_file, artifact_name=None, sequencing_run=None, sequencing_run_project=None): + """Export table with sample indications. Lookup samples by sample name or sequencing run (project).""" + samples = [] + + # Get samples by artifact_name + if artifact_name: + artifacts = lims.get_artifacts(name=artifact_name) + samples = {artifact_name: artifact.samples[0] for artifact in artifacts} + + # Get samples by sequencing run + elif sequencing_run: + udf_query = {'Dx Sequencing Run ID': sequencing_run} + if sequencing_run_project: + udf_query['Dx Sequencing Run Project'] = sequencing_run_project + + artifacts = lims.get_artifacts(type='Analyte', udf=udf_query) + samples = {artifact.name: artifact.samples[0] for artifact in artifacts} + + # Write result + if samples: + output_file.write('Sample\tIndication\n') + for sample_name, sample in samples.items(): + output_file.write( + '{sample}\t{indication}\n'.format(sample=sample_name, indication=sample.udf['Dx Onderzoeksindicatie']) + ) + else: + print("Can't find sample(s).") From 638a0eba27ebbe2f59715cd94f1448eaf4b7a557 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 29 Jul 2021 13:58:29 +0200 Subject: [PATCH 50/53] Add mutually exclusive group --- clarity_epp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clarity_epp.py b/clarity_epp.py index 60e3d77..fe1ef0c 100755 --- a/clarity_epp.py +++ b/clarity_epp.py @@ -277,8 +277,9 @@ def placement_complete_step(args): parser_export_sample_indications = subparser_export.add_parser( 'sample_indications', help='Export sample indication table.', parents=[output_parser] ) - parser_export_sample_indications.add_argument('-a', '--artifact_name', nargs='?', help='Artifact name') - parser_export_sample_indications.add_argument('-r', '--sequencing_run', nargs='?', help='Sequencing run name') + parser_export_sample_indications_group = parser_export_sample_indications.add_mutually_exclusive_group(required=True) + parser_export_sample_indications_group.add_argument('-a', '--artifact_name', help='Artifact name') + parser_export_sample_indications_group.add_argument('-r', '--sequencing_run', help='Sequencing run name') parser_export_sample_indications.add_argument( '-p', '--sequencing_run_project', nargs='?', help='Sequencing run project name' ) From ea091d8fd19d60bf199cf0b4b0a5d0493fccafba Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Thu, 5 Aug 2021 11:56:23 +0200 Subject: [PATCH 51/53] Add ng/ul support. --- clarity_epp/upload/tapestation.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/clarity_epp/upload/tapestation.py b/clarity_epp/upload/tapestation.py index 2cb3a20..d0a3c49 100644 --- a/clarity_epp/upload/tapestation.py +++ b/clarity_epp/upload/tapestation.py @@ -21,7 +21,12 @@ def results(lims, process_id): concentration_index = None else: # Tapestation compact region table size_index = header.index('Average Size [bp]') - concentration_index = header.index(u'Conc. [pg/\xb5l]') # micro sign + try: + concentration_index = header.index(u'Conc. [pg/\xb5l]') # micro sign + concentration_correction = 1000 # Used to transform pg/ul to ng/ul + except ValueError: + concentration_index = header.index(u'Conc. [ng/\xb5l]') # micro sign + concentration_correction = 1 sample_index = header.index('Sample Description') elif line: @@ -32,7 +37,8 @@ def results(lims, process_id): size = int(data[size_index]) sample_size_measurements[sample] = size if concentration_index and data[concentration_index]: - concentration = float(data[concentration_index]) / 1000 # pg/ul to ng/ul + # Correct concentration + concentration = float(data[concentration_index]) / concentration_correction sample_concentration_measurements[sample] = concentration # Set UDF From 70b19a804108ff8c49797d120f1b73b2548f0ce7 Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Mon, 9 Aug 2021 16:52:21 +0200 Subject: [PATCH 52/53] Fix tecan samplesheet for pools and python3 fixes --- clarity_epp/export/tecan.py | 7 +++++-- clarity_epp/export/utils.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/clarity_epp/export/tecan.py b/clarity_epp/export/tecan.py index b9440fc..e444c23 100755 --- a/clarity_epp/export/tecan.py +++ b/clarity_epp/export/tecan.py @@ -11,9 +11,12 @@ def samplesheet(lims, process_id, output_file): process = Process(lims, id=process_id) well_plate = {} - for placement, artifact in process.output_containers()[0].placements.iteritems(): + for placement, artifact in process.output_containers()[0].placements.items(): placement = ''.join(placement.split(':')) - well_plate[placement] = artifact.name.split('_')[0] + if len(artifact.samples) == 1: # Remove 'meet_id' from artifact name if artifact is not a pool + well_plate[placement] = artifact.name.split('_')[0] + else: + well_plate[placement] = artifact.name for well in clarity_epp.export.utils.sort_96_well_plate(well_plate.keys()): output_file.write('{well}\t{sample}\n'.format( diff --git a/clarity_epp/export/utils.py b/clarity_epp/export/utils.py index f9ca27a..2193ed8 100755 --- a/clarity_epp/export/utils.py +++ b/clarity_epp/export/utils.py @@ -17,7 +17,7 @@ def sort_96_well_plate(wells): ] order = dict(zip(order, range(len(order)))) - wells.sort(key=lambda val: order[val]) + wells = sorted(wells, key=lambda val: order[val]) return wells From b060c569b31d5090106033804f5c785abc6b8f5f Mon Sep 17 00:00:00 2001 From: Robert Ernst Date: Tue, 10 Aug 2021 09:14:31 +0200 Subject: [PATCH 53/53] Fix tecan upload for pools --- clarity_epp/upload/tecan.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/clarity_epp/upload/tecan.py b/clarity_epp/upload/tecan.py index 4d41340..4ab8222 100644 --- a/clarity_epp/upload/tecan.py +++ b/clarity_epp/upload/tecan.py @@ -65,7 +65,11 @@ def results(lims, process_id): for artifact in process.all_outputs(): if artifact.name not in ['Tecan Spark Output', 'Tecan Spark Samplesheet', 'check gemiddelde concentratie', 'Label plaat']: - artifact_name = artifact.name.split('_')[0] + if len(artifact.samples) == 1: # Remove 'meet_id' from artifact name if artifact is not a pool + artifact_name = artifact.name.split('_')[0] + else: + artifact_name = artifact.name + # Set Average Concentratie fluorescentie sample_fluorescence = sum(sample_measurements[artifact_name]) / float(len(sample_measurements[artifact_name])) sample_concentration = ((sample_fluorescence - baseline_fluorescence) * regression_slope) / 2.0