Merge pull request #34 from UMCUGenetics/release/v1.5.0

Release/v1.5.0

clarity_epp.py

@@ -28,6 +28,8 @@ def export_caliper(args):
     """Export samplesheets for caliper machine."""
     if args.type == 'normalise':
         clarity_epp.export.caliper.samplesheet_normalise(lims, args.process_id, args.output_file)
+    elif args.type == 'dilute':
+        clarity_epp.export.caliper.samplesheet_dilute(lims, args.process_id, args.output_file)
 
 
 def export_email(args):
@@ -59,6 +61,11 @@ def export_labels(args):
         clarity_epp.export.labels.storage_location(lims, args.process_id, args.output_file)
 
 
+def export_magnis(args):
+    """Export magnis samplesheet."""
+    clarity_epp.export.magnis.samplesheet(lims, args.process_id, args.output_file)
+
+
 def export_manual_pipetting(args):
     """Export samplesheets for manual pipetting."""
     if args.type == 'purify':
@@ -81,6 +88,10 @@ def export_manual_pipetting(args):
         clarity_epp.export.manual_pipetting.samplesheet_mip_multiplex_pool(lims, args.process_id, args.output_file)
     elif args.type == 'mip_dilute_pool':
         clarity_epp.export.manual_pipetting.samplesheet_mip_pool_dilution(lims, args.process_id, args.output_file)
+    elif args.type == 'pool_samples':
+        clarity_epp.export.manual_pipetting.samplesheet_pool_samples(lims, args.process_id, args.output_file)
+    elif args.type == 'pool_magnis_pools':
+        clarity_epp.export.manual_pipetting.samplesheet_pool_magnis_pools(lims, args.process_id, args.output_file)
 
 
 def export_ped_file(args):
@@ -123,6 +134,8 @@ def export_workflow(args):
         clarity_epp.export.workflow.helix_lab(lims, args.process_id, args.output_file)
     elif args.type == 'data_analysis':
         clarity_epp.export.workflow.helix_data_analysis(lims, args.process_id, args.output_file)
+    elif args.type == 'magnis':
+        clarity_epp.export.workflow.helix_all_magnis(lims, args.process_id, args.output_file)
 
 
 # Upload Functions
@@ -146,6 +159,11 @@ def upload_bioanalyzer_results(args):
     clarity_epp.upload.bioanalyzer.results(lims, args.process_id)
 
 
+def upload_magnis_results(args):
+    """Upload magnis results."""
+    clarity_epp.upload.magnis.results(lims, args.process_id)
+
+
 # QC functions
 def qc_fragment_length(args):
     """Set QC status based on fragment length measurement."""
@@ -178,7 +196,7 @@ def placement_artifact_set_name(args):
 
 def placement_route_artifact(args):
     """Route artifacts to a workflow"""
-    clarity_epp.placement.artifact.route_to_workflow(lims, args.process_id)
+    clarity_epp.placement.artifact.route_to_workflow(lims, args.process_id, args.workflow)
 
 
 def placement_barcode(args):
@@ -218,7 +236,7 @@ def placement_complete_step(args):
     parser_export_bioanalyzer.set_defaults(func=export_bioanalyzer)
 
     parser_export_caliper = subparser_export.add_parser('caliper', help='Create caliper samplesheets', parents=[output_parser])
-    parser_export_caliper.add_argument('type', choices=['normalise'], help='Samplesheet type')
+    parser_export_caliper.add_argument('type', choices=['normalise', 'dilute'], help='Samplesheet type')
     parser_export_caliper.add_argument('process_id', help='Clarity lims process id')
     parser_export_caliper.set_defaults(func=export_caliper)
 
@@ -247,14 +265,21 @@ def placement_complete_step(args):
     parser_export_labels.add_argument('-d', '--description',  nargs='?', help='Container name description')
     parser_export_labels.set_defaults(func=export_labels)
 
+    parser_export_magnis = subparser_export.add_parser(
+        'magnis', help='Export magnis samplesheet', parents=[output_parser]
+    )
+    parser_export_magnis.add_argument('process_id', help='Clarity lims process id')
+    parser_export_magnis.set_defaults(func=export_magnis)
+
     parser_export_manual_pipetting = subparser_export.add_parser(
         'manual', help='Create manual pipetting exports', parents=[output_parser]
     )
     parser_export_manual_pipetting.add_argument(
         'type',
         choices=[
             'purify', 'dilute_library_pool', 'multiplex_library_pool', 'multiplex_sequence_pool', 'normalization',
-            'capture', 'exonuclease', 'pcr_exonuclease', 'mip_multiplex_pool', 'mip_dilute_pool'
+            'capture', 'exonuclease', 'pcr_exonuclease', 'mip_multiplex_pool', 'mip_dilute_pool', 'pool_samples',
+            'pool_magnis_pools'
         ],
         help='Samplesheet type'
     )
@@ -298,7 +323,7 @@ def placement_complete_step(args):
     parser_export_workflow = subparser_export.add_parser(
         'workflow', help='Export workflow result file', parents=[output_parser]
     )
-    parser_export_workflow.add_argument('type', choices=['all', 'lab', 'data_analysis'], help='Workflow type')
+    parser_export_workflow.add_argument('type', choices=['all', 'lab', 'data_analysis', 'magnis'], help='Workflow type')
     parser_export_workflow.add_argument('process_id', help='Clarity lims process id')
     parser_export_workflow.set_defaults(func=export_workflow)
 
@@ -322,6 +347,10 @@ def placement_complete_step(args):
     parser_upload_tecan.add_argument('process_id', help='Clarity lims process id')
     parser_upload_tecan.set_defaults(func=upload_tecan_results)
 
+    parser_upload_magnis = subparser_upload.add_parser('magnis', help='Upload magnis results')
+    parser_upload_magnis.add_argument('process_id', help='Clarity lims process id')
+    parser_upload_magnis.set_defaults(func=upload_magnis_results)
+
     # QC
     parser_qc = subparser.add_parser('qc', help='Set QC values/flags')
     subparser_qc = parser_qc.add_subparsers()
@@ -353,6 +382,7 @@ def placement_complete_step(args):
 
     parser_placement_route_artifact = subparser_placement.add_parser('route_artifact', help='Route artifact to a workflow')
     parser_placement_route_artifact.add_argument('process_id', help='Clarity lims process id')
+    parser_placement_route_artifact.add_argument('workflow', choices=['post_bioinf', 'sequencing'], help='Workflow')
     parser_placement_route_artifact.set_defaults(func=placement_route_artifact)
 
     parser_placement_barcode = subparser_placement.add_parser('barcode_check', help='Check barcode clarity_epp.placement')

clarity_epp/export/__init__.py

@@ -6,6 +6,7 @@
 import clarity_epp.export.hamilton
 import clarity_epp.export.illumina
 import clarity_epp.export.labels
+import clarity_epp.export.magnis
 import clarity_epp.export.manual_pipetting
 import clarity_epp.export.merge
 import clarity_epp.export.ped

clarity_epp/export/caliper.py

@@ -33,12 +33,7 @@ def samplesheet_normalise(lims, process_id, output_file):
         parent_process_barcode = parent_process_barcode_manual
 
     # Get all Qubit and Tecan Spark QC types
-    qc_process_types = []
-    for process_type in lims.get_process_types():
-        if 'Dx Qubit QC' in process_type.name:
-            qc_process_types.append(process_type.name)
-        elif 'Dx Tecan Spark 10M QC' in process_type.name:
-            qc_process_types.append(process_type.name)
+    qc_process_types = clarity_epp.export.utils.get_process_types(lims, ['Dx Qubit QC', 'Dx Tecan Spark 10M QC'])
 
     # Get all unique input artifact ids
     parent_processes = list(set(parent_processes))
@@ -49,8 +44,8 @@ def samplesheet_normalise(lims, process_id, output_file):
     input_artifact_ids = list(set(input_artifact_ids))
 
     # Get unique QC processes for input artifacts
-    qc_processes = list(set(lims.get_processes(type=[qc_process_types], inputartifactlimsid=input_artifact_ids)))
-    
+    qc_processes = list(set(lims.get_processes(type=qc_process_types, inputartifactlimsid=input_artifact_ids)))
+
     samples_measurements_qubit = {}
     sample_concentration = {}
     samples_measurements_tecan = {}
@@ -69,7 +64,7 @@ def samplesheet_normalise(lims, process_id, output_file):
     volume_DNA = {}
     volume_H2O = {}
     conc_measured = {}
-    output_ng = process.udf['Output genormaliseerd gDNA']
+    output_ng = float(process.udf['Output genormaliseerd gDNA'])
     conc = {}
     output_ul = process.udf['Eindvolume (ul) genormaliseerd gDNA']
     output_plate_barcode = process.output_containers()[0].name
@@ -86,7 +81,7 @@ def samplesheet_normalise(lims, process_id, output_file):
                         samples_measurements_qubit[sample].append(measurement)
                     elif sample not in sample_concentration:
                         sample_concentration[sample] = 'geen'
-        
+
         elif 'Dx Tecan Spark 10M QC' in qc_process.type.name:
             for artifact in qc_process.all_outputs():
                 sample = artifact.samples[0].name
@@ -95,10 +90,10 @@ def samplesheet_normalise(lims, process_id, output_file):
                         measurement = artifact.udf['Dx Conc. goedgekeurde meting (ng/ul)']
                         if sample not in samples_measurements_tecan:
                             samples_measurements_tecan[sample] = []
-                        samples_measurements_tecan[sample].append(measurement)                            
+                        samples_measurements_tecan[sample].append(measurement)
                     elif sample not in sample_concentration:
                         sample_concentration[sample] = 'geen'
-    
+
     for qc_process in qc_processes:
         for artifact in qc_process.all_outputs():
             sample = artifact.samples[0].name
@@ -107,43 +102,43 @@ def samplesheet_normalise(lims, process_id, output_file):
                     machine = 'Tecan'
                 elif 'Dx Qubit QC' in qc_process.type.name and 'Dx Conc. goedgekeurde meting (ng/ul)' in artifact.udf:
                     machine = 'Qubit'
-            
-            if sample not in sample_concentration or machine == 'Qubit':
-                if sample in samples_measurements_tecan or sample in samples_measurements_qubit:
-                    if machine == 'Tecan':
-                        sample_measurements = samples_measurements_tecan[sample]
-                    elif machine == 'Qubit':
-                        sample_measurements = samples_measurements_qubit[sample]
-                    sample_measurements_average = sum(sample_measurements) / float(len(sample_measurements))
-                    sample_concentration[sample] = sample_measurements_average
-
-    for placement, artifact in process.output_containers()[0].placements.iteritems():
+
+                if sample not in sample_concentration or machine == 'Qubit':
+                    if sample in samples_measurements_tecan or sample in samples_measurements_qubit:
+                        if machine == 'Tecan':
+                            sample_measurements = samples_measurements_tecan[sample]
+                        elif machine == 'Qubit':
+                            sample_measurements = samples_measurements_qubit[sample]
+                        sample_measurements_average = sum(sample_measurements) / float(len(sample_measurements))
+                        sample_concentration[sample] = sample_measurements_average
+
+    for placement, artifact in process.output_containers()[0].placements.items():
         placement = ''.join(placement.split(':'))
         filled_wells.append(placement)
         if order[placement] > last_filled_well:
             last_filled_well = order[placement]
 
     for x in range(0, last_filled_well):
-        for well, number in order.iteritems():
+        for well, number in order.items():
             if number == x:
                 placement = well
         monsternummer[placement] = 'Leeg'
         volume_DNA[placement] = 0
         volume_H2O[placement] = 0
 
-    for placement, artifact in process.output_containers()[0].placements.iteritems():
+    for placement, artifact in process.output_containers()[0].placements.items():
         sample = artifact.samples[0].name
         if sample in process_samples:
             placement = ''.join(placement.split(':'))
             monsternummer[placement] = sample
             conc_measured[placement] = sample_concentration[sample]
             if conc_measured[placement] != 'geen':
-                if output_ng/conc_measured[placement] > 50:
-                    conc[placement] = output_ng/50
+                if output_ng/conc_measured[placement] > 100:
+                    conc[placement] = output_ng/100
                 else:
                     conc[placement] = conc_measured[placement]
-                volume_DNA[placement] = int(round(float(output_ng)/conc[placement]))
-                volume_H2O[placement] = output_ul-int(round(float(output_ng)/conc[placement]))
+                volume_DNA[placement] = int(round(output_ng/conc[placement]))
+                volume_H2O[placement] = output_ul-int(round(output_ng/conc[placement]))
 
     for well in clarity_epp.export.utils.sort_96_well_plate(monsternummer.keys()):
         output_file.write(
@@ -156,3 +151,42 @@ def samplesheet_normalise(lims, process_id, output_file):
                 volume_H2O=volume_H2O[well]
             )
         )
+
+
+def samplesheet_dilute(lims, process_id, output_file):
+    """Create Caliper samplesheet for diluting samples."""
+    output_file.write(
+        'Monsternummer\tPlate_Id_input\tWell\tPlate_Id_output\tPipetteervolume DNA (ul)\tPipetteervolume H2O (ul)\n'
+    )
+    process = Process(lims, id=process_id)
+
+    output = {}  # save output data to dict, to be able to sort on well.
+    nM_pool = process.udf['Dx Pool verdunning (nM)']
+    output_ul = process.udf['Eindvolume (ul)']
+
+    for input_artifact in process.all_inputs():
+        output_artifact = process.outputs_per_input(input_artifact.id, Analyte=True)[0]
+
+        # Get QC stats
+        size = float(input_artifact.udf['Dx Fragmentlengte (bp)'])
+        concentration = float(input_artifact.udf['Dx Concentratie fluorescentie (ng/ul)'])
+
+        # Calculate dilution
+        nM_dna = (concentration * 1000 * (1/660.0) * (1/size)) * 1000
+        ul_sample = (nM_pool/nM_dna) * output_ul
+        ul_water = output_ul - ul_sample
+
+        # Store output lines by well
+        well = ''.join(input_artifact.location[1].split(':'))
+        output[well] = '{name}\t{plate_id_input}\t{well}\t{plate_id_output}\t{volume_dna:.1f}\t{volume_water:.1f}\n'.format(
+            name=input_artifact.name,
+            plate_id_input=input_artifact.location[0].name,
+            well=well,
+            plate_id_output=output_artifact.location[0].name,
+            volume_dna=ul_sample,
+            volume_water=ul_water
+        )
+
+    # Write output, sort by well
+    for well in clarity_epp.export.utils.sort_96_well_plate(output.keys()):
+        output_file.write(output[well])

clarity_epp/export/illumina.py

@@ -29,9 +29,11 @@ def get_project(projects, urgent=False):
     families = {}
     for artifact in process.all_inputs():
         for sample in artifact.samples:
-            if ('Dx Familienummer' in list(sample.udf)
-                    and 'Dx NICU Spoed' in list(sample.udf)
-                    and 'Dx Protocolomschrijving' in list(sample.udf)):
+            if (
+                'Dx Familienummer' in list(sample.udf) and
+                'Dx NICU Spoed' in list(sample.udf) and
+                'Dx Protocolomschrijving' in list(sample.udf)
+            ):
                 # Dx production sample
                 family = sample.udf['Dx Familienummer']
 
@@ -43,7 +45,7 @@ def get_project(projects, urgent=False):
                         'project_type': 'unknown_project',
                         'split_project_type': False,
                         'urgent': False,
-                        'merge': False
+                        'deviating': False  # merge, deep sequencing (5x), etc samples
                     }
 
                 # Update family information
@@ -75,12 +77,21 @@ def get_project(projects, urgent=False):
                         project_type = 'CREv2'
                         families[family]['project_type'] = project_type
                         families[family]['split_project_type'] = True
+                    elif 'elidS31285117' in sample.udf['Dx Protocolomschrijving'] and not families[family]['NICU']:
+                        project_type = 'SSv7'
+                        families[family]['project_type'] = project_type
+                        families[family]['split_project_type'] = True
 
-                    # Set urgent / merge status
+                    # Set urgent status
                     if 'Dx Spoed' in list(sample.udf) and sample.udf['Dx Spoed']:
                         families[family]['urgent'] = True
-                    if 'Dx Mergen' in list(sample.udf) and sample.udf['Dx Mergen']:
-                        families[family]['merge'] = True
+
+                    # Set deviating status, remove urgent status if deviating
+                    if (
+                        ('Dx Mergen' in list(sample.udf) and sample.udf['Dx Mergen']) or
+                        ('Dx Exoomequivalent' in list(sample.udf) and sample.udf['Dx Exoomequivalent'] > 1)
+                    ):
+                        families[family]['deviating'] = True
                         families[family]['urgent'] = False
 
             else:  # Other samples
@@ -97,7 +108,7 @@ def get_project(projects, urgent=False):
                         'project_type': family,
                         'split_project_type': False,
                         'urgent': False,
-                        'merge': False
+                        'deviating': False
                     }
 
             # Add sample to family
@@ -128,27 +139,27 @@ def get_project(projects, urgent=False):
     sample_projects = {}
     sample_sequence_names = {}
 
-    # Urgent families / samples, skip merge
-    for family in [family for family in families.values() if family['urgent'] and not family['merge']]:
+    # Urgent families / samples, skip deviating
+    for family in [family for family in families.values() if family['urgent'] and not family['deviating']]:
         family_project = get_project(project_types[family['project_type']]['projects'], urgent=True)
         for sample in family['samples']:
             sample_sequence_name = get_sequence_name(sample)
             sample_sequence_names[sample.name] = sample_sequence_name
             sample_projects[sample_sequence_name] = family_project
             project_types[family['project_type']]['projects'][family_project] += 1
 
-    # Merge families / samples
-    for family in [family for family in families.values() if family['merge']]:
+    # Deviating families / samples
+    for family in [family for family in families.values() if family['deviating']]:
         family_project = get_project(project_types[family['project_type']]['projects'])
         for sample in family['samples']:
             sample_sequence_name = get_sequence_name(sample)
             sample_sequence_names[sample.name] = sample_sequence_name
             sample_projects[sample_sequence_name] = family_project
             project_types[family['project_type']]['projects'][family_project] += 1
 
-    # Non urgent and non merge families / samples
-    non_urgent_families = [family for family in families.values() if not family['urgent'] and not family['merge']]
-    for family in sorted(non_urgent_families, key=lambda fam: (len(fam['samples'])), reverse=True):
+    # Non urgent and non deviating families / samples
+    normal_families = [family for family in families.values() if not family['urgent'] and not family['deviating']]
+    for family in sorted(normal_families, key=lambda fam: (len(fam['samples'])), reverse=True):
         family_project = get_project(project_types[family['project_type']]['projects'])
         for sample in family['samples']:
             sample_sequence_name = get_sequence_name(sample)