Few lambda fixes

umccr · Nov 21, 2024 · 2026844 · 2026844
1 parent 881bf97
commit 2026844
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 12 deletions.
diff --git a/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts b/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts
@@ -136,7 +136,8 @@ export class OraCompressionIcav2PipelineManagerStack extends cdk.Stack {
         environment: {
           ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName,
         },
-        timeout: Duration.seconds(60),
+        memorySize: 1024,
+        timeout: Duration.seconds(300),
       }
     );
     const findAllFastqPairsInInstrumentRunLambdaObj = new PythonFunction(
@@ -151,7 +152,8 @@ export class OraCompressionIcav2PipelineManagerStack extends cdk.Stack {
         environment: {
           ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName,
         },
-        timeout: Duration.seconds(60),
+        memorySize: 1024,
+        timeout: Duration.seconds(300),
       }
     );
 
@@ -257,7 +259,8 @@ export class OraCompressionIcav2PipelineManagerStack extends cdk.Stack {
         environment: {
           ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName,
         },
-        timeout: Duration.seconds(60),
+        memorySize: 1024,
+        timeout: Duration.seconds(300),
       }
     );
 
@@ -401,7 +404,8 @@ export class OraCompressionIcav2PipelineManagerStack extends cdk.Stack {
       environment: {
         ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName,
       },
-      timeout: Duration.seconds(60),
+      memorySize: 1024,
+      timeout: Duration.seconds(300),
     });
 
     // Give the lambda function access to the secret

diff --git a/...mbdas/find_all_fastq_pairs_in_instrument_run_py/find_all_fastq_pairs_in_instrument_run.py b/...mbdas/find_all_fastq_pairs_in_instrument_run_py/find_all_fastq_pairs_in_instrument_run.py
@@ -19,6 +19,7 @@
 import boto3
 from os import environ
 import re
+import pandas as pd
 
 from wrapica.project_data import (
     find_project_data_bulk,
@@ -142,6 +143,11 @@ def handler(event, context):
             "read_2_file_uri": convert_project_data_obj_to_uri(r2_file)
         })
 
+    # Assert that the all rgid_partial are unique
+    assert \
+        len(pd.DataFrame(fastq_pair_list)['rgid_partial'].unique().tolist()) == len(fastq_pair_list), \
+        "rgid_partial are not unique"
+
     return fastq_pair_list
 
 
@@ -166,13 +172,13 @@ def handler(event, context):
 #     # [
 #     #     {
 #     #         "rgid_partial": "1.L2401526",
-#     #         "read1_file_uri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_1/L2401526/L2401526_S1_L001_R1_001.fastq.gz",
-#     #         "read2_file_uri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_1/L2401526/L2401526_S1_L001_R2_001.fastq.gz"
+#     #         "read_1_file_uri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_1/L2401526/L2401526_S1_L001_R1_001.fastq.gz",
+#     #         "read_2_file_uri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_1/L2401526/L2401526_S1_L001_R2_001.fastq.gz"
 #     #     },
 #     #     ...
 #     #     {
 #     #         "rgid_partial": "4.L2401553",
-#     #         "read1_file_uri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_4/L2401553/L2401553_S27_L004_R1_001.fastq.gz",
-#     #         "read2_file_uri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_4/L2401553/L2401553_S27_L004_R2_001.fastq.gz"
+#     #         "read_1_file_uri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_4/L2401553/L2401553_S27_L004_R1_001.fastq.gz",
+#     #         "read_2_file_uri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_4/L2401553/L2401553_S27_L004_R2_001.fastq.gz"
 #     #     }
 #     # ]
diff --git a/...ra-compression-manager/lambdas/find_all_fastq_pairs_in_instrument_run_py/requirements.txt b/...ra-compression-manager/lambdas/find_all_fastq_pairs_in_instrument_run_py/requirements.txt
@@ -1 +1,2 @@
 wrapica==2.27.1.post20240830140737
+pandas>=2
diff --git a/...nd_all_v2_samplesheets_in_instrument_run_py/find_all_v2_samplesheets_in_instrument_run.py b/...nd_all_v2_samplesheets_in_instrument_run_py/find_all_v2_samplesheets_in_instrument_run.py
@@ -19,6 +19,7 @@
 import boto3
 from typing import List, Dict
 from os import environ
+import pandas as pd
 
 from wrapica.project_data import (
     find_project_data_bulk,
@@ -122,12 +123,13 @@ def handler(event, context):
         for bclconvert_iter_ in samplesheet_data_dict["bclconvert_data"]:
             rgids_list.append(
                 {
-                    "rgid": f"{bclconvert_iter_['index']}.{bclconvert_iter_['index2']}.{bclconvert_iter_['lane']}.{bclconvert_iter_['sample_id']}.{instrument_run_id}",
-                    "rgid_partial": f"{bclconvert_iter_['lane']}.{bclconvert_iter_['sample_id']}",
+                    "rgid": f"{bclconvert_iter_['index']}.{bclconvert_iter_['index2']}.{bclconvert_iter_.get('lane', 1)}.{bclconvert_iter_['sample_id']}.{instrument_run_id}",
+                    "rgid_partial": f"{bclconvert_iter_.get('lane', 1)}.{bclconvert_iter_['sample_id']}",
                 }
             )
 
-    return rgids_list
+    # Convert rgids_list to pandas dataframe and drop duplicates
+    return pd.DataFrame(rgids_list).drop_duplicates().to_dict(orient='records')
 
 
 # if __name__ == "__main__":
@@ -159,3 +161,38 @@ def handler(event, context):
 #     #         "rgid_partial": "4.L2401553"
 #     #     }
 #     # ]
+
+
+# if __name__ == "__main__":
+#     # Test the handler function
+#     import json
+#     environ["AWS_PROFILE"] = "umccr-production"
+#     environ["ICAV2_ACCESS_TOKEN_SECRET_ID"] = "ICAv2JWTKey-umccr-prod-service-production"
+#     print(
+#         json.dumps(
+#             handler(
+#                 {
+#                     "instrument_run_folder_uri": "icav2://data-migration/primary_data/210701_A01052_0055_AH7KWGDSX2/202201052f795bab/",
+#                     "instrument_run_id": "210701_A01052_0055_AH7KWGDSX2"
+#                 },
+#                 None,
+#             ),
+#             indent=4
+#         )
+#     )
+#
+#     # [
+#     #     {
+#     #         "rgid": "TACCGAGG.AGTTCAGG.1.PRJ210449_L2100607.210701_A01052_0055_AH7KWGDSX2",
+#     #         "rgid_partial": "1.PRJ210449_L2100607"
+#     #     },
+#     #     {
+#     #         "rgid": "CGTTAGAA.GACCTGAA.1.PRJ210450_L2100608.210701_A01052_0055_AH7KWGDSX2",
+#     #         "rgid_partial": "1.PRJ210450_L2100608"
+#     #     },
+#     #     ...
+#     #     {
+#     #         "rgid": "TTACAGGA.GCTTGTCA.1.MDX210166_L2100720.210701_A01052_0055_AH7KWGDSX2",
+#     #         "rgid_partial": "1.MDX210166_L2100720"
+#     #     }
+#     # ]
diff --git a/...ompression-manager/lambdas/find_all_v2_samplesheets_in_instrument_run_py/requirements.txt b/...ompression-manager/lambdas/find_all_v2_samplesheets_in_instrument_run_py/requirements.txt
@@ -1,2 +1,3 @@
 wrapica==2.27.1.post20240830140737
-v2-samplesheet-maker==4.2.4.post20241110133537
+v2-samplesheet-maker==4.2.4.post20241110133537
+pandas>=2