From 9007fd821877ee44d6df8c7ec87ac95da4d6ae30 Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Thu, 7 Nov 2024 16:18:51 +1100 Subject: [PATCH 1/4] Setup to rearrange the outputs for the ora compression manager --- .../lambdas/set_outputs_json_py/set_outputs_json.py | 8 ++++---- .../set_compression_inputs.asl.json | 2 +- .../set_compression_outputs.asl.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/workload/stateless/stacks/ora-compression-manager/lambdas/set_outputs_json_py/set_outputs_json.py b/lib/workload/stateless/stacks/ora-compression-manager/lambdas/set_outputs_json_py/set_outputs_json.py index de5fae755..5bbc8fd99 100644 --- a/lib/workload/stateless/stacks/ora-compression-manager/lambdas/set_outputs_json_py/set_outputs_json.py +++ b/lib/workload/stateless/stacks/ora-compression-manager/lambdas/set_outputs_json_py/set_outputs_json.py @@ -76,7 +76,7 @@ def handler(events, context): # Get analysis uri analysis_uri = events.get("analysis_output_uri") - instrument_run_id = events.get("instrument_run_id") + portal_run_id = events.get("portal_run_id") # Convert analysis uri to project folder object analysis_project_data_obj = convert_uri_to_project_data_obj(analysis_uri) @@ -87,19 +87,19 @@ def handler(events, context): parent_folder_id=analysis_project_data_obj.data.id, ) - # Get multiqc directory + # Get instrument run id try: instrument_run_id_project_data_obj: ProjectData = next( filter( lambda project_data_iter: ( - project_data_iter.data.details.name == instrument_run_id and + project_data_iter.data.details.name == portal_run_id and DataType[project_data_iter.data.details.data_type] == DataType.FOLDER ), analysis_top_level_data_list ) ) except StopIteration: - raise ValueError(f"instrument run id directory '{instrument_run_id}' not found in '{analysis_uri}'") + raise ValueError(f"output directory '{portal_run_id}' not found in '{analysis_uri}'") return { "instrument_run_ora_output_uri": convert_project_data_obj_to_uri(instrument_run_id_project_data_obj, UriType.S3), diff --git a/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/set_compression_inputs.asl.json b/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/set_compression_inputs.asl.json index ba0d5ab91..82ffb6ba2 100644 --- a/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/set_compression_inputs.asl.json +++ b/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/set_compression_inputs.asl.json @@ -57,7 +57,7 @@ "input_json": { "instrument_run_directory": { "class": "Directory", - "basename.$": "$.get_input_parameters_from_event.inputs.instrumentRunId", + "basename.$": "$.portal_run_id", "location.$": "$.get_input_parameters_from_event.inputs.instrumentRunDirectoryUri" }, "ora_reference": { diff --git a/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/set_compression_outputs.asl.json b/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/set_compression_outputs.asl.json index fb11e337a..0426390ee 100644 --- a/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/set_compression_outputs.asl.json +++ b/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/set_compression_outputs.asl.json @@ -25,7 +25,7 @@ "Parameters": { "FunctionName": "${__set_outputs_json_lambda_function_arn__}", "Payload": { - "instrument_run_id.$": "$.get_db_attributes_step.ready_event_data_inputs.instrumentRunId", + "portal_run_id.$": "$.portal_run_id", "analysis_output_uri.$": "$.get_db_attributes_step.analysis_output_uri" } }, From 16769b845f6694fc51b22488ca24a709ce447683 Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Fri, 8 Nov 2024 14:38:01 +1100 Subject: [PATCH 2/4] Extend lambda timeout to two minutes --- .../stateless/stacks/ora-compression-manager/deploy/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts b/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts index 553802e0d..af472e43d 100644 --- a/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts +++ b/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts @@ -227,7 +227,7 @@ export class OraCompressionIcav2PipelineManagerStack extends cdk.Stack { environment: { ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName, }, - timeout: Duration.seconds(60), + timeout: Duration.seconds(120), memorySize: 1024, }); // Give the lambda function access to the secret From 1fb00c9c3bc869bc9118839037c061b07ac5e680 Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Fri, 8 Nov 2024 15:16:52 +1100 Subject: [PATCH 3/4] Fix output path for ora compression sfn --- .../fastq_list_row_compression_event.asl.json | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/fastq_list_row_compression_event.asl.json b/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/fastq_list_row_compression_event.asl.json index 52f34f2ec..f58ca2f21 100644 --- a/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/fastq_list_row_compression_event.asl.json +++ b/lib/workload/stateless/stacks/ora-compression-manager/step_functions_templates/fastq_list_row_compression_event.asl.json @@ -5,7 +5,6 @@ "Get fastq files from fastq list csv": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", "Parameters": { "Payload": { "output_dir_uri.$": "$.payload.data.outputs.instrumentRunOraOutputUri" From f68098d4046f63746548f9805b791012ebe41453 Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Fri, 8 Nov 2024 18:15:16 +1100 Subject: [PATCH 4/4] Fix step function substitutions --- .../stateless/stacks/ora-compression-manager/deploy/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts b/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts index af472e43d..372e7575e 100644 --- a/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts +++ b/lib/workload/stateless/stacks/ora-compression-manager/deploy/index.ts @@ -248,6 +248,8 @@ export class OraCompressionIcav2PipelineManagerStack extends cdk.Stack { ) ), definitionSubstitutions: { + __event_bus_name__: eventBusObj.eventBusName, + __detail_type__: this.globals.outputCompressionDetailType, __merge_sizes_lambda_function_arn__: setMergeSizesLambdaObj.currentVersion.functionArn, }, }