From 07a114c0b091eab8314e33a9a502d69b3c3e43fb Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Fri, 25 Oct 2024 12:11:53 +1100 Subject: [PATCH] Fixes for nf pipeline * Updated stacky names from mock to stacky * ICAV2_ACCESS_TOKEN_SECRET_ID an optional env var for fill placeholders lambda * Add tagResource policy to nf stack * Fix launch nextflow pipeline sfn * Fix rule for default Batch event * Fix batch parameter path link lambda * Fix override resource requirements * Add portal run id to engine parameters and to batch parameters * Add orcabus: true to parameters to allow v1 portal to filter out events * Fix url extension function * Fix rna pipeline list flatten --- config/constants.ts | 70 ++++++++++---- config/stacks/stackyMcStackFace.ts | 96 +++++++++---------- ...fill_placeholders_in_event_payload_data.py | 6 +- .../sfn-nf-batch-ready-event-handler/index.ts | 2 +- ...launch_nextflow_pipeline_template.asl.json | 19 ++-- .../index.ts | 20 ++-- .../launch_bs_runs_upload_tes/handler.py | 32 +++---- .../deploy/index.ts | 4 +- .../generate_batch_parameters.py | 77 ++++++++++++++- .../sash-pipeline-manager/deploy/index.ts | 4 +- .../generate_batch_parameters.py | 9 +- .../launch-oncoanalyser-ready-events/index.ts | 6 +- .../generate_dna_payload.py | 62 +++++++++++- .../generate_rna_payload.py | 57 ++++++++++- ...coanalyser_rna_ready_sfn_template.asl.json | 5 +- .../get_pieriandx_data_files.py | 62 ++++++------ .../generate_sash_payload.py | 4 +- 17 files changed, 374 insertions(+), 161 deletions(-) diff --git a/config/constants.ts b/config/constants.ts index c52f7fc7a..f7abdc484 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -711,6 +711,7 @@ export const oncoanalyserBatchJobDefinitionArn: Record = { [AppStage.PROD]: `arn:aws:batch:${region}:${accountIdAlias.prod}:job-definition/Nextflow-oncoanalyser`, // pragma: allowlist secret }; export const oncoanalyserStateMachinePrefix = 'oncoanalyser'; + export const oncoanalyserPipelineVersionSSMParameterPath = '/nextflow_stack/oncoanalyser/pipeline_version_tag'; @@ -750,28 +751,55 @@ export const sashBatchJobDefinitionArn: Record = { export const sashStateMachinePrefix = 'sash'; export const sashPipelineVersionSSMParameterPath = '/nextflow_stack/sash/pipeline_version_tag'; -// Mock Stack -export const mockEventBusName = eventBusName; -export const mockInstrumentRunTableName = 'stacky-instrument-run-table'; -export const mockInputMakerTableName = 'stacky-input-maker-table'; -export const mockCttsov2InputGlueTableName = 'stacky-cttsov2-workflow-glue-table'; -export const mockWgtsQcGlueTableName = 'stacky-wgts-qc-glue-table'; -export const mockTnGlueTableName = 'stacky-tn-glue-table'; -export const mockWtsGlueTableName = 'stacky-wts-glue-table'; -export const mockUmccriseGlueTableName = 'stacky-umccrise-glue-table'; -export const mockRnasumGlueTableName = 'stacky-rnasum-glue-table'; -export const mockPierianDxGlueTableName = 'stacky-pieriandx-glue-table'; -export const mockOncoanalyserGlueTableName = 'stacky-oncoanalyser-glue-table'; -export const mockOncoanalyserBothSashGlueTableName = 'stacky-oncoanalyser-both-sash-glue-table'; -export const mockWorkflowManagerTableName = 'stacky-workflow-manager-table'; - +// Stacky Stack +export const stackyEventBusName = eventBusName; +export const stackyInstrumentRunTableName = 'stacky-instrument-run-table'; +export const stackyInputMakerTableName = 'stacky-input-maker-table'; +export const stackyCttsov2InputGlueTableName = 'stacky-cttsov2-workflow-glue-table'; +export const stackyWgtsQcGlueTableName = 'stacky-wgts-qc-glue-table'; +export const stackyTnGlueTableName = 'stacky-tn-glue-table'; +export const stackyWtsGlueTableName = 'stacky-wts-glue-table'; +export const stackyUmccriseGlueTableName = 'stacky-umccrise-glue-table'; +export const stackyRnasumGlueTableName = 'stacky-rnasum-glue-table'; +export const stackyPierianDxGlueTableName = 'stacky-pieriandx-glue-table'; +export const stackyOncoanalyserGlueTableName = 'stacky-oncoanalyser-glue-table'; +export const stackyOncoanalyserBothSashGlueTableName = 'stacky-oncoanalyser-both-sash-glue-table'; +export const stackyWorkflowManagerTableName = 'stacky-workflow-manager-table'; + +// dev // { // "project_id":"ea19a3f5-ec7c-4940-a474-c31cd91dbad4", // "project_name": "development" // } -export const mockIcav2ProjectIdSsmParameterName = '/orcabus/stacky/icav2_project_id_and_name_json'; -export const mockPrimaryOutputUriSsmParameterName = '/orcabus/stacky/primary_output_uri'; // icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/primary/__instrument_run_id__/__portal_run_id__/ -export const mockAnalysisOutputUriSsmParameterName = '/orcabus/stacky/analysis_output_uri'; // icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/analysis/__workflow_name__/__portal_run_id__/ -export const mockAnalysisLogsUriSsmParameterName = '/orcabus/stacky/analysis_logs_uri'; // icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/logs/__workflow_name__/__portal_run_id__/ - -export const mockAnalysisCacheUriSsmParameterName = '/orcabus/stacky/analysis_cache_uri'; // icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/cache/__workflow_name__/__portal_run_id__/ +// stg +// { +// "project_id": "157b9e78-b2e1-45a7-bfcd-691159995f7c", +// "project_name": "staging" +// } +// prod +// { +// "project_id": "eba5c946-1677-441d-bbce-6a11baadecbb", +// "project_name": "production" +// } +export const stackyIcav2ProjectIdSsmParameterName = + '/orcabus/stacky/icav2_project_id_and_name_json'; + +// dev: s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/__instrument_run_id__/__portal_run_id__/ +// stg: s3://pipeline-stg-cache-503977275616-ap-southeast-2/byob-icav2/staging/primary/__instrument_run_id__/__portal_run_id__/ +// prod: s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/primary/__instrument_run_id__/__portal_run_id__/ +export const stackyPrimaryOutputUriSsmParameterName = '/orcabus/stacky/primary_output_uri'; + +// dev: s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/__workflow_name__/__portal_run_id__/ +// stg: s3://pipeline-stg-cache-503977275616-ap-southeast-2/byob-icav2/staging/analysis/__workflow_name__/__portal_run_id__/ +// prod: s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/__workflow_name__/__portal_run_id__/ +export const stackyAnalysisOutputUriSsmParameterName = '/orcabus/stacky/analysis_output_uri'; + +// dev: s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/logs/__workflow_name__/__portal_run_id__/ +// stg: s3://pipeline-stg-cache-503977275616-ap-southeast-2/byob-icav2/staging/logs/__workflow_name__/__portal_run_id__/ +// prod: s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/logs/__workflow_name__/__portal_run_id__/ +export const stackyAnalysisLogsUriSsmParameterName = '/orcabus/stacky/analysis_logs_uri'; + +// dev: s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/cache/__workflow_name__/__portal_run_id__/ +// stg: s3://pipeline-stg-cache-503977275616-ap-southeast-2/byob-icav2/staging/cache/__workflow_name__/__portal_run_id__/ +// prod: s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/cache/__workflow_name__/__portal_run_id__/ +export const stackyAnalysisCacheUriSsmParameterName = '/orcabus/stacky/analysis_cache_uri'; diff --git a/config/stacks/stackyMcStackFace.ts b/config/stacks/stackyMcStackFace.ts index ebc198024..88580a43e 100644 --- a/config/stacks/stackyMcStackFace.ts +++ b/config/stacks/stackyMcStackFace.ts @@ -1,26 +1,26 @@ import { - mockPrimaryOutputUriSsmParameterName, - mockAnalysisCacheUriSsmParameterName, - mockAnalysisLogsUriSsmParameterName, - mockAnalysisOutputUriSsmParameterName, - mockEventBusName, - mockIcav2ProjectIdSsmParameterName, - mockInputMakerTableName, - mockInstrumentRunTableName, - mockWorkflowManagerTableName, - mockCttsov2InputGlueTableName, + stackyPrimaryOutputUriSsmParameterName, + stackyAnalysisCacheUriSsmParameterName, + stackyAnalysisLogsUriSsmParameterName, + stackyAnalysisOutputUriSsmParameterName, + stackyEventBusName, + stackyIcav2ProjectIdSsmParameterName, + stackyInputMakerTableName, + stackyInstrumentRunTableName, + stackyWorkflowManagerTableName, + stackyCttsov2InputGlueTableName, icav2AccessTokenSecretName, - mockWgtsQcGlueTableName, + stackyWgtsQcGlueTableName, AppStage, - mockTnGlueTableName, - mockWtsGlueTableName, - mockUmccriseGlueTableName, - mockRnasumGlueTableName, - mockPierianDxGlueTableName, + stackyTnGlueTableName, + stackyWtsGlueTableName, + stackyUmccriseGlueTableName, + stackyRnasumGlueTableName, + stackyPierianDxGlueTableName, pieriandxProjectInfoSsmParameterPath, redcapLambdaFunctionName, - mockOncoanalyserGlueTableName, - mockOncoanalyserBothSashGlueTableName, + stackyOncoanalyserGlueTableName, + stackyOncoanalyserBothSashGlueTableName, } from '../constants'; import { GlueStackConfig } from '../../lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs'; import { StackyStatefulTablesConfig } from '../../lib/workload/stateful/stacks/stacky-mcstackface-dynamodb'; @@ -28,33 +28,33 @@ import { StackyStatefulTablesConfig } from '../../lib/workload/stateful/stacks/s export const getGlueStackProps = (stage: AppStage): GlueStackConfig => { return { /* Events */ - eventBusName: mockEventBusName, + eventBusName: stackyEventBusName, /* Tables */ - inputMakerTableName: mockInputMakerTableName, - instrumentRunTableName: mockInstrumentRunTableName, - workflowManagerTableName: mockWorkflowManagerTableName, - cttsov2GlueTableName: mockCttsov2InputGlueTableName, - wgtsQcGlueTableName: mockWgtsQcGlueTableName, - tnGlueTableName: mockTnGlueTableName, - wtsGlueTableName: mockWtsGlueTableName, - umccriseGlueTableName: mockUmccriseGlueTableName, - rnasumGlueTableName: mockRnasumGlueTableName, - pieriandxGlueTableName: mockPierianDxGlueTableName, - oncoanalyserGlueTableName: mockOncoanalyserGlueTableName, - oncoanalyserBothSashGlueTableName: mockOncoanalyserBothSashGlueTableName, + inputMakerTableName: stackyInputMakerTableName, + instrumentRunTableName: stackyInstrumentRunTableName, + workflowManagerTableName: stackyWorkflowManagerTableName, + cttsov2GlueTableName: stackyCttsov2InputGlueTableName, + wgtsQcGlueTableName: stackyWgtsQcGlueTableName, + tnGlueTableName: stackyTnGlueTableName, + wtsGlueTableName: stackyWtsGlueTableName, + umccriseGlueTableName: stackyUmccriseGlueTableName, + rnasumGlueTableName: stackyRnasumGlueTableName, + pieriandxGlueTableName: stackyPierianDxGlueTableName, + oncoanalyserGlueTableName: stackyOncoanalyserGlueTableName, + oncoanalyserBothSashGlueTableName: stackyOncoanalyserBothSashGlueTableName, /* SSM Parameters */ - analysisCacheUriSsmParameterName: mockAnalysisCacheUriSsmParameterName, - analysisOutputUriSsmParameterName: mockAnalysisOutputUriSsmParameterName, - icav2ProjectIdSsmParameterName: mockIcav2ProjectIdSsmParameterName, - analysisLogsUriSsmParameterName: mockAnalysisLogsUriSsmParameterName, + analysisCacheUriSsmParameterName: stackyAnalysisCacheUriSsmParameterName, + analysisOutputUriSsmParameterName: stackyAnalysisOutputUriSsmParameterName, + icav2ProjectIdSsmParameterName: stackyIcav2ProjectIdSsmParameterName, + analysisLogsUriSsmParameterName: stackyAnalysisLogsUriSsmParameterName, /* Secrets */ icav2AccessTokenSecretName: icav2AccessTokenSecretName[stage], /* BSSH SSM Parameters */ - bsshOutputFastqCopyUriSsmParameterName: mockPrimaryOutputUriSsmParameterName, + bsshOutputFastqCopyUriSsmParameterName: stackyPrimaryOutputUriSsmParameterName, /* PierianDx SSM Parameters */ pieriandxProjectInfoSsmParameterPath: pieriandxProjectInfoSsmParameterPath, @@ -64,17 +64,17 @@ export const getGlueStackProps = (stage: AppStage): GlueStackConfig => { export const getStatefulGlueStackProps = (): StackyStatefulTablesConfig => { return { - dynamodbInstrumentRunManagerTableName: mockInstrumentRunTableName, - dynamodbWorkflowManagerTableName: mockWorkflowManagerTableName, - dynamodbInputGlueTableName: mockInputMakerTableName, - dynamodbCttsov2WorkflowGlueTableName: mockCttsov2InputGlueTableName, - dynamodbWgtsQcGlueTableName: mockWgtsQcGlueTableName, - dynamodbTnGlueTableName: mockTnGlueTableName, - dynamodbWtsGlueTableName: mockWtsGlueTableName, - dynamodbUmccriseGlueTableName: mockUmccriseGlueTableName, - dynamodbRnasumGlueTableName: mockRnasumGlueTableName, - dynamodbPieriandxGlueTableName: mockPierianDxGlueTableName, - dynamodbOncoanalyserGlueTableName: mockOncoanalyserGlueTableName, - dynamodbOncoanalyserBothSashGlueTableName: mockOncoanalyserBothSashGlueTableName, + dynamodbInstrumentRunManagerTableName: stackyInstrumentRunTableName, + dynamodbWorkflowManagerTableName: stackyWorkflowManagerTableName, + dynamodbInputGlueTableName: stackyInputMakerTableName, + dynamodbCttsov2WorkflowGlueTableName: stackyCttsov2InputGlueTableName, + dynamodbWgtsQcGlueTableName: stackyWgtsQcGlueTableName, + dynamodbTnGlueTableName: stackyTnGlueTableName, + dynamodbWtsGlueTableName: stackyWtsGlueTableName, + dynamodbUmccriseGlueTableName: stackyUmccriseGlueTableName, + dynamodbRnasumGlueTableName: stackyRnasumGlueTableName, + dynamodbPieriandxGlueTableName: stackyPierianDxGlueTableName, + dynamodbOncoanalyserGlueTableName: stackyOncoanalyserGlueTableName, + dynamodbOncoanalyserBothSashGlueTableName: stackyOncoanalyserBothSashGlueTableName, }; }; diff --git a/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/fill_placeholders_in_event_payload_data_py/fill_placeholders_in_event_payload_data.py b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/fill_placeholders_in_event_payload_data_py/fill_placeholders_in_event_payload_data.py index 6548c17c2..b614eefc3 100644 --- a/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/fill_placeholders_in_event_payload_data_py/fill_placeholders_in_event_payload_data.py +++ b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/fill_placeholders_in_event_payload_data_py/fill_placeholders_in_event_payload_data.py @@ -33,7 +33,6 @@ import boto3 import typing - if typing.TYPE_CHECKING: from mypy_boto3_ssm import SSMClient from mypy_boto3_secretsmanager import SecretsManagerClient @@ -191,8 +190,9 @@ def set_icav2_env_vars(): def handler(event, context): - # Set env vars - set_icav2_env_vars() + if environ.get('ICAV2_ACCESS_TOKEN_SECRET_ID', None) is not None: + # Set env vars + set_icav2_env_vars() # Get the portal run id from the event portal_run_id: str = event.get('portal_run_id', None) diff --git a/lib/workload/components/sfn-nf-batch-ready-event-handler/index.ts b/lib/workload/components/sfn-nf-batch-ready-event-handler/index.ts index 0b7dd5580..9e1eda206 100644 --- a/lib/workload/components/sfn-nf-batch-ready-event-handler/index.ts +++ b/lib/workload/components/sfn-nf-batch-ready-event-handler/index.ts @@ -111,7 +111,7 @@ export class WfmWorkflowStateChangeNfBatchReadyEventHandlerConstruct extends Con const submitJobPolicy = new iam.Policy(this, 'submitJobPolicy', { statements: [ new iam.PolicyStatement({ - actions: ['batch:SubmitJob'], + actions: ['batch:SubmitJob', 'batch:TagResource'], resources: [ props.batchJobDefinitionObj.jobDefinitionArn, props.batchJobQueueObj.jobQueueArn, diff --git a/lib/workload/components/sfn-nf-batch-ready-event-handler/step_functions_templates/launch_nextflow_pipeline_template.asl.json b/lib/workload/components/sfn-nf-batch-ready-event-handler/step_functions_templates/launch_nextflow_pipeline_template.asl.json index 70ec06924..09e8b7c61 100644 --- a/lib/workload/components/sfn-nf-batch-ready-event-handler/step_functions_templates/launch_nextflow_pipeline_template.asl.json +++ b/lib/workload/components/sfn-nf-batch-ready-event-handler/step_functions_templates/launch_nextflow_pipeline_template.asl.json @@ -54,14 +54,14 @@ "Get Default Pipeline Version": { "Type": "Task", "Parameters": { - "Name": "__pipeline_version_ssm_path__" + "Name": "${__pipeline_version_ssm_path__}" }, "Resource": "arn:aws:states:::aws-sdk:ssm:getParameter", - "Next": "Generate Batch Submission Job", - "ResultPath": "$.get_default_pipeline_version_step", "ResultSelector": { - "default_pipeline_version.$": "$.Parameter.String" - } + "default_pipeline_version.$": "$.Parameter.Value" + }, + "ResultPath": "$.get_default_pipeline_version_step", + "Next": "Generate Batch Submission Job" }, "Generate Batch Submission Job": { "Type": "Task", @@ -71,7 +71,8 @@ "Payload": { "inputs.$": "$.workflow_inputs.payload.data.inputs", "engine_parameters.$": "$.workflow_inputs.payload.data.engineParameters", - "default_pipeline_version.$": "$.get_default_pipeline_version_step.default_pipeline_version" + "default_pipeline_version.$": "$.get_default_pipeline_version_step.default_pipeline_version", + "portal_run_id.$": "$.workflow_inputs.portalRunId" } }, "Retry": [ @@ -99,7 +100,7 @@ "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob", "Parameters": { - "JobName": "$.workflow_inputs.workflowRunName", + "JobName.$": "$.workflow_inputs.workflowRunName", "JobDefinition": "${__job_definition_arn__}", "JobQueue": "${__job_queue_name__}", "ContainerOverrides": { @@ -111,7 +112,7 @@ "Parameters.$": "$.get_batch_submission_step.parameters" }, "ResultSelector": { - "jobId.$": "$.Output.jobId" + "jobId.$": "$.JobId" }, "ResultPath": "$.submit_oncoanalyser_job_step", "Next": "Append Job Id to Engine Parameters" @@ -136,7 +137,7 @@ "UpdateExpression": "SET engine_parameters = :engine_parameters", "ExpressionAttributeValues": { ":engine_parameters": { - "S.$": "States.StringToJson($.merge_engine_parameters_step.engine_parameters)" + "S.$": "States.JsonToString($.merge_engine_parameters_step.engine_parameters)" } } }, diff --git a/lib/workload/components/sfn-nf-batch-state-change-event-handler/index.ts b/lib/workload/components/sfn-nf-batch-state-change-event-handler/index.ts index f99a7b312..aaf82c5a2 100644 --- a/lib/workload/components/sfn-nf-batch-state-change-event-handler/index.ts +++ b/lib/workload/components/sfn-nf-batch-state-change-event-handler/index.ts @@ -37,11 +37,13 @@ export interface WfmWorkflowStateChangeNfBatchStateChangeEventHandlerConstructPr export class WfmWorkflowStateChangeNfBatchStateChangeEventHandlerConstruct extends Construct { public readonly stateMachineObj: sfn.StateMachine; private readonly globals = { - defaultEventBusName: 'default', eventStatus: 'SUBMITTED', portalRunTablePartitionName: 'portal_run_id', eventDetailType: 'WorkflowRunStateChange', serviceVersion: '2024.10.17', + batchEventBusName: 'default', + batchEventSource: 'aws.batch', + batchEventDetailType: 'Batch Job State Change', }; constructor( @@ -56,7 +58,7 @@ export class WfmWorkflowStateChangeNfBatchStateChangeEventHandlerConstruct exten const defaultEventBus = events.EventBus.fromEventBusName( this, 'default-event-bus', - this.globals.defaultEventBusName + this.globals.batchEventBusName ); // Build state machine object @@ -94,12 +96,18 @@ export class WfmWorkflowStateChangeNfBatchStateChangeEventHandlerConstruct exten // Create a rule for this state machine const rule = new events.Rule(this, 'rule', { eventBus: defaultEventBus, - ruleName: `${props.stateMachinePrefix}-wrsc-rule`, + ruleName: `${props.stateMachinePrefix}-batch-state-change-rule`, eventPattern: { - source: [props.triggerLaunchSource], - detailType: [props.detailType], + source: [this.globals.batchEventSource], + detailType: [this.globals.batchEventDetailType], detail: { - jobDefinition: [{ 'equals-ignore-case': props.batchJobDefinitionObj.jobDefinitionArn }], + // Has a trailing ':1' at the end of the job definition object + jobDefinition: [ + { prefix: { 'equals-ignore-case': props.batchJobDefinitionObj.jobDefinitionArn } }, + ], + parameters: { + orcabus: [{ exists: true }], + }, }, }, }); diff --git a/lib/workload/stateless/stacks/bs-runs-upload-manager/lambdas/launch_bs_runs_upload_tes/handler.py b/lib/workload/stateless/stacks/bs-runs-upload-manager/lambdas/launch_bs_runs_upload_tes/handler.py index 441ae6e08..321206b77 100644 --- a/lib/workload/stateless/stacks/bs-runs-upload-manager/lambdas/launch_bs_runs_upload_tes/handler.py +++ b/lib/workload/stateless/stacks/bs-runs-upload-manager/lambdas/launch_bs_runs_upload_tes/handler.py @@ -110,19 +110,19 @@ def handler(event, context): ) -if __name__ == "__main__": - import json - print( - json.dumps( - handler( - { - "gds_folder_path": "/Runs/231109_A01052_0171_BHLJW7DSX7_r.NULhvzxcSEWmqZw8QljXfQ", - "samplesheet_name": "SampleSheet.V2.1711336300.924772.csv", - "gds_volume_name": "bssh.acddbfda498038ed99fa94fe79523959", - "instrument_run_id": "231109_A01052_0171_BHLJW7DSX7" - }, - None - ), - indent=2 - ) - ) +# if __name__ == "__main__": +# import json +# print( +# json.dumps( +# handler( +# { +# "gds_folder_path": "/Runs/231109_A01052_0171_BHLJW7DSX7_r.NULhvzxcSEWmqZw8QljXfQ", +# "samplesheet_name": "SampleSheet.V2.1711336300.924772.csv", +# "gds_volume_name": "bssh.acddbfda498038ed99fa94fe79523959", +# "instrument_run_id": "231109_A01052_0171_BHLJW7DSX7" +# }, +# None +# ), +# indent=2 +# ) +# ) diff --git a/lib/workload/stateless/stacks/oncoanalyser-pipeline-manager/deploy/index.ts b/lib/workload/stateless/stacks/oncoanalyser-pipeline-manager/deploy/index.ts index b3de3517b..d15bd5b3a 100644 --- a/lib/workload/stateless/stacks/oncoanalyser-pipeline-manager/deploy/index.ts +++ b/lib/workload/stateless/stacks/oncoanalyser-pipeline-manager/deploy/index.ts @@ -93,10 +93,10 @@ export class OncoanalyserNfPipelineManagerStack extends cdk.Stack { Build lambdas */ const setBatchParametersLambdaObj = new PythonFunction(this, 'get_batch_parameters', { - entry: path.join(__dirname, '../lambdas/get_outputs_py'), + entry: path.join(__dirname, '../lambdas/generate_batch_parameters_py'), runtime: lambda.Runtime.PYTHON_3_12, architecture: lambda.Architecture.ARM_64, - index: 'get_outputs.py', + index: 'generate_batch_parameters.py', handler: 'handler', memorySize: 1024, }); diff --git a/lib/workload/stateless/stacks/oncoanalyser-pipeline-manager/lambdas/generate_batch_parameters_py/generate_batch_parameters.py b/lib/workload/stateless/stacks/oncoanalyser-pipeline-manager/lambdas/generate_batch_parameters_py/generate_batch_parameters.py index 48dfc1f7f..160dd5690 100644 --- a/lib/workload/stateless/stacks/oncoanalyser-pipeline-manager/lambdas/generate_batch_parameters_py/generate_batch_parameters.py +++ b/lib/workload/stateless/stacks/oncoanalyser-pipeline-manager/lambdas/generate_batch_parameters_py/generate_batch_parameters.py @@ -116,7 +116,7 @@ def handler(event, context): tags.update(NEXTFLOW_TAGS) # Add the portal run id as a tag - tags['PortalRunId'] = engine_parameters.get("portal_run_id") + tags['PortalRunId'] = event.get("portal_run_id") # Convert inputs and engine_parameters to snake case inputs = dict(map( @@ -138,6 +138,12 @@ def handler(event, context): engine_parameters.items() )) + # Add portal run id to engine parameters + engine_parameters.update( + { + "portal_run_id": event.get("portal_run_id") + } + ) # Pop the pipeline version from engine_parameters pipeline_version = engine_parameters.pop('pipeline_version', event.get('default_pipeline_version')) @@ -147,10 +153,10 @@ def handler(event, context): "overrides": { "resource_requirements": [ { - "type": "MEMORY", "value": "15000" + "Type": "MEMORY", "Value": "15000" }, { - "type": "VCPU", "value": "2" + "Type": "VCPU", "Value": "2" } ], "command": [ @@ -166,7 +172,7 @@ def handler(event, context): ], }, "parameters": { - "portal_run_id": engine_parameters.get("portal_run_id"), + "portal_run_id": event.get("portal_run_id"), "workflow": ANALYSIS_TYPE_TO_WORKFLOW_MAPPER.get(inputs.get("analysis_type")), "version": pipeline_version, "output": json.dumps( @@ -174,7 +180,68 @@ def handler(event, context): "output_directory": engine_parameters.get("output_results_dir") }, separators=(',', ':') - ) + ), + "orcabus": True, }, "tags": tags } + + +# if __name__ == "__main__": +# print( +# json.dumps( +# handler( +# { +# "inputs": { +# "mode": "wgts", +# "analysisType": "DNA", +# "subjectId": "SN_PMC-141", +# "tumorDnaSampleId": "L2400231", +# "normalDnaSampleId": "L2400238", +# "tumorDnaBamUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400231_dragen_somatic/L2400231_tumor.bam", +# "normalDnaBamUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400231_dragen_somatic/L2400238_normal.bam" +# }, +# "default_pipeline_version": "42ee926", +# "engine_parameters": { +# "outputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/oncoanalyser-wgts-dna/20241024e8deca09/", +# "logsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/logs/oncoanalyser-wgts-dna/20241024e8deca09/", +# "cacheUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/cache/oncoanalyser-wgts-dna/20241024e8deca09/" +# }, +# "portal_run_id": "20241024e8deca09" # pragma: allowlist secret +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "overrides": { +# # "resource_requirements": [ +# # { +# # "type": "MEMORY", +# # "value": "15000" +# # }, +# # { +# # "type": "VCPU", +# # "value": "2" +# # } +# # ], +# # "command": [ +# # "./assets/run-v2.sh", +# # "--manifest-json", +# # "{\"inputs\":{\"mode\":\"wgts\",\"analysis_type\":\"DNA\",\"subject_id\":\"SN_PMC-141\",\"tumor_dna_sample_id\":\"L2400231\",\"normal_dna_sample_id\":\"L2400238\",\"tumor_dna_bam_uri\":\"s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400231_dragen_somatic/L2400231_tumor.bam\",\"normal_dna_bam_uri\":\"s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400231_dragen_somatic/L2400238_normal.bam\"},\"engine_parameters\":{\"output_results_dir\":\"s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/oncoanalyser-wgts-dna/20241024e8deca09/\",\"logs_uri\":\"s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/logs/oncoanalyser-wgts-dna/20241024e8deca09/\",\"output_scratch_dir\":\"s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/cache/oncoanalyser-wgts-dna/20241024e8deca09/\"}}" +# # ] +# # }, +# # "parameters": { +# # "portal_run_id": "20241024e8deca09", # pragma: allowlist secret +# # "workflow": "oncoanalyser-wgts-dna", +# # "version": "42ee926", +# # "output": "{\"output_directory\":\"s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/oncoanalyser-wgts-dna/20241024e8deca09/\"}" +# # }, +# # "tags": { +# # "Stack": "NextflowStack", +# # "SubStack": "OncoanalyserStack", +# # "PortalRunId": "20241024e8deca09" # pragma: allowlist secret +# # } +# # } \ No newline at end of file diff --git a/lib/workload/stateless/stacks/sash-pipeline-manager/deploy/index.ts b/lib/workload/stateless/stacks/sash-pipeline-manager/deploy/index.ts index e29d75d09..f31269941 100644 --- a/lib/workload/stateless/stacks/sash-pipeline-manager/deploy/index.ts +++ b/lib/workload/stateless/stacks/sash-pipeline-manager/deploy/index.ts @@ -90,10 +90,10 @@ export class SashNfPipelineManagerStack extends cdk.Stack { Build lambdas */ const setBatchParametersLambdaObj = new PythonFunction(this, 'get_batch_parameters', { - entry: path.join(__dirname, '../lambdas/get_outputs_py'), + entry: path.join(__dirname, '../lambdas/generate_batch_parameters_py'), runtime: lambda.Runtime.PYTHON_3_12, architecture: lambda.Architecture.ARM_64, - index: 'get_outputs.py', + index: 'generate_batch_parameters.py', handler: 'handler', memorySize: 1024, }); diff --git a/lib/workload/stateless/stacks/sash-pipeline-manager/lambdas/generate_batch_parameters_py/generate_batch_parameters.py b/lib/workload/stateless/stacks/sash-pipeline-manager/lambdas/generate_batch_parameters_py/generate_batch_parameters.py index d4ed27d96..a78effca3 100644 --- a/lib/workload/stateless/stacks/sash-pipeline-manager/lambdas/generate_batch_parameters_py/generate_batch_parameters.py +++ b/lib/workload/stateless/stacks/sash-pipeline-manager/lambdas/generate_batch_parameters_py/generate_batch_parameters.py @@ -71,6 +71,12 @@ def handler(event, context): engine_parameters.items() )) + # Add portal run id to engine parameters + engine_parameters.update( + { + "portal_run_id": event.get("portal_run_id") + } + ) # Pop the pipeline version from engine_parameters pipeline_version = engine_parameters.pop('pipeline_version', None) @@ -107,7 +113,8 @@ def handler(event, context): "output_directory": engine_parameters.get("output_results_dir") }, separators=(',', ':') - ) + ), + "orcabus": True, }, "tags": tags } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/index.ts index 3e11bf362..4f10824d1 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/index.ts @@ -61,7 +61,7 @@ export class OncoanalyserDnaOrRnaReadyConstruct extends Construct { payloadVersion: '2024.07.23', tablePartitionName: { library: 'library', - fastqListRow: 'fastq_list_Row', + fastqListRow: 'fastq_list_row', }, }; @@ -80,8 +80,8 @@ export class OncoanalyserDnaOrRnaReadyConstruct extends Construct { triggerSource: 'orcabus.wgtsqcinputeventglue', triggerStatus: 'QC_COMPLETE', triggerDetailType: 'LibraryStateChange', - sampleType: 'wts', - workflowName: 'rnadna', + sampleType: 'WTS', + workflowName: 'oncoanalyser-wgts-rna', workflowVersion: '1.0.0', }; diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/lambdas/generate_dna_payload_py/generate_dna_payload.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/lambdas/generate_dna_payload_py/generate_dna_payload.py index e86fadf65..43976f858 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/lambdas/generate_dna_payload_py/generate_dna_payload.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/lambdas/generate_dna_payload_py/generate_dna_payload.py @@ -60,9 +60,9 @@ def join_url_paths(url: str, path_ext: str) -> str: return str( urlunparse( ( - url_obj.netloc, url_obj.scheme, - url_path, + url_obj.netloc, + str(url_path), None, None, None ) ) @@ -85,9 +85,6 @@ def handler(event, context) -> Dict: tumor_fastq_list_row_ids: List[str] = event['tumor_fastq_list_row_ids'] normal_fastq_list_row_ids: List[str] = event['normal_fastq_list_row_ids'] - # Get the bam uris by taking the library ids - - return { "input_event_data": { "mode": MODE, @@ -109,3 +106,58 @@ def handler(event, context) -> Dict: "dragenGermlineOutputS3Uri": dragen_germline_output_s3_uri, } } + + +# if __name__ == "__main__": +# import json +# print( +# json.dumps( +# handler( +# { +# "subject_id": "SN_PMC-141", +# "tumor_fastq_list_row_ids": [ +# "TCGTAGTG.CCAAGTCT.2.240229_A00130_0288_BH5HM2DSXC.L2400231", +# "TCGTAGTG.CCAAGTCT.3.240229_A00130_0288_BH5HM2DSXC.L2400231" +# ], +# "dragen_somatic_output_s3_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400231_dragen_somatic/", +# "normal_library_id": "L2400238", +# "tumor_library_id": "L2400231", +# "normal_fastq_list_row_ids": [ +# "GGAGCGTC.GCACGGAC.2.240229_A00130_0288_BH5HM2DSXC.L2400238", +# "GGAGCGTC.GCACGGAC.3.240229_A00130_0288_BH5HM2DSXC.L2400238" +# ], +# "dragen_germline_output_s3_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400238_dragen_germline/", +# "individual_id": "SBJ04659" +# }, +# None +# ), +# indent=4 +# ) +# ) +# # { +# # "input_event_data": { +# # "mode": "wgts", +# # "analysisType": "DNA", +# # "subjectId": "SN_PMC-141", +# # "tumorDnaSampleId": "L2400231", +# # "normalDnaSampleId": "L2400238", +# # "tumorDnaBamUri": "pipeline-dev-cache-503977275616-ap-southeast-2://s3/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400231_dragen_somatic/L2400231_tumor.bam", +# # "normalDnaBamUri": "pipeline-dev-cache-503977275616-ap-southeast-2://s3/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400231_dragen_somatic/L2400238_normal.bam" +# # }, +# # "event_tags": { +# # "subjectId": "SN_PMC-141", +# # "individualId": "SBJ04659", +# # "tumorLibraryId": "L2400231", +# # "normalLibraryId": "L2400238", +# # "tumorFastqListRowIds": [ +# # "TCGTAGTG.CCAAGTCT.2.240229_A00130_0288_BH5HM2DSXC.L2400231", +# # "TCGTAGTG.CCAAGTCT.3.240229_A00130_0288_BH5HM2DSXC.L2400231" +# # ], +# # "normalFastqListRowIds": [ +# # "GGAGCGTC.GCACGGAC.2.240229_A00130_0288_BH5HM2DSXC.L2400238", +# # "GGAGCGTC.GCACGGAC.3.240229_A00130_0288_BH5HM2DSXC.L2400238" +# # ], +# # "dragenSomaticOutputS3Uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400231_dragen_somatic/", +# # "dragenGermlineOutputS3Uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/tumor-normal/20241003500edb11/L2400238_dragen_germline/" +# # } +# # } \ No newline at end of file diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/lambdas/generate_rna_payload_py/generate_rna_payload.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/lambdas/generate_rna_payload_py/generate_rna_payload.py index 734a975d0..f32030f3a 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/lambdas/generate_rna_payload_py/generate_rna_payload.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/lambdas/generate_rna_payload_py/generate_rna_payload.py @@ -50,7 +50,7 @@ def handler(event, context) -> Dict: "analysisType": ANALYSIS_TYPE, "subjectId": subject_id, "tumorRnaSampleId": tumor_library_id, - "tumorRnaFastqUriList": flatten( + "tumorRnaFastqUriList": list(flatten( list( map( lambda fastq_list_row_iter_: [ @@ -58,9 +58,8 @@ def handler(event, context) -> Dict: fastq_list_row_iter_.get("read2FileUri") ], tumor_fastq_list_rows - ) ) - - ), + )) + )), }, "event_tags": { "subjectId": subject_id, @@ -69,3 +68,53 @@ def handler(event, context) -> Dict: "fastqListRowIds": tumor_fastq_list_row_ids, } } + + +# if __name__ == "__main__": +# import json +# +# print( +# json.dumps( +# handler( +# { +# "tumor_fastq_list_row_ids": [ +# "CTGGAGTA.GTTCGGTT.4.240229_A00130_0288_BH5HM2DSXC.L2400254" +# ], +# "subject_id": "218-007", +# "tumor_fastq_list_rows": [ +# { +# "rgid": "CTGGAGTA.GTTCGGTT.4.240229_A00130_0288_BH5HM2DSXC.L2400254", +# "rgsm": "L2400254", +# "rglb": "L2400254", +# "lane": 4, +# "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R1_001.fastq.gz", +# "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R2_001.fastq.gz" +# } +# ], +# "tumor_library_id": "L2400254", +# "individual_id": "SBJ04661" +# }, +# None +# ), indent=4 +# ) +# ) +# # { +# # "input_event_data": { +# # "mode": "wgts", +# # "analysisType": "RNA", +# # "subjectId": "218-007", +# # "tumorRnaSampleId": "L2400254", +# # "tumorRnaFastqUriList": [ +# # "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R1_001.fastq.gz", +# # "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R2_001.fastq.gz" +# # ] +# # }, +# # "event_tags": { +# # "subjectId": "218-007", +# # "individualId": "SBJ04661", +# # "libraryId": "L2400254", +# # "fastqListRowIds": [ +# # "CTGGAGTA.GTTCGGTT.4.240229_A00130_0288_BH5HM2DSXC.L2400254" +# # ] +# # } +# # } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/step_functions_templates/library_wts_qc_complete_to_oncoanalyser_rna_ready_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/step_functions_templates/library_wts_qc_complete_to_oncoanalyser_rna_ready_sfn_template.asl.json index 4f72c5220..dfb5d6a19 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/step_functions_templates/library_wts_qc_complete_to_oncoanalyser_rna_ready_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/handy-pal/part_3/launch-oncoanalyser-ready-events/step_functions_templates/library_wts_qc_complete_to_oncoanalyser_rna_ready_sfn_template.asl.json @@ -185,7 +185,8 @@ "tumor_library_id.$": "$.payload_data.library.libraryId", "tumor_fastq_list_rows.$": "$.get_parameters_step.tumor_fastq_list_rows", "tumor_fastq_list_row_ids.$": "$.get_parameters_step.tumor_fastq_list_row_ids", - "subject_id.$": "$.get_library_item_step.Item.subject_id.S" + "subject_id.$": "$.get_library_item_step.Item.subject_id.S", + "individual_id.$": "$.get_subject_obj_step.individual_id" } }, "Retry": [ @@ -243,7 +244,7 @@ "StatePayload": { "portal_run_id.$": "$.get_parameters_step.portal_run_id", "workflow_run_name.$": "$.get_parameters_step.workflow_run_name", - "linked_libraries": "$.generate_draft_event_payload_data_step.linked_libraries", + "linked_libraries.$": "$.generate_draft_event_payload_data_step.linked_libraries", "data_inputs.$": "$.generate_draft_event_payload_data_step.input_event_data", "data_tags.$": "$.generate_draft_event_payload_data_step.event_tags" } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/get_pieriandx_data_files.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/get_pieriandx_data_files.py index f671a684c..078bea0ab 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/get_pieriandx_data_files.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/get_pieriandx_data_files.py @@ -148,34 +148,34 @@ def handler(event, context) -> Dict[str, Dict]: } -if __name__ == "__main__": - import json - from os import environ - environ['AWS_PROFILE'] = 'umccr-development' - environ['AWS_DEFAULT_REGION'] = 'ap-southeast-2' - environ['ICAV2_ACCESS_TOKEN_SECRET_ID'] = "ICAv2JWTKey-umccr-prod-service-dev" - - print( - json.dumps( - handler( - { - "output_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/", - "sample_id": "L2400160" - }, - None - ), - indent=4 - ) - ) - - # { - # "data_files": { - # "microsatOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/DragenCaller/L2400161/L2400161.microsat_output.json", - # "tmbMetricsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/Tmb/L2400161/L2400161.tmb.metrics.csv", - # "cnvVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.cnv.vcf.gz", - # "hardFilteredVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.hard-filtered.vcf.gz", - # "fusionsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_Fusions.csv", - # "metricsOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_MetricsOutput.tsv", - # "samplesheetUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" - # } - # } +# if __name__ == "__main__": +# import json +# from os import environ +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_DEFAULT_REGION'] = 'ap-southeast-2' +# environ['ICAV2_ACCESS_TOKEN_SECRET_ID'] = "ICAv2JWTKey-umccr-prod-service-dev" +# +# print( +# json.dumps( +# handler( +# { +# "output_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/", +# "sample_id": "L2400160" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "data_files": { +# # "microsatOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/DragenCaller/L2400161/L2400161.microsat_output.json", +# # "tmbMetricsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/Tmb/L2400161/L2400161.tmb.metrics.csv", +# # "cnvVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.cnv.vcf.gz", +# # "hardFilteredVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.hard-filtered.vcf.gz", +# # "fusionsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_Fusions.csv", +# # "metricsOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_MetricsOutput.tsv", +# # "samplesheetUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" +# # } +# # } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/t-rex/part_2/oncoanalyser-dna-complete-to-sash-ready/lambdas/generate_sash_payload_py/generate_sash_payload.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/t-rex/part_2/oncoanalyser-dna-complete-to-sash-ready/lambdas/generate_sash_payload_py/generate_sash_payload.py index d611943db..f72e596b1 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/t-rex/part_2/oncoanalyser-dna-complete-to-sash-ready/lambdas/generate_sash_payload_py/generate_sash_payload.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/t-rex/part_2/oncoanalyser-dna-complete-to-sash-ready/lambdas/generate_sash_payload_py/generate_sash_payload.py @@ -52,9 +52,9 @@ def join_url_paths(url: str, path_ext: str) -> str: return str( urlunparse( ( - url_obj.netloc, url_obj.scheme, - url_path, + url_obj.netloc, + str(url_path), None, None, None ) )