From c6d867a98d16231c979f760a7612ebe9b99245b3 Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Tue, 19 Nov 2024 22:33:23 +1100 Subject: [PATCH 1/4] Add holmes glue Still need to add index for cttso but sfn template is there ready to go --- config/constants.ts | 4 +- .../get_workflow_payload.py | 106 +++++++ .../index.ts | 94 ++++++ .../index.ts | 55 ++++ .../list_service_instances.py | 95 ++++++ .../python-lambda-metadata-mapper/index.ts | 2 +- .../python-lambda-service-discovery/index.ts | 50 +++ .../service_discovery_py/service_discovery.py | 68 ++++ .../launch-holmes-extract-event/index.ts | 3 + ...olmes_bam_extraction_sfn_template.asl.json | 239 ++++++++++++++ .../glue-constructs/kwik/index.ts | 21 ++ .../launch-holmes-extract-event/index.ts | 244 +++++++++++++++ ...lmes_extract_wrapper_sfn_template.asl.json | 291 ++++++++++++++++++ 13 files changed, 1269 insertions(+), 3 deletions(-) create mode 100644 lib/workload/components/python-lambda-get-workflow-payload/get_workflow_payload_py/get_workflow_payload.py create mode 100644 lib/workload/components/python-lambda-get-workflow-payload/index.ts create mode 100644 lib/workload/components/python-lambda-list-service-instances/index.ts create mode 100644 lib/workload/components/python-lambda-list-service-instances/list_service_instances_py/list_service_instances.py create mode 100644 lib/workload/components/python-lambda-service-discovery/index.ts create mode 100644 lib/workload/components/python-lambda-service-discovery/service_discovery_py/service_discovery.py create mode 100644 lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts create mode 100644 lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json create mode 100644 lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/index.ts create mode 100644 lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/step_functions_templates/holmes_extract_wrapper_sfn_template.asl.json diff --git a/config/constants.ts b/config/constants.ts index d8b3911a0..0ddef0bb0 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -207,8 +207,8 @@ export const bclconvertInteropQcStateMachinePrefix = 'bclconvertInteropQcSfn'; Resources used by the bclConvert InteropQc Pipeline */ -// Release can be found here: https://github.com/umccr/cwl-ica/releases/tag/bclconvert-interop-qc%2F1.3.1--1.21__20240627051309 -// Pipeline ID is: 35cae57c-8895-4814-ae89-db4b5e9668b2 +// Release can be found here: https://github.com/umccr/cwl-ica/releases/tag/bclconvert-interop-qc%2F1.3.1--1.21__20241119001529 +// Pipeline ID is: a147ad9f-af8f-409d-95b7-49018782ab4d export const bclconvertInteropQcIcav2PipelineIdSSMParameterPath = '/icav2/umccr-prod/bclconvert_interop_qc_pipeline_id'; diff --git a/lib/workload/components/python-lambda-get-workflow-payload/get_workflow_payload_py/get_workflow_payload.py b/lib/workload/components/python-lambda-get-workflow-payload/get_workflow_payload_py/get_workflow_payload.py new file mode 100644 index 000000000..6a225c5a3 --- /dev/null +++ b/lib/workload/components/python-lambda-get-workflow-payload/get_workflow_payload_py/get_workflow_payload.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +""" +Given either a portal run id or a orcabus workflow id, and a workflow status, return the payload of the workflow at that state +""" +from workflow_tools.utils.payload_helpers import get_payload +from workflow_tools.utils.workflow_run_helpers import get_workflow_run_from_portal_run_id, get_workflow_run, \ + get_workflow_run_state + + +def handler(event, context): + """ + Takes in either (portal_run_id or orcabus_workflow_id) and workflow_status + + Gets the workflow payload at that state + + Returns the payload object + :param event: + :param context: + :return: + """ + + if event.get("portal_run_id", None) is not None: + # Get the workflow object from the portal run id + workflow_obj = get_workflow_run_from_portal_run_id(event.get("portal_run_id")) + elif event.get("orcabus_workflow_id", None) is not None: + # Get the workflow object from the orcabus workflow id + workflow_obj = get_workflow_run(event.get("orcabus_workflow_id")) + else: + raise ValueError("Must provide either portal_run_id or orcabus_workflow_id") + + status = event.get("workflow_status", None) + + if status is None: + raise ValueError("Must provide a workflow_status, i.e 'READY' or 'SUCCEEDED'") + + # Get the READY run state + workflow_ready_run_state_obj = get_workflow_run_state(workflow_obj.get("orcabusId"), status) + # Get the payload from the READY run state + return { + "payload": get_payload(workflow_ready_run_state_obj.get("payload")) + } + + +# if __name__ == "__main__": +# import json +# from os import environ +# environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt' +# environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name' +# environ['AWS_PROFILE'] = 'umccr-development' +# print( +# json.dumps( +# handler( +# { +# "portal_run_id": "202411071a2c31a3", +# "workflow_status": "SUCCEEDED" +# }, +# None +# ), +# indent=4 +# ), +# ) +# +# # { +# # "payload": { +# # "orcabusId": "pld.01JC28FD1GSVPDKH322NQKTYHR", +# # "payloadRefId": "2b5a3f90-4da5-4183-be2d-a88df6c21f1b", +# # "version": "2024.07.01", +# # "data": { +# # "tags": { +# # "libraryId": "L2401547", +# # "sampleType": "WGS", +# # "fastqListRowId": "GACCTGAA.CTCACCAA.3.241024_A00130_0336_BHW7MVDSXC.L2401547", +# # "instrumentRunId": "241024_A00130_0336_BHW7MVDSXC" +# # }, +# # "inputs": { +# # "sampleType": "WGS", +# # "fastqListRow": { +# # "lane": 3, +# # "rgid": "GACCTGAA.CTCACCAA.3.241024_A00130_0336_BHW7MVDSXC.L2401547", +# # "rglb": "L2401547", +# # "rgsm": "L2401547", +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_3/L2401547/L2401547_S16_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_3/L2401547/L2401547_S16_L003_R2_001.fastq.gz" +# # }, +# # "outputPrefix": "L2401547", +# # "fastqListRowId": "GACCTGAA.CTCACCAA.3.241024_A00130_0336_BHW7MVDSXC.L2401547", +# # "dragenReferenceVersion": "v9-r3" +# # }, +# # "outputs": { +# # "multiqcOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/L2401547_dragen_alignment_multiqc/", +# # "multiqcHtmlReportUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/L2401547_dragen_alignment_multiqc/L2401547_dragen_alignment_multiqc.html", +# # "dragenAlignmentBamUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/L2401547_dragen_alignment/L2401547.bam", +# # "dragenAlignmentOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/L2401547_dragen_alignment/" +# # }, +# # "engineParameters": { +# # "logsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/logs/wgts-qc/202411071a2c31a3/", +# # "cacheUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/cache/wgts-qc/202411071a2c31a3/", +# # "outputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/", +# # "projectId": "ea19a3f5-ec7c-4940-a474-c31cd91dbad4", +# # "analysisId": "0f2959fa-880a-4fa2-9908-8d12e3c998d5", +# # "pipelineId": "03689516-b7f8-4dca-bba9-8405b85fae45" +# # } +# # } +# # } +# # } diff --git a/lib/workload/components/python-lambda-get-workflow-payload/index.ts b/lib/workload/components/python-lambda-get-workflow-payload/index.ts new file mode 100644 index 000000000..a3ff8354a --- /dev/null +++ b/lib/workload/components/python-lambda-get-workflow-payload/index.ts @@ -0,0 +1,94 @@ +/* +Quick and dirty way to map orcabus ids to complementary ids for each +of the databases from the metadata manager + +Comes with the bells and whistles of metadata tools layer and +permissions to use the orcabus token. + +User will need to use the 'addEnvironment' method on the returned lambda object in order +to specify what command will be run + +User will need + +ENV: +CONTEXT: One of the following: + - library + - subject + - individual + - sample + - project + - contact + +FROM_ORCABUS or FROM_ID +RETURN_STR or RETURN_OBJ + +Look at ./map_metadata_py/map_metadata.py for examples of outputs + +*/ + +import { Construct } from 'constructs'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; +import path from 'path'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; +import { Duration } from 'aws-cdk-lib'; +import { WorkflowToolsPythonLambdaLayer } from '../python-workflow-tools-layer'; + +interface GetWorkflowPayloadLambdaObj { + functionNamePrefix: string; +} + +export class GetWorkflowPayloadLambdaConstruct extends Construct { + public readonly lambdaObj: PythonFunction; + + // Globals + private readonly hostnameSsmParameterPath = '/hosted_zone/umccr/name'; + private readonly orcabusTokenSecretId = 'orcabus/token-service-jwt'; // pragma: allowlist secret + + constructor(scope: Construct, id: string, props: GetWorkflowPayloadLambdaObj) { + super(scope, id); + + // Get the metadata layer object + const workflowToolsLayer = new WorkflowToolsPythonLambdaLayer(this, 'workflow-tools-layer', { + layerPrefix: `${props.functionNamePrefix}-wtl`, + }); + + /* + Collect the required secret and ssm parameters for getting metadata + */ + const hostnameSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 'hostname_ssm_parameter', + this.hostnameSsmParameterPath + ); + const orcabusTokenSecretObj = secretsmanager.Secret.fromSecretNameV2( + this, + 'orcabus_token_secret', + this.orcabusTokenSecretId + ); + + // Get library objects + this.lambdaObj = new PythonFunction(this, 'get_workflow_payload_py', { + functionName: `${props.functionNamePrefix}-get-workflow-payload-py`, + entry: path.join(__dirname, 'get_workflow_payload_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'get_workflow_payload.py', + handler: 'handler', + memorySize: 1024, + layers: [workflowToolsLayer.lambdaLayerVersionObj], + environment: { + HOSTNAME_SSM_PARAMETER: hostnameSsmParameterObj.parameterName, + ORCABUS_TOKEN_SECRET_ID: orcabusTokenSecretObj.secretName, + }, + timeout: Duration.seconds(60), + }); + + // Allow the lambda to read the secret + orcabusTokenSecretObj.grantRead(this.lambdaObj.currentVersion); + + // Allow the lambda to read the ssm parameter + hostnameSsmParameterObj.grantRead(this.lambdaObj.currentVersion); + } +} diff --git a/lib/workload/components/python-lambda-list-service-instances/index.ts b/lib/workload/components/python-lambda-list-service-instances/index.ts new file mode 100644 index 000000000..ac0c08251 --- /dev/null +++ b/lib/workload/components/python-lambda-list-service-instances/index.ts @@ -0,0 +1,55 @@ +import { Construct } from 'constructs'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as serviceDiscovery from 'aws-cdk-lib/aws-servicediscovery'; +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; +import path from 'path'; +import { NagSuppressions } from 'cdk-nag'; + +interface LambdaDiscoverInstancesProps { + functionNamePrefix: string; +} + +export class LambdaDiscoverInstancesConstruct extends Construct { + public readonly lambdaObj: PythonFunction; + + constructor(scope: Construct, id: string, props: LambdaDiscoverInstancesProps) { + super(scope, id); + + // ServiceDiscovery lambda + this.lambdaObj = new PythonFunction(this, 'list_service_instances_py', { + functionName: `${props.functionNamePrefix}-list-service-instances`, + entry: path.join(__dirname, 'list_service_instances_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'list_service_instances.py', + handler: 'handler', + memorySize: 1024, + }); + + // Grant permissions to the lambda + this.lambdaObj.addToRolePolicy( + new iam.PolicyStatement({ + actions: [ + 'servicediscovery:ListInstances', + 'servicediscovery:DiscoverInstances', + 'servicediscovery:GetService', + ], + resources: ['*'], + }) + ); + + // Suppress CDK NAGs + NagSuppressions.addResourceSuppressions( + this.lambdaObj, + [ + { + id: 'AwsSolutions-IAM5', + reason: + 'Need to run the DiscoverInstances against all services since we dont know which one will be used', + }, + ], + true + ); + } +} diff --git a/lib/workload/components/python-lambda-list-service-instances/list_service_instances_py/list_service_instances.py b/lib/workload/components/python-lambda-list-service-instances/list_service_instances_py/list_service_instances.py new file mode 100644 index 000000000..23fdc9de4 --- /dev/null +++ b/lib/workload/components/python-lambda-list-service-instances/list_service_instances_py/list_service_instances.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +""" +Use the boto3 api to list a service's instances +""" + +import typing +import boto3 +from os import environ + +if typing.TYPE_CHECKING: + from mypy_boto3_servicediscovery import ServiceDiscoveryClient + + +def get_service_discovery_client() -> 'ServiceDiscoveryClient': + return boto3.client('servicediscovery') + + +def handler(event, context): + """ + + :return: + """ + service_id = event["service_id"] + + service_discovery_client = get_service_discovery_client() + + service_instances = list( + map( + lambda service_instance_iter_: { + "instance_id": service_instance_iter_['Id'], + "instance_attributes": list( + map( + lambda instance_kv_iter_: { + "attr_key": instance_kv_iter_[0], + "attr_value": instance_kv_iter_[1], + }, + service_instance_iter_['Attributes'].items() + ) + ), + }, + service_discovery_client.list_instances(ServiceId=service_id)['Instances'] + ) + ) + + return { + "service_instances": service_instances + } + + +# if __name__ == "__main__": +# import json +# environ['AWS_PROFILE'] = 'umccr-development' +# print( +# json.dumps( +# handler( +# { +# "service_id": "srv-zuvfka3fyqxswwme" +# }, +# None +# ), +# indent=4 +# ) +# +# ) +# +# # { +# # "service_instances": [ +# # { +# # "instance_id": "HolmesLocalDevTestStackServiceNonIpD37CD7BB", +# # "instance_attributes": [ +# # { +# # "attr_key": "checkLambdaArn", +# # "attr_value": "arn:aws:lambda:ap-southeast-2:843407916570:function:HolmesLocalDevTestStack-CheckFunction82225D96-gtKl1C5USxIO" +# # }, +# # { +# # "attr_key": "controlLambdaArn", +# # "attr_value": "arn:aws:lambda:ap-southeast-2:843407916570:function:HolmesLocalDevTestStack-ControlFunction8A0764F3-zunc6vUJbVGz" +# # }, +# # { +# # "attr_key": "extractStepsArn", +# # "attr_value": "arn:aws:states:ap-southeast-2:843407916570:stateMachine:SomalierExtractStateMachine59E102CC-CEqwe36xUrru" +# # }, +# # { +# # "attr_key": "listLambdaArn", +# # "attr_value": "arn:aws:lambda:ap-southeast-2:843407916570:function:HolmesLocalDevTestStack-ListFunction89E6AFAD-WXt699CK8CMD" +# # }, +# # { +# # "attr_key": "relateLambdaArn", +# # "attr_value": "arn:aws:lambda:ap-southeast-2:843407916570:function:HolmesLocalDevTestStack-RelateFunction666B2CBC-2KKtCnoqHQ5F" +# # } +# # ] +# # } +# # ] +# # } diff --git a/lib/workload/components/python-lambda-metadata-mapper/index.ts b/lib/workload/components/python-lambda-metadata-mapper/index.ts index 369224988..c2a0f2239 100644 --- a/lib/workload/components/python-lambda-metadata-mapper/index.ts +++ b/lib/workload/components/python-lambda-metadata-mapper/index.ts @@ -51,7 +51,7 @@ export class GetMetadataLambdaConstruct extends Construct { // Get the metadata layer object const metadataLayerObj = new MetadataToolsPythonLambdaLayer(this, 'metadata-tools-layer', { - layerPrefix: 'get-library-objects', + layerPrefix: `${props.functionNamePrefix}-mtl`, }); /* diff --git a/lib/workload/components/python-lambda-service-discovery/index.ts b/lib/workload/components/python-lambda-service-discovery/index.ts new file mode 100644 index 000000000..54c74707b --- /dev/null +++ b/lib/workload/components/python-lambda-service-discovery/index.ts @@ -0,0 +1,50 @@ +import { Construct } from 'constructs'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as serviceDiscovery from 'aws-cdk-lib/aws-servicediscovery'; +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; +import path from 'path'; +import { NagSuppressions } from 'cdk-nag'; + +interface LambdaServiceDiscoveryProps { + functionNamePrefix: string; +} + +export class LambdaServiceDiscoveryConstruct extends Construct { + public readonly lambdaObj: PythonFunction; + + constructor(scope: Construct, id: string, props: LambdaServiceDiscoveryProps) { + super(scope, id); + + // ServiceDiscovery lambda + this.lambdaObj = new PythonFunction(this, 'service_discovery_py', { + functionName: `${props.functionNamePrefix}-service-discovery`, + entry: path.join(__dirname, 'service_discovery_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'service_discovery.py', + handler: 'handler', + memorySize: 1024, + }); + + // Grant permissions to the lambda + this.lambdaObj.addToRolePolicy( + new iam.PolicyStatement({ + actions: ['servicediscovery:ListServices'], + resources: ['*'], + }) + ); + + // Suppress CDK NAGs + NagSuppressions.addResourceSuppressions( + this.lambdaObj, + [ + { + id: 'AwsSolutions-IAM5', + reason: 'Need to run the ListServices against all services', + }, + ], + true + ); + } +} diff --git a/lib/workload/components/python-lambda-service-discovery/service_discovery_py/service_discovery.py b/lib/workload/components/python-lambda-service-discovery/service_discovery_py/service_discovery.py new file mode 100644 index 000000000..9426e693f --- /dev/null +++ b/lib/workload/components/python-lambda-service-discovery/service_discovery_py/service_discovery.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +""" +Use the boto3 api to discover the services running in the AWS account. + +SERVICE NAME is 'fingerprint' +""" + +import typing +import boto3 +from os import environ + +if typing.TYPE_CHECKING: + from mypy_boto3_servicediscovery import ServiceDiscoveryClient + + +def get_service_discovery_client() -> 'ServiceDiscoveryClient': + return boto3.client('servicediscovery') + + +def handler(event, context): + """ + + :return: + """ + service_name = event['service_name'] + + service_discovery_client = get_service_discovery_client() + + service_object = next( + filter( + lambda service_iter_: service_iter_['Name'] == service_name, + service_discovery_client.list_services()['Services'] + ) + ) + + return { + "service_obj": { + "service_id": service_object['Id'], + "service_name": service_object['Name'], + "service_arn": service_object['Arn'], + } + } + + +# if __name__ == "__main__": +# import json +# environ['AWS_PROFILE'] = 'umccr-development' +# print( +# json.dumps( +# handler( +# { +# "service_name": 'fingerprint' +# }, +# None +# ), +# indent=4 +# ) +# +# ) +# +# # { +# # "service_obj": { +# # "service_id": "srv-zuvfka3fyqxswwme", +# # "service_name": "fingerprint", +# # "service_arn": "arn:aws:servicediscovery:ap-southeast-2:843407916570:service/srv-zuvfka3fyqxswwme" +# # } +# # } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts new file mode 100644 index 000000000..b62dd6acc --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts @@ -0,0 +1,3 @@ +// FIXME + +// Launch holmes extract command from cttsov2 bam file diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json new file mode 100644 index 000000000..5408f3f09 --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json @@ -0,0 +1,239 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "Get Extract Inputs", + "States": { + "Get Extract Inputs": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Get Service", + "States": { + "Get Service": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_cloudmap_service_lambda_function_arn__}", + "Payload": { + "service_name": "${__service_name__}" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultSelector": { + "service_obj.$": "$.Payload.service_obj" + }, + "ResultPath": "$.get_service_obj_step", + "Next": "Get Service Instances" + }, + "Get Service Instances": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_service_instances_lambda_function_arn__}", + "Payload": { + "service_id.$": "$.get_service_obj_step.service_obj.service_id" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultSelector": { + "service_instances.$": "$.Payload.service_instances" + }, + "ResultPath": "$.get_service_instances_step", + "Next": "Get extract ARN" + }, + "Get extract ARN": { + "Type": "Pass", + "Parameters": { + "extract_arn.$": "States.ArrayGetItem($.get_service_instances_step.service_instances[0].instance_attributes[?(@.attr_key=='${__extract_arn_key__}')].attr_value, 0)" + }, + "End": true + } + } + }, + { + "StartAt": "Get Library Obj from Library Orcabus ID", + "States": { + "Get Library Obj from Library Orcabus ID": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_library_obj_lambda_function_arn__}", + "Payload": { + "value.$": "$.payload.data.library.orcabusId" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultSelector": { + "library_obj.$": "$.Payload" + }, + "ResultPath": "$.get_library_obj_step", + "Next": "Get payload inputs" + }, + "Get payload inputs": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Get tumor bam uri from portal run id", + "States": { + "Get tumor bam uri from portal run id": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_alignment_bam_uri_lambda_function_arn__}", + "Payload": { + "portal_run_id.$": "$.get_portal_run_id_step.portal_run_id", + "workflow_status": "SUCCEEDED" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultSelector": { + "alignment_bam_uri.$": "States.Format('{}DragenCaller/{}/{}_tumor.bam', $.Payload.payload.data.outputs.logsIntermediatesDir, $.Payload.payload.data.inputs.sampleId, $.Payload.payload.data.inputs.sampleId)" + }, + "ResultPath": "$.get_alignment_bam_uri_step", + "End": true + } + } + }, + { + "StartAt": "Get Individual ID from Subject OrcaBus ID", + "States": { + "Get Individual ID from Subject OrcaBus ID": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_individual_obj_lambda_function_arn__}", + "Payload": { + "value.$": "$.get_library_obj_step.library_obj.subject.orcabusId" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultPath": "$.get_individual_id_step", + "ResultSelector": { + "individual_id.$": "States.ArrayGetItem($.Payload.individualSet[?(@.individualId =~ /SBJ.*?/i)].individualId, 0)" + }, + "End": true + } + } + } + ], + "ResultSelector": { + "alignment_bam.$": "$.[0].get_alignment_bam_uri_step.alignment_bam_uri", + "individual_id.$": "$.[1].get_individual_id_step.individual_id" + }, + "End": true + } + } + } + ], + "ResultSelector": { + "extract_arn.$": "$.[0].extract_arn", + "individual_id.$": "$.[1].individual_id", + "alignment_bam.$": "$.[1].alignment_bam" + }, + "ResultPath": "$.extract_inputs", + "Next": "Launch Holmes Extract" + }, + "Launch Holmes Extract": { + "Type": "Task", + "Resource": "arn:aws:states:::states:startExecution.sync:2", + "Parameters": { + "StateMachineArn.$": "$.extract_inputs.extract_arn", + "Input": { + "subjectIdentifier.$": "$.extract_inputs.individual_id", + "libraryIdentifier.$": "$.payload.data.library.libraryId", + "indexes.$": "States.Array($.extract_inputs.alignment_bam)", + "reference": "${__reference_name__}" + } + }, + "ResultPath": "$.launch_holmes_extract_step", + "Next": "Holmes Extract Complete" + }, + "Holmes Extract Complete": { + "Type": "Task", + "Resource": "arn:aws:states:::events:putEvents", + "Parameters": { + "Entries": [ + { + "Detail": { + "status": "${__event_status__}", + "payload": { + "data": { + "libraryId.$": "$.extract_inputs.individual_id", + "outputs.$": "$.launch_holmes_extract_step.Output" + } + } + }, + "DetailType": "${__detail_type__}", + "EventBusName": "${__event_bus_name__}", + "Source": "${__event_source__}" + } + ] + }, + "End": true + } + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/index.ts index 6d807fdeb..dbdf71805 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/index.ts @@ -9,6 +9,7 @@ import { WgtsQcFastqListRowShowerCompleteToWorkflowReadyConstruct } from './part import { FastqListRowQcCompleteConstruct } from './part_4/push-fastq-list-row-qc-complete-event'; import { WgtsQcLibraryQcCompleteConstruct } from './part_5/library-qc-complete-event'; import { NestedStack } from 'aws-cdk-lib/core'; +import { HolmesExtractConstruct } from './part_6/launch-holmes-extract-event'; /* Provide the glue to get from the bssh fastq copy manager to submitting wgts qc analyses @@ -152,5 +153,25 @@ export class WgtsQcGlueHandlerConstruct extends NestedStack { tableObj: props.wgtsQcGlueTableObj, } ); + + /* + Part 6 + + Input Event Source: `orcabus.wgtsqcinputeventglue` + Input Event DetailType: `LibraryStateChange` + Input Event status: `QC_COMPLETE` + + Output Event Source: `orcabus.wgtsqcinputeventglue` + Output Event DetailType: `LibraryStateChange` + Output Event status: `HOLMES_EXTRACTION_COMPLETE` + */ + const LibraryQcCompleteToHolmesExtract = new HolmesExtractConstruct( + this, + 'wgts_qc_complete_to_holmes_extract_complete', + { + eventBusObj: props.eventBusObj, + tableObj: props.wgtsQcGlueTableObj, + } + ); } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/index.ts new file mode 100644 index 000000000..5bf378d2f --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/index.ts @@ -0,0 +1,244 @@ +import { Construct } from 'constructs'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import path from 'path'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; +import { LambdaServiceDiscoveryConstruct } from '../../../../../../../components/python-lambda-service-discovery'; +import { LambdaDiscoverInstancesConstruct } from '../../../../../../../components/python-lambda-list-service-instances'; +import { GetMetadataLambdaConstruct } from '../../../../../../../components/python-lambda-metadata-mapper'; +import { GetWorkflowPayloadLambdaConstruct } from '../../../../../../../components/python-lambda-get-workflow-payload'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as cdk from 'aws-cdk-lib'; +import { NagSuppressions } from 'cdk-nag'; + +/* +Part 7 + +Input Event Source: `orcabus.wgtsqcinputeventglue` +Input Event DetailType: `LibraryStateChange` +Input Event status: `QC_COMPLETE` + +Output Event Source: `orcabus.wgtsqcinputeventglue` +Output Event DetailType: `LibraryStateChange` +Output Event status: `HOLMES_EXTRACTION_COMPLETE` + +* Once all fastq list rows have been processed for a given library, we fire off a library state change event +* This will contain the qc information such as coverage + duplicate rate (for wgs) or exon coverage (for wts) + +*/ + +export interface HolmesExtractConstructProps { + tableObj: dynamodb.ITableV2; + eventBusObj: events.IEventBus; +} + +export class HolmesExtractConstruct extends Construct { + public readonly HolmesExtractMap = { + prefix: 'kwik-holmes-extract-complete', + payloadVersion: '2024.07.16', + triggerSource: 'orcabus.wgtsqcinputeventglue', + triggerStatus: 'QC_COMPLETE', + triggerDetailType: 'LibraryStateChange', + outputSource: 'orcabus.wgtsqcinputeventglue', + outputDetailType: 'LibraryStateChange', + outputStatus: 'HOLMES_EXTRACTION_COMPLETE', + serviceName: 'fingerprint', + extractStepsArnKey: 'extractStepsArn', + referenceName: 'hg38.rna', + tablePartitions: { + fastqListRow: 'fastq_list_row', + library: 'library', + }, + /* + FIXME - cloudmap paradox + we cannot find the cloudmap service inside cdk + but we need to give the parent step function permission + to invoke the child step function + so in the meantime just hardcode the arn prefix + and set a cdk nag suppression + */ + extractStepsArnPrefix: 'SomalierExtractStateMachine', + }; + + constructor(scope: Construct, id: string, props: HolmesExtractConstructProps) { + super(scope, id); + + /* + Part 1a: Lambdas for collecting the cloud-map services + */ + const serviceDiscoveryLambdaObj = new LambdaServiceDiscoveryConstruct( + this, + 'service_discovery_lambda', + { + functionNamePrefix: this.HolmesExtractMap.prefix, + } + ).lambdaObj; + + const serviceListInstancesLambdaObj = new LambdaDiscoverInstancesConstruct( + this, + 'service_list_instances_lambda', + { + functionNamePrefix: this.HolmesExtractMap.prefix, + } + ).lambdaObj; + + /* + Part 1b: Lambdas for getting the library object from the library id + */ + // Generate the lambda to collect the orcabus id from the subject id + const collectLibraryObjLambdaObj = new GetMetadataLambdaConstruct( + this, + 'get_library_obj_from_library_id_lambda', + { + functionNamePrefix: `${this.HolmesExtractMap.prefix}-lib`, + } + ).lambdaObj; + + // Add CONTEXT, FROM_ID and RETURN_OBJ environment variables to the lambda + collectLibraryObjLambdaObj.addEnvironment('CONTEXT', 'library'); + collectLibraryObjLambdaObj.addEnvironment('FROM_ORCABUS', ''); + collectLibraryObjLambdaObj.addEnvironment('RETURN_OBJ', ''); + + const collectIndividualObjLambdaObj = new GetMetadataLambdaConstruct( + this, + 'get_individual_obj_from_subject_lambda', + { + functionNamePrefix: `${this.HolmesExtractMap.prefix}-idv`, + } + ).lambdaObj; + // Add CONTEXT, FROM_ID and RETURN_OBJ environment variables to the lambda + collectIndividualObjLambdaObj.addEnvironment('CONTEXT', 'subject'); + collectIndividualObjLambdaObj.addEnvironment('FROM_ORCABUS', ''); + collectIndividualObjLambdaObj.addEnvironment('RETURN_OBJ', ''); + + /* + Part 1c: Get the alignment bam uri from the portal run id + Requires the workflow lambda layer and the orcabus token + */ + const getWorkflowPayloadLambdaObj = new GetWorkflowPayloadLambdaConstruct( + this, + 'get_workflow_payload_lambda', + { + functionNamePrefix: this.HolmesExtractMap.prefix, + } + ).lambdaObj; + + /* + Part 1: Build the sfn + */ + const holmesWrapperSfn = new sfn.StateMachine(this, 'holmes_wrapper_sfn', { + stateMachineName: `${this.HolmesExtractMap.prefix}-wrapper-sfn`, + definitionBody: sfn.DefinitionBody.fromFile( + path.join( + __dirname, + 'step_functions_templates', + 'holmes_extract_wrapper_sfn_template.asl.json' + ) + ), + definitionSubstitutions: { + /* Event stuff */ + __event_bus_name__: props.eventBusObj.eventBusName, + __event_source__: this.HolmesExtractMap.outputSource, + __detail_type__: this.HolmesExtractMap.outputDetailType, + __payload_version__: this.HolmesExtractMap.payloadVersion, + __status__: this.HolmesExtractMap.outputStatus, + /* Table stuff */ + __table_name__: props.tableObj.tableName, + __fastq_list_row_table_partition_name__: this.HolmesExtractMap.tablePartitions.fastqListRow, + __library_table_partition_name__: this.HolmesExtractMap.tablePartitions.library, + /* Cloud Map Stuff */ + __service_name__: this.HolmesExtractMap.serviceName, + __extract_arn_key__: this.HolmesExtractMap.extractStepsArnKey, + /* Lambdas */ + __get_cloudmap_service_lambda_function_arn__: + serviceDiscoveryLambdaObj.currentVersion.functionArn, + __get_service_instances_lambda_function_arn__: + serviceListInstancesLambdaObj.currentVersion.functionArn, + __get_library_obj_lambda_function_arn__: + collectLibraryObjLambdaObj.currentVersion.functionArn, + __get_alignment_bam_uri_lambda_function_arn__: + getWorkflowPayloadLambdaObj.currentVersion.functionArn, + __get_individual_obj_lambda_function_arn__: + collectIndividualObjLambdaObj.currentVersion.functionArn, + /* Reference input */ + __reference_name__: this.HolmesExtractMap.referenceName, + }, + }); + + /* + Part 2: Grant the internal sfn permissions + */ + // access the dynamodb table + props.tableObj.grantReadWriteData(holmesWrapperSfn.role); + // invoke the lambda function + [ + serviceDiscoveryLambdaObj, + serviceListInstancesLambdaObj, + collectLibraryObjLambdaObj, + getWorkflowPayloadLambdaObj, + collectIndividualObjLambdaObj, + ].forEach((lambdaObj) => { + lambdaObj.currentVersion.grantInvoke(holmesWrapperSfn); + }); + + // Push events to the event bus + props.eventBusObj.grantPutEventsTo(holmesWrapperSfn.role); + + // Because we run a nested state machine, we need to add the permissions to the state machine role + // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr + holmesWrapperSfn.addToRolePolicy( + new iam.PolicyStatement({ + resources: [ + `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, + ], + actions: ['events:PutTargets', 'events:PutRule', 'events:DescribeRule'], + }) + ); + + // Add permissions to the statemachine to allow execution of the holmes extract state machine + holmesWrapperSfn.addToRolePolicy( + new iam.PolicyStatement({ + resources: [ + `arn:aws:states:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:stateMachine:${this.HolmesExtractMap.extractStepsArnPrefix}*`, + ], + actions: ['states:StartExecution'], + }) + ); + + // Add cdk nag suppressions + // FIXME - cannot get the full arn of the extractStepsArn in CDK + NagSuppressions.addResourceSuppressions( + holmesWrapperSfn, + [ + { + id: 'AwsSolutions-IAM5', + reason: 'Cannot get the extractStepsArn full path in CDK', + }, + ], + true + ); + + /* + Part 3: Subscribe to the event bus and trigger the internal sfn + */ + const rule = new events.Rule(this, 'wgts_subscribe_to_library_qc_complete', { + ruleName: `stacky-${this.HolmesExtractMap.prefix}-rule`, + eventBus: props.eventBusObj, + eventPattern: { + source: [this.HolmesExtractMap.triggerSource], + detailType: [this.HolmesExtractMap.triggerDetailType], + detail: { + status: [{ 'equals-ignore-case': this.HolmesExtractMap.triggerStatus }], + }, + }, + }); + + // Add target of event to be the state machine + rule.addTarget( + new eventsTargets.SfnStateMachine(holmesWrapperSfn, { + input: events.RuleTargetInput.fromEventPath('$.detail'), + }) + ); + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/step_functions_templates/holmes_extract_wrapper_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/step_functions_templates/holmes_extract_wrapper_sfn_template.asl.json new file mode 100644 index 000000000..61f32bedf --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/step_functions_templates/holmes_extract_wrapper_sfn_template.asl.json @@ -0,0 +1,291 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "Get Extract Inputs", + "States": { + "Get Extract Inputs": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Get Service", + "States": { + "Get Service": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_cloudmap_service_lambda_function_arn__}", + "Payload": { + "service_name": "${__service_name__}" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultSelector": { + "service_obj.$": "$.Payload.service_obj" + }, + "ResultPath": "$.get_service_obj_step", + "Next": "Get Service Instances" + }, + "Get Service Instances": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_service_instances_lambda_function_arn__}", + "Payload": { + "service_id.$": "$.get_service_obj_step.service_obj.service_id" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultSelector": { + "service_instances.$": "$.Payload.service_instances" + }, + "ResultPath": "$.get_service_instances_step", + "Next": "Get extract ARN" + }, + "Get extract ARN": { + "Type": "Pass", + "Parameters": { + "extract_arn.$": "States.ArrayGetItem($.get_service_instances_step.service_instances[0].instance_attributes[?(@.attr_key=='${__extract_arn_key__}')].attr_value, 0)" + }, + "End": true + } + } + }, + { + "StartAt": "Get Library Obj from Library Orcabus ID", + "States": { + "Get Library Obj from Library Orcabus ID": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_library_obj_lambda_function_arn__}", + "Payload": { + "value.$": "$.payload.data.library.orcabusId" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultSelector": { + "library_obj.$": "$.Payload" + }, + "ResultPath": "$.get_library_obj_step", + "Next": "Get payload inputs" + }, + "Get payload inputs": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Get Fastq List Rows from Library", + "States": { + "Get Fastq List Rows from Library": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.payload.data.library.orcabusId", + "id_type": "${__library_table_partition_name__}" + } + }, + "ResultSelector": { + "rgids_list.$": "$.Item.fastq_list_row_id_set.SS" + }, + "ResultPath": "$.get_fastq_list_rows_step", + "Next": "Get WGTS QC alignment bams" + }, + "Get WGTS QC alignment bams": { + "Type": "Map", + "ItemsPath": "$.get_fastq_list_rows_step.rgids_list", + "ItemSelector": { + "rgid.$": "$$.Map.Item.Value" + }, + "ItemProcessor": { + "ProcessorConfig": { + "Mode": "INLINE" + }, + "StartAt": "Get Portal Run ID", + "States": { + "Get Portal Run ID": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.rgid", + "id_type": "${__fastq_list_row_table_partition_name__}" + } + }, + "ResultSelector": { + "portal_run_id.$": "$.Item.portal_run_id.S" + }, + "ResultPath": "$.get_portal_run_id_step", + "Next": "Get alignment bam uri from portal run id" + }, + "Get alignment bam uri from portal run id": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_alignment_bam_uri_lambda_function_arn__}", + "Payload": { + "portal_run_id.$": "$.get_portal_run_id_step.portal_run_id", + "workflow_status": "SUCCEEDED" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultSelector": { + "alignment_bam_uri.$": "$.Payload.payload.data.outputs.dragenAlignmentBamUri" + }, + "ResultPath": "$.get_alignment_bam_uri_step", + "End": true + } + } + }, + "ResultSelector": { + "alignment_bams.$": "$[*].get_alignment_bam_uri_step.alignment_bam_uri" + }, + "ResultPath": "$.get_alignment_bams_step", + "End": true + } + } + }, + { + "StartAt": "Get Individual ID from Subject OrcaBus ID", + "States": { + "Get Individual ID from Subject OrcaBus ID": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_individual_obj_lambda_function_arn__}", + "Payload": { + "value.$": "$.get_library_obj_step.library_obj.subject.orcabusId" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultPath": "$.get_individual_id_step", + "ResultSelector": { + "individual_id.$": "States.ArrayGetItem($.Payload.individualSet[?(@.individualId =~ /SBJ.*?/i)].individualId, 0)" + }, + "End": true + } + } + } + ], + "ResultSelector": { + "alignment_bams.$": "$.[0].get_alignment_bams_step.alignment_bams", + "individual_id.$": "$.[1].get_individual_id_step.individual_id" + }, + "End": true + } + } + } + ], + "ResultSelector": { + "extract_arn.$": "$.[0].extract_arn", + "individual_id.$": "$.[1].individual_id", + "alignment_bams.$": "$.[1].alignment_bams" + }, + "ResultPath": "$.extract_inputs", + "Next": "Launch Holmes Extract" + }, + "Launch Holmes Extract": { + "Type": "Task", + "Resource": "arn:aws:states:::states:startExecution.sync:2", + "Parameters": { + "StateMachineArn.$": "$.extract_inputs.extract_arn", + "Input": { + "subjectIdentifier.$": "$.extract_inputs.individual_id", + "libraryIdentifier.$": "$.payload.data.library.libraryId", + "indexes.$": "$.extract_inputs.alignment_bams", + "reference": "${__reference_name__}" + } + }, + "ResultPath": "$.launch_holmes_extract_step", + "Next": "Holmes Extract Complete" + }, + "Holmes Extract Complete": { + "Type": "Task", + "Resource": "arn:aws:states:::events:putEvents", + "Parameters": { + "Entries": [ + { + "Detail": { + "status": "${__event_status__}", + "payload": { + "data": { + "libraryId.$": "$.extract_inputs.individual_id", + "outputs.$": "$.launch_holmes_extract_step.Output" + } + } + }, + "DetailType": "${__detail_type__}", + "EventBusName": "${__event_bus_name__}", + "Source": "${__event_source__}" + } + ] + }, + "End": true + } + } +} From e683ffd8ef2aa01d0d4b47a7b419b10dc7b67139 Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Wed, 20 Nov 2024 12:27:24 +1100 Subject: [PATCH 2/4] Added step function to run holmes extract on cttsov2 data --- .../glue-constructs/jb-weld/index.ts | 6 + .../launch-holmes-extract-event/index.ts | 235 +++++++++++++++++- ...olmes_bam_extraction_sfn_template.asl.json | 39 +-- .../launch-holmes-extract-event/index.ts | 4 +- 4 files changed, 253 insertions(+), 31 deletions(-) diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/index.ts index 913d771e1..cfac78b76 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/index.ts @@ -7,6 +7,7 @@ import { Cttsov2InitialiseLibraryAndFastqListRowConstruct } from './part_1/initi import { Cttsov2PopulateFastqListRowConstruct } from './part_2/populate-fastq-list-row-dbs'; import { Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct } from './part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready'; import { NestedStack } from 'aws-cdk-lib/core'; +import { HolmesExtractConstruct } from './part_4/launch-holmes-extract-event'; /* Provide the glue to get from the bssh fastq copy manager to submitting cttsov2 analyses @@ -99,5 +100,10 @@ export class Cttsov2GlueHandlerConstruct extends NestedStack { icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, } ); + + const holmesExtractConstruct = new HolmesExtractConstruct(this, 'holmes_extract', { + eventBusObj: props.eventBusObj, + tableObj: props.cttsov2GlueTableObj, + }); } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts index b62dd6acc..60b8b5796 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts @@ -1,3 +1,234 @@ -// FIXME - // Launch holmes extract command from cttsov2 bam file + +import { Construct } from 'constructs'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import path from 'path'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; +import { LambdaServiceDiscoveryConstruct } from '../../../../../../../components/python-lambda-service-discovery'; +import { LambdaDiscoverInstancesConstruct } from '../../../../../../../components/python-lambda-list-service-instances'; +import { GetMetadataLambdaConstruct } from '../../../../../../../components/python-lambda-metadata-mapper'; +import { GetWorkflowPayloadLambdaConstruct } from '../../../../../../../components/python-lambda-get-workflow-payload'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as cdk from 'aws-cdk-lib'; +import { NagSuppressions } from 'cdk-nag'; + +/* +Part 6 + +Input Event Source: `orcabus.workflowmanager` +Input Event DetailType: `WorkflowRunStateChange` +Input Event status: `SUCCEEDED` +Input Event Workflow Run Name: `cttsov2` + +Output Event Source: `orcabus.cttsov2inputeventglue` +Output Event DetailType: `LibraryStateChange` +Output Event status: `HOLMES_EXTRACTION_COMPLETE` + +* Once all fastq list rows have been processed for a given library, we fire off a library state change event +* This will contain the qc information such as coverage + duplicate rate (for wgs) or exon coverage (for wts) + +*/ + +export interface HolmesExtractConstructProps { + tableObj: dynamodb.ITableV2; + eventBusObj: events.IEventBus; +} + +export class HolmesExtractConstruct extends Construct { + public readonly HolmesExtractMap = { + prefix: 'jb-weld-holmes-extract-complete', + payloadVersion: '2024.07.16', + triggerSource: 'orcabus.workflowmanager', + triggerStatus: 'SUCCEEDED', + triggerDetailType: 'WorkflowRunStateChange', + triggerWorkflowName: 'cttsov2', + outputSource: 'orcabus.cttsov2inputeventglue', + outputDetailType: 'LibraryStateChange', + outputStatus: 'HOLMES_EXTRACTION_COMPLETE', + serviceName: 'fingerprint', + extractStepsArnKey: 'extractStepsArn', + referenceName: 'hg19.rna', + tablePartitions: { + fastqListRow: 'fastq_list_row', + library: 'library', + }, + /* + FIXME - cloudmap paradox + we cannot find the cloudmap service inside cdk + but we need to give the parent step function permission + to invoke the child step function + so in the meantime just hardcode the arn prefix + and set a cdk nag suppression + */ + extractStepsArnPrefix: 'SomalierExtractStateMachine', + }; + + constructor(scope: Construct, id: string, props: HolmesExtractConstructProps) { + super(scope, id); + + /* + Part 1a: Lambdas for collecting the cloud-map services + */ + const serviceDiscoveryLambdaObj = new LambdaServiceDiscoveryConstruct( + this, + 'service_discovery_lambda', + { + functionNamePrefix: this.HolmesExtractMap.prefix, + } + ).lambdaObj; + + const serviceListInstancesLambdaObj = new LambdaDiscoverInstancesConstruct( + this, + 'service_list_instances_lambda', + { + functionNamePrefix: this.HolmesExtractMap.prefix, + } + ).lambdaObj; + + /* + Part 1b: Lambdas for getting the library object from the library id + */ + // Generate the lambda to collect the orcabus id from the subject id + const collectLibraryObjLambdaObj = new GetMetadataLambdaConstruct( + this, + 'get_library_obj_from_library_id_lambda', + { + functionNamePrefix: `${this.HolmesExtractMap.prefix}-lib`, + } + ).lambdaObj; + + // Add CONTEXT, FROM_ID and RETURN_OBJ environment variables to the lambda + collectLibraryObjLambdaObj.addEnvironment('CONTEXT', 'library'); + collectLibraryObjLambdaObj.addEnvironment('FROM_ORCABUS', ''); + collectLibraryObjLambdaObj.addEnvironment('RETURN_OBJ', ''); + + const collectIndividualObjLambdaObj = new GetMetadataLambdaConstruct( + this, + 'get_individual_obj_from_subject_lambda', + { + functionNamePrefix: `${this.HolmesExtractMap.prefix}-idv`, + } + ).lambdaObj; + // Add CONTEXT, FROM_ID and RETURN_OBJ environment variables to the lambda + collectIndividualObjLambdaObj.addEnvironment('CONTEXT', 'subject'); + collectIndividualObjLambdaObj.addEnvironment('FROM_ORCABUS', ''); + collectIndividualObjLambdaObj.addEnvironment('RETURN_OBJ', ''); + + /* + Part 1: Build the sfn + */ + const holmesWrapperSfn = new sfn.StateMachine(this, 'holmes_wrapper_sfn', { + stateMachineName: `${this.HolmesExtractMap.prefix}-wrapper-sfn`, + definitionBody: sfn.DefinitionBody.fromFile( + path.join( + __dirname, + 'step_functions_templates', + 'cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json' + ) + ), + definitionSubstitutions: { + /* Event stuff */ + __event_bus_name__: props.eventBusObj.eventBusName, + __event_source__: this.HolmesExtractMap.outputSource, + __detail_type__: this.HolmesExtractMap.outputDetailType, + __payload_version__: this.HolmesExtractMap.payloadVersion, + __event_status__: this.HolmesExtractMap.outputStatus, + /* Table stuff */ + __table_name__: props.tableObj.tableName, + __fastq_list_row_table_partition_name__: this.HolmesExtractMap.tablePartitions.fastqListRow, + __library_table_partition_name__: this.HolmesExtractMap.tablePartitions.library, + /* Cloud Map Stuff */ + __service_name__: this.HolmesExtractMap.serviceName, + __extract_arn_key__: this.HolmesExtractMap.extractStepsArnKey, + /* Lambdas */ + __get_cloudmap_service_lambda_function_arn__: + serviceDiscoveryLambdaObj.currentVersion.functionArn, + __get_service_instances_lambda_function_arn__: + serviceListInstancesLambdaObj.currentVersion.functionArn, + __get_library_obj_lambda_function_arn__: + collectLibraryObjLambdaObj.currentVersion.functionArn, + __get_individual_obj_lambda_function_arn__: + collectIndividualObjLambdaObj.currentVersion.functionArn, + /* Reference input */ + __reference_name__: this.HolmesExtractMap.referenceName, + }, + }); + + /* + Part 2: Grant the internal sfn permissions + */ + // access the dynamodb table + props.tableObj.grantReadWriteData(holmesWrapperSfn.role); + // invoke the lambda function + [ + serviceDiscoveryLambdaObj, + serviceListInstancesLambdaObj, + collectLibraryObjLambdaObj, + collectIndividualObjLambdaObj, + ].forEach((lambdaObj) => { + lambdaObj.currentVersion.grantInvoke(holmesWrapperSfn); + }); + + // Push events to the event bus + props.eventBusObj.grantPutEventsTo(holmesWrapperSfn.role); + + // Because we run a nested state machine, we need to add the permissions to the state machine role + // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr + holmesWrapperSfn.addToRolePolicy( + new iam.PolicyStatement({ + resources: [ + `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, + ], + actions: ['events:PutTargets', 'events:PutRule', 'events:DescribeRule'], + }) + ); + + // Add permissions to the statemachine to allow execution of the holmes extract state machine + holmesWrapperSfn.addToRolePolicy( + new iam.PolicyStatement({ + resources: [ + `arn:aws:states:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:stateMachine:${this.HolmesExtractMap.extractStepsArnPrefix}*`, + ], + actions: ['states:StartExecution'], + }) + ); + + // Add cdk nag suppressions + // FIXME - cannot get the full arn of the extractStepsArn in CDK + NagSuppressions.addResourceSuppressions( + holmesWrapperSfn, + [ + { + id: 'AwsSolutions-IAM5', + reason: 'Cannot get the extractStepsArn full path in CDK', + }, + ], + true + ); + + /* + Part 3: Subscribe to the event bus and trigger the internal sfn + */ + const rule = new events.Rule(this, 'wgts_subscribe_to_library_qc_complete', { + ruleName: `stacky-${this.HolmesExtractMap.prefix}-rule`, + eventBus: props.eventBusObj, + eventPattern: { + source: [this.HolmesExtractMap.triggerSource], + detailType: [this.HolmesExtractMap.triggerDetailType], + detail: { + status: [{ 'equals-ignore-case': this.HolmesExtractMap.triggerStatus }], + workflowRunName: [{ 'equals-ignore-case': this.HolmesExtractMap.triggerWorkflowName }], + }, + }, + }); + + // Add target of event to be the state machine + rule.addTarget( + new eventsTargets.SfnStateMachine(holmesWrapperSfn, { + input: events.RuleTargetInput.fromEventPath('$.detail'), + }) + ); + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json index 5408f3f09..1f53c2508 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json @@ -1,7 +1,14 @@ { "Comment": "A description of my state machine", - "StartAt": "Get Extract Inputs", + "StartAt": "Move Workflow Inputs", "States": { + "Move Workflow Inputs": { + "Type": "Pass", + "Parameters": { + "workflow_inputs.$": "$" + }, + "Next": "Get Extract Inputs" + }, "Get Extract Inputs": { "Type": "Parallel", "Branches": [ @@ -84,7 +91,7 @@ "Parameters": { "FunctionName": "${__get_library_obj_lambda_function_arn__}", "Payload": { - "value.$": "$.payload.data.library.orcabusId" + "value.$": "$.workflow_inputs.linkedLibraries[0].orcabusId" } }, "Retry": [ @@ -114,31 +121,9 @@ "StartAt": "Get tumor bam uri from portal run id", "States": { "Get tumor bam uri from portal run id": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Pass", "Parameters": { - "FunctionName": "${__get_alignment_bam_uri_lambda_function_arn__}", - "Payload": { - "portal_run_id.$": "$.get_portal_run_id_step.portal_run_id", - "workflow_status": "SUCCEEDED" - } - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2, - "JitterStrategy": "FULL" - } - ], - "ResultSelector": { - "alignment_bam_uri.$": "States.Format('{}DragenCaller/{}/{}_tumor.bam', $.Payload.payload.data.outputs.logsIntermediatesDir, $.Payload.payload.data.inputs.sampleId, $.Payload.payload.data.inputs.sampleId)" + "alignment_bam_uri.$": "States.Format('{}DragenCaller/{}/{}_tumor.bam', $.workflow_inputs.payload.data.outputs.logsIntermediatesDir, $.workflow_inputs.payload.data.inputs.sampleId, $.workflow_inputs.payload.data.inputs.sampleId)" }, "ResultPath": "$.get_alignment_bam_uri_step", "End": true @@ -204,7 +189,7 @@ "StateMachineArn.$": "$.extract_inputs.extract_arn", "Input": { "subjectIdentifier.$": "$.extract_inputs.individual_id", - "libraryIdentifier.$": "$.payload.data.library.libraryId", + "libraryIdentifier.$": "$.workflow_inputs.linkedLibraries[0].libraryId", "indexes.$": "States.Array($.extract_inputs.alignment_bam)", "reference": "${__reference_name__}" } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/index.ts index 5bf378d2f..f430da7b9 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/index.ts @@ -13,7 +13,7 @@ import * as cdk from 'aws-cdk-lib'; import { NagSuppressions } from 'cdk-nag'; /* -Part 7 +Part 6 Input Event Source: `orcabus.wgtsqcinputeventglue` Input Event DetailType: `LibraryStateChange` @@ -142,7 +142,7 @@ export class HolmesExtractConstruct extends Construct { __event_source__: this.HolmesExtractMap.outputSource, __detail_type__: this.HolmesExtractMap.outputDetailType, __payload_version__: this.HolmesExtractMap.payloadVersion, - __status__: this.HolmesExtractMap.outputStatus, + __event_status__: this.HolmesExtractMap.outputStatus, /* Table stuff */ __table_name__: props.tableObj.tableName, __fastq_list_row_table_partition_name__: this.HolmesExtractMap.tablePartitions.fastqListRow, From 099f94fae7bc430125b97f96357519547c235ae5 Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Wed, 20 Nov 2024 12:34:40 +1100 Subject: [PATCH 3/4] Update event outputs for holmes extract completion --- ...te_to_holmes_bam_extraction_sfn_template.asl.json | 12 ++++++------ .../holmes_extract_wrapper_sfn_template.asl.json | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json index 1f53c2508..81f304fda 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/step_functions_templates/cttsov2_complete_to_holmes_bam_extraction_sfn_template.asl.json @@ -203,18 +203,18 @@ "Parameters": { "Entries": [ { + "EventBusName": "${__event_bus_name__}", + "DetailType": "${__detail_type__}", + "Source": "${__event_source__}", "Detail": { "status": "${__event_status__}", "payload": { "data": { - "libraryId.$": "$.extract_inputs.individual_id", - "outputs.$": "$.launch_holmes_extract_step.Output" + "library.$": "$.workflow_inputs.linkedLibraries[0]", + "bamFiles.$": "States.Array($.extract_inputs.alignment_bam)" } } - }, - "DetailType": "${__detail_type__}", - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}" + } } ] }, diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/step_functions_templates/holmes_extract_wrapper_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/step_functions_templates/holmes_extract_wrapper_sfn_template.asl.json index 61f32bedf..187398503 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/step_functions_templates/holmes_extract_wrapper_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/launch-holmes-extract-event/step_functions_templates/holmes_extract_wrapper_sfn_template.asl.json @@ -270,18 +270,18 @@ "Parameters": { "Entries": [ { + "EventBusName": "${__event_bus_name__}", + "DetailType": "${__detail_type__}", + "Source": "${__event_source__}", "Detail": { "status": "${__event_status__}", "payload": { "data": { - "libraryId.$": "$.extract_inputs.individual_id", - "outputs.$": "$.launch_holmes_extract_step.Output" + "library.$": "$.payload.data.library", + "bamFiles.$": "$.extract_inputs.alignment_bams" } } - }, - "DetailType": "${__detail_type__}", - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}" + } } ] }, From 70ba6b539f72edb547d2d727aae5ac99b8fea587 Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Wed, 20 Nov 2024 12:35:55 +1100 Subject: [PATCH 4/4] Remove unnecessary imports --- .../components/python-lambda-list-service-instances/index.ts | 1 - lib/workload/components/python-lambda-service-discovery/index.ts | 1 - .../jb-weld/part_4/launch-holmes-extract-event/index.ts | 1 - 3 files changed, 3 deletions(-) diff --git a/lib/workload/components/python-lambda-list-service-instances/index.ts b/lib/workload/components/python-lambda-list-service-instances/index.ts index ac0c08251..4d387b452 100644 --- a/lib/workload/components/python-lambda-list-service-instances/index.ts +++ b/lib/workload/components/python-lambda-list-service-instances/index.ts @@ -1,7 +1,6 @@ import { Construct } from 'constructs'; import * as lambda from 'aws-cdk-lib/aws-lambda'; import * as iam from 'aws-cdk-lib/aws-iam'; -import * as serviceDiscovery from 'aws-cdk-lib/aws-servicediscovery'; import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; import path from 'path'; import { NagSuppressions } from 'cdk-nag'; diff --git a/lib/workload/components/python-lambda-service-discovery/index.ts b/lib/workload/components/python-lambda-service-discovery/index.ts index 54c74707b..b40f5079b 100644 --- a/lib/workload/components/python-lambda-service-discovery/index.ts +++ b/lib/workload/components/python-lambda-service-discovery/index.ts @@ -1,7 +1,6 @@ import { Construct } from 'constructs'; import * as lambda from 'aws-cdk-lib/aws-lambda'; import * as iam from 'aws-cdk-lib/aws-iam'; -import * as serviceDiscovery from 'aws-cdk-lib/aws-servicediscovery'; import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; import path from 'path'; import { NagSuppressions } from 'cdk-nag'; diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts index 60b8b5796..ad68bb6d4 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/launch-holmes-extract-event/index.ts @@ -9,7 +9,6 @@ import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; import { LambdaServiceDiscoveryConstruct } from '../../../../../../../components/python-lambda-service-discovery'; import { LambdaDiscoverInstancesConstruct } from '../../../../../../../components/python-lambda-list-service-instances'; import { GetMetadataLambdaConstruct } from '../../../../../../../components/python-lambda-metadata-mapper'; -import { GetWorkflowPayloadLambdaConstruct } from '../../../../../../../components/python-lambda-get-workflow-payload'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as cdk from 'aws-cdk-lib'; import { NagSuppressions } from 'cdk-nag';