From 12e29e2fe06df04dd0043ec647fe82d09726fd37 Mon Sep 17 00:00:00 2001 From: Alexis Lucattini Date: Tue, 20 Aug 2024 18:21:32 +1000 Subject: [PATCH] Implemented umccrise pipeline manager into the orcabus stack --- config/config.ts | 6 ++ config/constants.ts | 51 +++++++++++++++- config/stacks/umccrisePipelineManager.ts | 58 +++++++++++++++++++ .../deploy/stack.ts | 54 +++++++++++++++++ .../stateful/statefulStackCollectionClass.ts | 15 +++++ .../deploy/index.ts | 0 .../get_cwl_object_from_s3_inputs.py} | 0 .../requirements.txt | 0 .../lambdas/get_outputs_py/get_outputs.py | 0 .../set_umccrise_cwl_inputs_sfn.asl.json | 0 .../set_umccrise_cwl_outputs_sfn.asl.json | 0 .../statelessStackCollectionClass.ts | 15 +++++ 12 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 config/stacks/umccrisePipelineManager.ts create mode 100644 lib/workload/stateful/stacks/umccrise-pipeline-dynamo-db/deploy/stack.ts rename lib/workload/stateless/stacks/{umccrise-manager => umccrise-pipeline-manager}/deploy/index.ts (100%) rename lib/workload/stateless/stacks/{umccrise-manager/lambdas/get_cwl_object_from_s3_inputs_py/get_cwl_objects_from_s3_inputs.py => umccrise-pipeline-manager/lambdas/get_cwl_object_from_s3_inputs_py/get_cwl_object_from_s3_inputs.py} (100%) rename lib/workload/stateless/stacks/{umccrise-manager => umccrise-pipeline-manager}/lambdas/get_cwl_object_from_s3_inputs_py/requirements.txt (100%) rename lib/workload/stateless/stacks/{umccrise-manager => umccrise-pipeline-manager}/lambdas/get_outputs_py/get_outputs.py (100%) rename lib/workload/stateless/stacks/{umccrise-manager => umccrise-pipeline-manager}/step_functions_templates/set_umccrise_cwl_inputs_sfn.asl.json (100%) rename lib/workload/stateless/stacks/{umccrise-manager => umccrise-pipeline-manager}/step_functions_templates/set_umccrise_cwl_outputs_sfn.asl.json (100%) diff --git a/config/config.ts b/config/config.ts index d57e40a0a..383eb82e6 100644 --- a/config/config.ts +++ b/config/config.ts @@ -39,6 +39,10 @@ import { getWtsIcav2PipelineManagerStackProps, getWtsIcav2PipelineTableStackProps, } from './stacks/wtsPipelineManager'; +import { + getUmccriseIcav2PipelineManagerStackProps, + getUmccriseIcav2PipelineTableStackProps, +} from './stacks/umccrisePipelineManager'; interface EnvironmentConfig { name: string; @@ -70,6 +74,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null wgtsQcIcav2PipelineTableStackProps: getWgtsQcIcav2PipelineTableStackProps(), tnIcav2PipelineTableStackProps: getTnIcav2PipelineTableStackProps(), wtsIcav2PipelineTableStackProps: getWtsIcav2PipelineTableStackProps(), + umccriseIcav2PipelineTableStackProps: getUmccriseIcav2PipelineTableStackProps(), BclConvertTableStackProps: getBclConvertManagerTableStackProps(stage), stackyStatefulTablesStackProps: getStatefulGlueStackProps(), }, @@ -85,6 +90,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null wgtsQcIcav2PipelineManagerStackProps: getWgtsQcIcav2PipelineManagerStackProps(stage), tnIcav2PipelineManagerStackProps: getTnIcav2PipelineManagerStackProps(stage), wtsIcav2PipelineManagerStackProps: getWtsIcav2PipelineManagerStackProps(stage), + umccriseIcav2PipelineManagerStackProps: getUmccriseIcav2PipelineManagerStackProps(stage), eventSchemaStackProps: getEventSchemaStackProps(), dataSchemaStackProps: getDataSchemaStackProps(), bclConvertManagerStackProps: getBclConvertManagerStackProps(stage), diff --git a/config/constants.ts b/config/constants.ts index 0a272f3ee..c07146927 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -396,6 +396,56 @@ export const wtsDefaultArribaVersion = '2.4.0'; export const wtsDefaultQcReferenceSamplesVersion = '2023-07-21--4.2.4'; +/* +Resources required by the umccrise stacks +*/ + +// Deployed under dev/stg/prod +// ' +// [ +// { +// "name": "202303", +// "uri": "icav2://reference-data/umccrise/202303/genomes.tar.gz" +// } +// ] +// ' +export const icav2UmccriseGenomesReferenceUriMappingSSMParameterPath = + '/icav2/umccr-prod/umccrise_genomes_reference_mapping'; // + +/* +Resources generated by the UMCCRise Stateful Stack +*/ +export const umccriseIcav2PipelineManagerSSMRoot = '/orcabus/umccrise'; + +/* +UMCCRise Stateful stack +*/ +export const umccriseIcav2PipelineManagerDynamodbTableName = 'umccriseICAv2AnalysesDynamoDBTable'; +export const umccriseDynamoDbTableSSMName = path.join( + umccriseIcav2PipelineManagerSSMRoot, + 'dynamodb_table_name' +); +export const umccriseDynamoDbTableSSMArn = path.join( + umccriseIcav2PipelineManagerSSMRoot, + 'dynamodb_table_arn' +); + +/* +UMCCRise Stateless stack +*/ + +// Deployed in dev/stg/prod +export const umccriseIcav2PipelineIdSSMParameterPath = + '/icav2/umccr-prod/umccrise_2.3.1_pipeline_id'; // 61254f38-b56e-4576-a8a1-341e5c412d11 +export const umccriseIcav2PipelineWorkflowType = 'umccrise'; +export const umccriseIcav2PipelineWorkflowTypeVersion = '2.3.1'; +export const umccriseIcav2ServiceVersion = '2024.07.01'; +export const umccriseIcav2ReadyEventSource = 'orcabus.workflowmanager'; +export const umccriseIcav2EventSource = 'orcabus.umccrise'; +export const umccriseIcav2EventDetailType = 'WorkflowRunStateChange'; +export const umccriseStateMachinePrefix = 'umccriseSfn'; +export const umccriseDefaultGenomeVersion = '202303'; + // Mock Stack export const mockEventBusName = eventBusName; export const mockInstrumentRunTableName = 'stacky-instrument-run-table'; @@ -405,7 +455,6 @@ export const mockWgtsQcGlueTableName = 'stacky-wgts-qc-glue-table'; export const mockTnGlueTableName = 'stacky-tn-glue-table'; export const mockWtsGlueTableName = 'stacky-wts-glue-table'; export const mockUmccriseGlueTableName = 'stacky-umccrise-glue-table'; - export const mockWorkflowManagerTableName = 'stacky-workflow-manager-table'; // { diff --git a/config/stacks/umccrisePipelineManager.ts b/config/stacks/umccrisePipelineManager.ts new file mode 100644 index 000000000..b253645b8 --- /dev/null +++ b/config/stacks/umccrisePipelineManager.ts @@ -0,0 +1,58 @@ +import { + AppStage, + eventBusName, + icaEventPipeStackName, + icav2AccessTokenSecretName, + dragenIcav2ReferenceUriMappingSSMParameterPath, + umccriseIcav2PipelineIdSSMParameterPath, + umccriseIcav2PipelineManagerDynamodbTableName, + umccriseIcav2PipelineWorkflowType, + umccriseIcav2PipelineWorkflowTypeVersion, + umccriseIcav2ServiceVersion, + umccriseIcav2ReadyEventSource, + umccriseIcav2EventSource, + umccriseIcav2EventDetailType, + umccriseStateMachinePrefix, + umccriseDefaultGenomeVersion, + umccriseDynamoDbTableSSMArn, + umccriseDynamoDbTableSSMName, +} from '../constants'; +import { UmccriseIcav2PipelineManagerConfig } from '../../lib/workload/stateless/stacks/umccrise-pipeline-manager/deploy'; +import { UmccriseIcav2PipelineTableConfig } from '../../lib/workload/stateful/stacks/umccrise-pipeline-dynamo-db/deploy/stack'; + +// Stateful +export const getUmccriseIcav2PipelineTableStackProps = (): UmccriseIcav2PipelineTableConfig => { + return { + umccriseIcav2DynamodbTableArnSsmParameterPath: umccriseDynamoDbTableSSMArn, + umccriseIcav2DynamodbTableNameSsmParameterPath: umccriseDynamoDbTableSSMName, + dynamodbTableName: umccriseIcav2PipelineManagerDynamodbTableName, + }; +}; + +// Stateless +export const getUmccriseIcav2PipelineManagerStackProps = ( + stage: AppStage +): UmccriseIcav2PipelineManagerConfig => { + return { + /* ICAv2 Pipeline analysis essentials */ + icav2TokenSecretId: icav2AccessTokenSecretName[stage], // "/icav2/umccr-prod/service-production-jwt-token-secret-arn" + pipelineIdSsmPath: umccriseIcav2PipelineIdSSMParameterPath, // List of parameters the workflow session state machine will need access to + /* Table to store analyis metadata */ + dynamodbTableName: umccriseIcav2PipelineManagerDynamodbTableName, + /* Internal and external buses */ + eventBusName: eventBusName, + icaEventPipeName: `${icaEventPipeStackName}Pipe`, + /* Event handling */ + workflowType: umccriseIcav2PipelineWorkflowType, + workflowVersion: umccriseIcav2PipelineWorkflowTypeVersion, + serviceVersion: umccriseIcav2ServiceVersion, + triggerLaunchSource: umccriseIcav2ReadyEventSource, + internalEventSource: umccriseIcav2EventSource, + detailType: umccriseIcav2EventDetailType, + /* Names for statemachines */ + stateMachinePrefix: umccriseStateMachinePrefix, + /* SSM Workflow Parameters */ + defaultReferenceVersion: umccriseDefaultGenomeVersion, + referenceUriSsmPath: dragenIcav2ReferenceUriMappingSSMParameterPath, + }; +}; diff --git a/lib/workload/stateful/stacks/umccrise-pipeline-dynamo-db/deploy/stack.ts b/lib/workload/stateful/stacks/umccrise-pipeline-dynamo-db/deploy/stack.ts new file mode 100644 index 000000000..7dedab2c6 --- /dev/null +++ b/lib/workload/stateful/stacks/umccrise-pipeline-dynamo-db/deploy/stack.ts @@ -0,0 +1,54 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import { DynamodbPartitionedPipelineConstruct } from '../../../../components/dynamodb-partitioned-table'; + +export interface UmccriseIcav2PipelineTableConfig { + dynamodbTableName: string; + umccriseIcav2DynamodbTableArnSsmParameterPath: string; + umccriseIcav2DynamodbTableNameSsmParameterPath: string; +} + +export type UmccriseIcav2PipelineTableStackProps = UmccriseIcav2PipelineTableConfig & + cdk.StackProps; + +export class UmccriseIcav2PipelineTable extends cdk.Stack { + public readonly umccriseIcav2DynamodbTableArnSsmParameterPath: string; + public readonly umccriseIcav2DynamodbTableNameSsmParameterPath: string; + + constructor(scope: Construct, id: string, props: UmccriseIcav2PipelineTableStackProps) { + super(scope, id, props); + + /* + Initialise dynamodb table, where portal_run_id is the primary sort key + */ + const dynamodb_table = new DynamodbPartitionedPipelineConstruct( + this, + 'umccrise_icav2_pipeline_table', + { + tableName: props.dynamodbTableName, + } + ); + + /* + Generate a ssm parameter to store the table arn so it can be referred to be other stacks + */ + this.umccriseIcav2DynamodbTableArnSsmParameterPath = new ssm.StringParameter( + this, + 'umccrise_icav2_pipeline_table_arn_ssm_path', + { + parameterName: props.umccriseIcav2DynamodbTableArnSsmParameterPath, + stringValue: dynamodb_table.tableNameArn, + } + ).parameterName; + + this.umccriseIcav2DynamodbTableNameSsmParameterPath = new ssm.StringParameter( + this, + 'umccrise_icav2_pipeline_table_name_ssm_path', + { + parameterName: props.umccriseIcav2DynamodbTableNameSsmParameterPath, + stringValue: props.dynamodbTableName, + } + ).parameterName; + } +} diff --git a/lib/workload/stateful/statefulStackCollectionClass.ts b/lib/workload/stateful/statefulStackCollectionClass.ts index 732712267..e0cfe01fa 100644 --- a/lib/workload/stateful/statefulStackCollectionClass.ts +++ b/lib/workload/stateful/statefulStackCollectionClass.ts @@ -37,6 +37,10 @@ import { WtsIcav2PipelineTable, WtsIcav2PipelineTableStackProps, } from './stacks/wts-dynamo-db/deploy/stack'; +import { + UmccriseIcav2PipelineTable, + UmccriseIcav2PipelineTableStackProps, +} from './stacks/umccrise-pipeline-dynamo-db/deploy/stack'; export interface StatefulStackCollectionProps { dataBucketStackProps: DataBucketStackProps; @@ -49,6 +53,7 @@ export interface StatefulStackCollectionProps { wgtsQcIcav2PipelineTableStackProps: WgtsQcIcav2PipelineTableStackProps; tnIcav2PipelineTableStackProps: TnIcav2PipelineTableStackProps; wtsIcav2PipelineTableStackProps: WtsIcav2PipelineTableStackProps; + umccriseIcav2PipelineTableStackProps: UmccriseIcav2PipelineTableStackProps; BclConvertTableStackProps: BclConvertTableStackProps; stackyStatefulTablesStackProps: StackyStatefulTablesStackProps; } @@ -66,6 +71,7 @@ export class StatefulStackCollection { readonly wgtsQcIcav2PipelineTableStack: Stack; readonly tnIcav2PipelineTableStack: Stack; readonly wtsIcav2PipelineTableStack: Stack; + readonly umccriseIcav2PipelineTableStack: Stack; readonly BclConvertTableStack: Stack; readonly stackyStatefulTablesStack: Stack; @@ -144,6 +150,15 @@ export class StatefulStackCollection { } ); + this.umccriseIcav2PipelineTableStack = new UmccriseIcav2PipelineTable( + scope, + 'UmccriseIcav2PipelineTableStack', + { + ...this.createTemplateProps(env, 'UmccriseIcav2PipelineTableStack'), + ...statefulConfiguration.umccriseIcav2PipelineTableStackProps, + } + ); + this.BclConvertTableStack = new BclConvertTable(scope, 'BclConvertTableStack', { ...this.createTemplateProps(env, 'BclConvertTableStack'), ...statefulConfiguration.BclConvertTableStackProps, diff --git a/lib/workload/stateless/stacks/umccrise-manager/deploy/index.ts b/lib/workload/stateless/stacks/umccrise-pipeline-manager/deploy/index.ts similarity index 100% rename from lib/workload/stateless/stacks/umccrise-manager/deploy/index.ts rename to lib/workload/stateless/stacks/umccrise-pipeline-manager/deploy/index.ts diff --git a/lib/workload/stateless/stacks/umccrise-manager/lambdas/get_cwl_object_from_s3_inputs_py/get_cwl_objects_from_s3_inputs.py b/lib/workload/stateless/stacks/umccrise-pipeline-manager/lambdas/get_cwl_object_from_s3_inputs_py/get_cwl_object_from_s3_inputs.py similarity index 100% rename from lib/workload/stateless/stacks/umccrise-manager/lambdas/get_cwl_object_from_s3_inputs_py/get_cwl_objects_from_s3_inputs.py rename to lib/workload/stateless/stacks/umccrise-pipeline-manager/lambdas/get_cwl_object_from_s3_inputs_py/get_cwl_object_from_s3_inputs.py diff --git a/lib/workload/stateless/stacks/umccrise-manager/lambdas/get_cwl_object_from_s3_inputs_py/requirements.txt b/lib/workload/stateless/stacks/umccrise-pipeline-manager/lambdas/get_cwl_object_from_s3_inputs_py/requirements.txt similarity index 100% rename from lib/workload/stateless/stacks/umccrise-manager/lambdas/get_cwl_object_from_s3_inputs_py/requirements.txt rename to lib/workload/stateless/stacks/umccrise-pipeline-manager/lambdas/get_cwl_object_from_s3_inputs_py/requirements.txt diff --git a/lib/workload/stateless/stacks/umccrise-manager/lambdas/get_outputs_py/get_outputs.py b/lib/workload/stateless/stacks/umccrise-pipeline-manager/lambdas/get_outputs_py/get_outputs.py similarity index 100% rename from lib/workload/stateless/stacks/umccrise-manager/lambdas/get_outputs_py/get_outputs.py rename to lib/workload/stateless/stacks/umccrise-pipeline-manager/lambdas/get_outputs_py/get_outputs.py diff --git a/lib/workload/stateless/stacks/umccrise-manager/step_functions_templates/set_umccrise_cwl_inputs_sfn.asl.json b/lib/workload/stateless/stacks/umccrise-pipeline-manager/step_functions_templates/set_umccrise_cwl_inputs_sfn.asl.json similarity index 100% rename from lib/workload/stateless/stacks/umccrise-manager/step_functions_templates/set_umccrise_cwl_inputs_sfn.asl.json rename to lib/workload/stateless/stacks/umccrise-pipeline-manager/step_functions_templates/set_umccrise_cwl_inputs_sfn.asl.json diff --git a/lib/workload/stateless/stacks/umccrise-manager/step_functions_templates/set_umccrise_cwl_outputs_sfn.asl.json b/lib/workload/stateless/stacks/umccrise-pipeline-manager/step_functions_templates/set_umccrise_cwl_outputs_sfn.asl.json similarity index 100% rename from lib/workload/stateless/stacks/umccrise-manager/step_functions_templates/set_umccrise_cwl_outputs_sfn.asl.json rename to lib/workload/stateless/stacks/umccrise-pipeline-manager/step_functions_templates/set_umccrise_cwl_outputs_sfn.asl.json diff --git a/lib/workload/stateless/statelessStackCollectionClass.ts b/lib/workload/stateless/statelessStackCollectionClass.ts index 369d3efe1..34258feac 100644 --- a/lib/workload/stateless/statelessStackCollectionClass.ts +++ b/lib/workload/stateless/statelessStackCollectionClass.ts @@ -49,6 +49,10 @@ import { WtsIcav2PipelineManagerStack, WtsIcav2PipelineManagerStackProps, } from './stacks/transcriptome-pipeline-manager/deploy'; +import { + UmccriseIcav2PipelineManagerStack, + UmccriseIcav2PipelineManagerStackProps, +} from './stacks/umccrise-pipeline-manager/deploy'; export interface StatelessStackCollectionProps { metadataManagerStackProps: MetadataManagerStackProps; @@ -61,6 +65,7 @@ export interface StatelessStackCollectionProps { wgtsQcIcav2PipelineManagerStackProps: WgtsQcIcav2PipelineManagerStackProps; tnIcav2PipelineManagerStackProps: TnIcav2PipelineManagerStackProps; wtsIcav2PipelineManagerStackProps: WtsIcav2PipelineManagerStackProps; + umccriseIcav2PipelineManagerStackProps: UmccriseIcav2PipelineManagerStackProps; eventSchemaStackProps: SchemaStackProps; dataSchemaStackProps: SchemaStackProps; bclConvertManagerStackProps: BclConvertManagerStackProps; @@ -80,6 +85,7 @@ export class StatelessStackCollection { readonly wgtsQcIcav2PipelineManagerStack: Stack; readonly tnIcav2PipelineManagerStack: Stack; readonly wtsIcav2PipelineManagerStack: Stack; + readonly umccriseIcav2PipelineManagerStack: Stack; readonly eventSchemaStack: Stack; readonly dataSchemaStack: Stack; readonly bclConvertManagerStack: Stack; @@ -180,6 +186,15 @@ export class StatelessStackCollection { } ); + this.umccriseIcav2PipelineManagerStack = new UmccriseIcav2PipelineManagerStack( + scope, + 'UmccriseIcav2PipelineManagerStack', + { + ...this.createTemplateProps(env, 'UmccriseIcav2PipelineManagerStack'), + ...statelessConfiguration.umccriseIcav2PipelineManagerStackProps, + } + ); + this.bclConvertManagerStack = new BclConvertManagerStack(scope, 'BclConvertManagerStack', { ...this.createTemplateProps(env, 'BclConvertManagerStack'), ...statelessConfiguration.bclConvertManagerStackProps,