From 621c63df3d52ff741fb95912978f75c9d5997b72 Mon Sep 17 00:00:00 2001 From: alexiswl <8197659+alexiswl@users.noreply.github.com> Date: Mon, 29 Apr 2024 10:20:40 +1000 Subject: [PATCH] Added cttso pipeline to orcabus stack For both stateless and stateful --- config/config.ts | 13 ++ config/constants.ts | 32 ++++ config/stacks/cttsov2Icav2PipelineManager.ts | 57 ++++++ .../components/dynamodb-icav2-table/index.ts | 32 ++++ .../cttso_v2_pipeline_dynamo_db/Readme.md | 5 - .../deploy/.gitignore | 8 - .../deploy/.npmignore | 6 - .../deploy/README.md | 14 -- .../deploy/bin/deploy.ts | 24 --- .../deploy/cdk.json | 66 ------- .../deploy/constants.ts | 6 - .../deploy/jest.config.js | 8 - .../deploy/lib/cttso-v2-dynamodb-table.ts | 57 ------ .../deploy/package.json | 27 --- .../deploy/test/deploy.test.ts | 16 -- .../deploy/tsconfig.json | 31 ---- .../cttso-v2-pipeline-dynamo-db/Readme.md | 7 + .../deploy/stack.ts | 53 ++++++ .../stateful/statefulStackCollectionClass.ts | 15 ++ .../deploy/.gitignore | 8 - .../deploy/.npmignore | 6 - .../deploy/README.md | 14 -- .../deploy/bin/deploy.ts | 27 --- .../cttso_v2_pipeline_manager/deploy/cdk.json | 65 ------- .../deploy/constants.ts | 15 -- .../deploy/jest.config.js | 8 - .../deploy/lib/constructs/lambda_layer.ts | 44 ----- .../lib/stacks/cttso_v2_launch_stack.ts | 135 -------------- .../deploy/package.json | 27 --- .../deploy/test/deploy.test.ts | 17 -- .../deploy/tsconfig.json | 31 ---- .../cttso-v2-pipeline-manager}/Readme.md | 1 - .../cttsov2-icav2-manager/index.ts} | 164 +++++++----------- .../cttso-v2-pipeline-manager/deploy/stack.ts | 125 +++++++++++++ .../images/step_functions_image.png | Bin .../handler.py | 2 +- .../generate_copy_manifest_dict/handler.py | 2 +- .../lambdas/generate_db_uuid/handler.py | 0 .../launch_cttso_nextflow_pipeline/handler.py | 0 .../handler.py | 0 .../layers/poetry.lock | 0 .../layers/pyproject.toml | 0 .../__init__.py | 0 .../utils/__init__.py | 0 .../utils/aws_ssm_helpers.py | 0 .../utils/compression_helpers.py | 0 .../utils/path_helpers.py | 0 ..._v2_launch_workflow_state_machine.asl.json | 5 +- .../statelessStackCollectionClass.ts | 15 ++ 49 files changed, 420 insertions(+), 768 deletions(-) create mode 100644 config/stacks/cttsov2Icav2PipelineManager.ts create mode 100644 lib/workload/components/dynamodb-icav2-table/index.ts delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/Readme.md delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/.gitignore delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/.npmignore delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/README.md delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/bin/deploy.ts delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/cdk.json delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/constants.ts delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/jest.config.js delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/lib/cttso-v2-dynamodb-table.ts delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/package.json delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/test/deploy.test.ts delete mode 100644 lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/tsconfig.json create mode 100644 lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/Readme.md create mode 100644 lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/deploy/stack.ts delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.gitignore delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.npmignore delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/README.md delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/bin/deploy.ts delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/cdk.json delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/constants.ts delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/jest.config.js delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/lambda_layer.ts delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/stacks/cttso_v2_launch_stack.ts delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/package.json delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/test/deploy.test.ts delete mode 100644 lib/workload/stateless/cttso_v2_pipeline_manager/deploy/tsconfig.json rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/Readme.md (99%) rename lib/workload/stateless/{cttso_v2_pipeline_manager/deploy/lib/constructs/cttso_v2_launch_step_function.ts => stacks/cttso-v2-pipeline-manager/deploy/constructs/cttsov2-icav2-manager/index.ts} (66%) create mode 100644 lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack.ts rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/images/step_functions_image.png (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py (99%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/lambdas/generate_copy_manifest_dict/handler.py (99%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/lambdas/generate_db_uuid/handler.py (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/lambdas/launch_cttso_nextflow_pipeline/handler.py (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/lambdas/upload_samplesheet_to_cache_dir/handler.py (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/layers/poetry.lock (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/layers/pyproject.toml (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/layers/src/cttso_v2_pipeline_manager_tools/__init__.py (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/layers/src/cttso_v2_pipeline_manager_tools/utils/__init__.py (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/layers/src/cttso_v2_pipeline_manager_tools/utils/aws_ssm_helpers.py (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/layers/src/cttso_v2_pipeline_manager_tools/utils/compression_helpers.py (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/layers/src/cttso_v2_pipeline_manager_tools/utils/path_helpers.py (100%) rename lib/workload/stateless/{cttso_v2_pipeline_manager => stacks/cttso-v2-pipeline-manager}/step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json (99%) diff --git a/config/config.ts b/config/config.ts index b749362e6..40c0d427c 100644 --- a/config/config.ts +++ b/config/config.ts @@ -11,6 +11,10 @@ import { getFileManagerStackProps } from './stacks/fileManager'; import { getBsRunsUploadManagerStackProps } from './stacks/bsRunsUploadManager'; import { getICAv2CopyBatchUtilityStackProps } from './stacks/icav2CopyBatchUtility'; import { getBsshIcav2FastqCopyManagerStackProps } from './stacks/bsshIcav2FastqCopyManager'; +import { + getCttsov2Icav2PipelineManagerStackProps, + getCttsov2Icav2PipelineTableStackProps, +} from './stacks/cttsov2Icav2PipelineManager'; interface EnvironmentConfig { name: string; @@ -42,6 +46,7 @@ export const getEnvironmentConfig = ( sharedStackProps: getSharedStackProps(accountName), tokenServiceStackProps: getTokenServiceStackProps(), icaEventPipeStackProps: getIcaEventPipeStackProps(), + cttsov2Icav2PipelineTableStackProps: getCttsov2Icav2PipelineTableStackProps(), }, statelessConfig: { postgresManagerStackProps: getPostgresManagerStackProps(), @@ -52,6 +57,8 @@ export const getEnvironmentConfig = ( icav2CopyBatchUtilityStackProps: getICAv2CopyBatchUtilityStackProps(accountName), bsshIcav2FastqCopyManagerStackProps: getBsshIcav2FastqCopyManagerStackProps(accountName), + cttsov2Icav2PipelineManagerStackProps: + getCttsov2Icav2PipelineManagerStackProps(accountName), }, }, }; @@ -66,6 +73,7 @@ export const getEnvironmentConfig = ( sharedStackProps: getSharedStackProps(accountName), tokenServiceStackProps: getTokenServiceStackProps(), icaEventPipeStackProps: getIcaEventPipeStackProps(), + cttsov2Icav2PipelineTableStackProps: getCttsov2Icav2PipelineTableStackProps(), }, statelessConfig: { postgresManagerStackProps: getPostgresManagerStackProps(), @@ -76,6 +84,8 @@ export const getEnvironmentConfig = ( icav2CopyBatchUtilityStackProps: getICAv2CopyBatchUtilityStackProps(accountName), bsshIcav2FastqCopyManagerStackProps: getBsshIcav2FastqCopyManagerStackProps(accountName), + cttsov2Icav2PipelineManagerStackProps: + getCttsov2Icav2PipelineManagerStackProps(accountName), }, }, }; @@ -90,6 +100,7 @@ export const getEnvironmentConfig = ( sharedStackProps: getSharedStackProps(accountName), tokenServiceStackProps: getTokenServiceStackProps(), icaEventPipeStackProps: getIcaEventPipeStackProps(), + cttsov2Icav2PipelineTableStackProps: getCttsov2Icav2PipelineTableStackProps(), }, statelessConfig: { postgresManagerStackProps: getPostgresManagerStackProps(), @@ -100,6 +111,8 @@ export const getEnvironmentConfig = ( icav2CopyBatchUtilityStackProps: getICAv2CopyBatchUtilityStackProps(accountName), bsshIcav2FastqCopyManagerStackProps: getBsshIcav2FastqCopyManagerStackProps(accountName), + cttsov2Icav2PipelineManagerStackProps: + getCttsov2Icav2PipelineManagerStackProps(accountName), }, }, }; diff --git a/config/constants.ts b/config/constants.ts index 220ff8bc7..8a1464527 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -114,6 +114,38 @@ export const bsshFastqCopyManagerSSMArn = path.join( ); export const bsshFastqCopyManagerSfnName = 'bssh_fastq_copy_manager_sfn'; +/* +External resources required by the ctTSO v2 Stack +*/ +export const cttsov2Icav2PipelineIdSSMParameterPath = + '/icav2/umccr-prod/tso500_ctdna_2.1_pipeline_id'; + +/* +Resources generated by the ctTSO v2 Stack +*/ +export const cttsov2Icav2PipelineManagerSSMRoot = '/orcabus/ctTSOv2'; + +// Stateful +export const cttsov2Icav2PipelineManagerDynamodbTableName = 'ctTSOv2ICAv2AnalysesDynamoDBTable'; +export const cttsov2DynamoDbTableSSMName = path.join( + cttsov2Icav2PipelineManagerSSMRoot, + 'dynamodb_table_name' +); +export const cttsov2DynamoDbTableSSMArn = path.join( + cttsov2Icav2PipelineManagerSSMRoot, + 'dynamodb_table_arn' +); + +// Stateless +export const cttsov2Icav2PipelineSfnSSMName = path.join( + cttsov2Icav2PipelineManagerSSMRoot, + 'state_machine_name' +); +export const cttsov2Icav2PipelineSfnSSMArn = path.join( + cttsov2Icav2PipelineManagerSSMRoot, + 'state_machine_arn' +); + // const statelessConfig = { // multiSchemaConstructProps: { // registryName: regName, diff --git a/config/stacks/cttsov2Icav2PipelineManager.ts b/config/stacks/cttsov2Icav2PipelineManager.ts new file mode 100644 index 000000000..7ebe77620 --- /dev/null +++ b/config/stacks/cttsov2Icav2PipelineManager.ts @@ -0,0 +1,57 @@ +import { + AccountName, + eventBusName, + cttsov2Icav2PipelineIdSSMParameterPath, + icav2CopyBatchUtilityName, + cttsov2Icav2PipelineSfnSSMName, + cttsov2Icav2PipelineSfnSSMArn, + cttsov2Icav2PipelineManagerDynamodbTableName, + icav2AccessTokenSecretNameDev, + icav2AccessTokenSecretNameStg, + icav2AccessTokenSecretNameProd, + cttsov2DynamoDbTableSSMArn, + cttsov2DynamoDbTableSSMName, +} from '../constants'; +import { Cttsov2Icav2PipelineManagerConfig } from '../../lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack'; +import { Cttsov2Icav2PipelineTableConfig } from '../../lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/deploy/stack'; + +// Stateful +export const getCttsov2Icav2PipelineTableStackProps = (): Cttsov2Icav2PipelineTableConfig => { + return { + cttsov2_icav2_dynamodb_table_arn_ssm_parameter_path: cttsov2DynamoDbTableSSMArn, + cttsov2_icav2_dynamodb_table_name_ssm_parameter_path: cttsov2DynamoDbTableSSMName, + dynamodb_table_name: cttsov2Icav2PipelineManagerDynamodbTableName, + }; +}; + +// Stateless +export const getCttsov2Icav2PipelineManagerStackProps = ( + n: AccountName +): Cttsov2Icav2PipelineManagerConfig => { + const baseConfig = { + ssm_parameter_list: [cttsov2Icav2PipelineIdSSMParameterPath], + icav2_copy_batch_utility_state_machine_name: icav2CopyBatchUtilityName, + cttso_v2_launch_state_machine_arn_ssm_parameter_path: cttsov2Icav2PipelineSfnSSMArn, + cttso_v2_launch_state_machine_name_ssm_parameter_path: cttsov2Icav2PipelineSfnSSMName, + dynamodb_table_name: cttsov2Icav2PipelineManagerDynamodbTableName, + eventbus_name: eventBusName, + }; + + switch (n) { + case 'beta': + return { + ...baseConfig, + icav2_token_secret_id: icav2AccessTokenSecretNameDev, + }; + case 'gamma': + return { + ...baseConfig, + icav2_token_secret_id: icav2AccessTokenSecretNameStg, + }; + case 'prod': + return { + ...baseConfig, + icav2_token_secret_id: icav2AccessTokenSecretNameProd, + }; + } +}; diff --git a/lib/workload/components/dynamodb-icav2-table/index.ts b/lib/workload/components/dynamodb-icav2-table/index.ts new file mode 100644 index 000000000..5a848e67a --- /dev/null +++ b/lib/workload/components/dynamodb-icav2-table/index.ts @@ -0,0 +1,32 @@ +import { Construct } from 'constructs'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; + +export interface DynamodbIcav2PipelineConstructProps { + table_name: string; +} + +export class DynamodbIcav2PipelineConstruct extends Construct { + public readonly table_obj: dynamodb.ITableV2; + public readonly table_name_arn: string; + + constructor(scope: Construct, id: string, props: DynamodbIcav2PipelineConstructProps) { + super(scope, id); + + this.table_obj = new dynamodb.TableV2(this, 'dynamodb_icav2_pipeline_table', { + /* Either a db_uuid or an icav2 analysis id or a portal run id */ + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING, + }, + /* One of 'db_uuid', 'icav2_analysis_id', 'portal_run_id' */ + sortKey: { + name: 'id_type', + type: dynamodb.AttributeType.STRING, + }, + tableName: props.table_name, + }); + + // Set outputs + this.table_name_arn = this.table_obj.tableArn; + } +} diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/Readme.md b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/Readme.md deleted file mode 100644 index 5215420ef..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/Readme.md +++ /dev/null @@ -1,5 +0,0 @@ -# ctTSOV2 Pipeline Dynamo Db - -The dynamo db service that stores icav2 cttsov2 analyses - -Analysis are stored by their diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/.gitignore b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/.gitignore deleted file mode 100644 index f60797b6a..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -*.js -!jest.config.js -*.d.ts -node_modules - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/.npmignore b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/.npmignore deleted file mode 100644 index c1d6d45dc..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/.npmignore +++ /dev/null @@ -1,6 +0,0 @@ -*.ts -!*.d.ts - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/README.md b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/README.md deleted file mode 100644 index 9315fe5b9..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Welcome to your CDK TypeScript project - -This is a blank project for CDK development with TypeScript. - -The `cdk.json` file tells the CDK Toolkit how to execute your app. - -## Useful commands - -* `npm run build` compile typescript to js -* `npm run watch` watch for changes and compile -* `npm run test` perform the jest unit tests -* `npx cdk deploy` deploy this stack to your default AWS account/region -* `npx cdk diff` compare deployed stack with current state -* `npx cdk synth` emits the synthesized CloudFormation template diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/bin/deploy.ts b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/bin/deploy.ts deleted file mode 100644 index 8a00302e5..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/bin/deploy.ts +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env node -import * as cdk from 'aws-cdk-lib'; -import { ctTSOV2DynamoDBTable } from '../lib/cttso-v2-dynamodb-table'; - -const app = new cdk.App(); -new ctTSOV2DynamoDBTable(app, 'ctTSOV2DynamoDBTable', { - /* If you don't specify 'env', this stack will be environment-agnostic. - * Account/Region-dependent features and context lookups will not work, - * but a single synthesized template can be deployed anywhere. */ - - /* Uncomment the next line to specialize this stack for the AWS Account - * and Region that are implied by the current CLI configuration. */ - // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, - - /* Uncomment the next line if you know exactly what Account and Region you - * want to deploy the stack to. */ - // env: { account: '123456789012', region: 'us-east-1' }, - - /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */ - env: { - account: process.env.CDK_DEFAULT_ACCOUNT, - region: process.env.CDK_DEFAULT_REGION, - }, -}); diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/cdk.json b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/cdk.json deleted file mode 100644 index e8f04c654..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/cdk.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "app": "npx ts-node --prefer-ts-exts bin/deploy.ts", - "watch": { - "include": [ - "**" - ], - "exclude": [ - "README.md", - "cdk*.json", - "**/*.d.ts", - "**/*.js", - "tsconfig.json", - "package*.json", - "yarn.lock", - "node_modules", - "test" - ] - }, - "context": { - "@aws-cdk/aws-lambda:recognizeLayerVersion": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/core:target-partitions": [ - "aws", - "aws-cn" - ], - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/core:validateSnapshotRemovalPolicy": true, - "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, - "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, - "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, - "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, - "@aws-cdk/core:enablePartitionLiterals": true, - "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, - "@aws-cdk/aws-iam:standardizedServicePrincipals": true, - "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, - "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, - "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, - "@aws-cdk/aws-route53-patters:useCertificate": true, - "@aws-cdk/customresources:installLatestAwsSdkDefault": false, - "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, - "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, - "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, - "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, - "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, - "@aws-cdk/aws-redshift:columnId": true, - "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, - "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, - "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, - "@aws-cdk/aws-kms:aliasNameRef": true, - "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, - "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, - "@aws-cdk/aws-efs:denyAnonymousAccess": true, - "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, - "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, - "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, - "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, - "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, - "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, - "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, - "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true, - "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true - } -} diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/constants.ts b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/constants.ts deleted file mode 100644 index 75e494b45..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/constants.ts +++ /dev/null @@ -1,6 +0,0 @@ -const ROOT_SSM_PARAMETER_PATH = '/umccr/orcabus/stateful/cttso_v2_pipeline_dynamo_db'; - -export const DYNAMODB_TABLE_ARN_SSM_PARAMETER_PATH = `${ROOT_SSM_PARAMETER_PATH}/analysis_table_arn`; -export const DYNAMODB_TABLE_NAME_SSM_PARAMETER_PATH = `${ROOT_SSM_PARAMETER_PATH}/analysis_table_name`; - -export const DYNAMODB_TABLE_NAME = 'ctTSOv2ICAv2AnalysesDynamoDBTable'; diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/jest.config.js b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/jest.config.js deleted file mode 100644 index 44ead8540..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/jest.config.js +++ /dev/null @@ -1,8 +0,0 @@ -module.exports = { - testEnvironment: 'node', - roots: ['/test'], - testMatch: ['**/*.test.ts'], - transform: { - '^.+\\.tsx?$': 'ts-jest', - }, -}; diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/lib/cttso-v2-dynamodb-table.ts b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/lib/cttso-v2-dynamodb-table.ts deleted file mode 100644 index 65091e656..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/lib/cttso-v2-dynamodb-table.ts +++ /dev/null @@ -1,57 +0,0 @@ -import * as cdk from 'aws-cdk-lib'; -import { Construct } from 'constructs'; -// import * as sqs from 'aws-cdk-lib/aws-sqs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import { - DYNAMODB_TABLE_NAME, - DYNAMODB_TABLE_ARN_SSM_PARAMETER_PATH, - DYNAMODB_TABLE_NAME_SSM_PARAMETER_PATH, -} from '../constants'; - -export class ctTSOV2DynamoDBTable extends cdk.Stack { - public readonly cttso_v2_dynamodb_arn_ssm_parameter_path: string; - public readonly cttso_v2_dynamodb_table_name_ssm_parameter_path: string; - - constructor(scope: Construct, id: string, props?: cdk.StackProps) { - super(scope, id, props); - - /* - Initialise dynamodb table, where portal_run_id is the primary sort key - */ - const dynamodb_table = new dynamodb.TableV2(this, 'ctTSOv2DynamoDBTable', { - /* Either a db_uuid or an icav2 analysis id or a portal run id */ - partitionKey: { - name: 'id', - type: dynamodb.AttributeType.STRING, - }, - /* One of 'db_uuid', 'icav2_analysis_id', 'portal_run_id' */ - sortKey: { - name: 'id_type', - type: dynamodb.AttributeType.STRING, - }, - tableName: DYNAMODB_TABLE_NAME, - }); - - /* - Generate a ssm parameter to store the table arn so it can be referred to be other stacks - */ - this.cttso_v2_dynamodb_arn_ssm_parameter_path = new ssm.StringParameter( - this, - 'ctTSOv2DynamoDBTableArn', - { - parameterName: DYNAMODB_TABLE_ARN_SSM_PARAMETER_PATH, - stringValue: dynamodb_table.tableArn, - } - ).parameterName; - - this.cttso_v2_dynamodb_table_name_ssm_parameter_path = new ssm.StringParameter( - this, - 'ctTSOv2DynamoDBTableName', - { - parameterName: DYNAMODB_TABLE_NAME_SSM_PARAMETER_PATH, - stringValue: dynamodb_table.tableName, - } - ).parameterName; - } -} diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/package.json b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/package.json deleted file mode 100644 index f5839b2f0..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "deploy", - "version": "0.1.0", - "bin": { - "deploy": "bin/deploy.js" - }, - "scripts": { - "build": "tsc", - "watch": "tsc -w", - "test": "jest", - "cdk": "cdk" - }, - "devDependencies": { - "@types/jest": "^29.5.12", - "@types/node": "20.11.16", - "jest": "^29.7.0", - "ts-jest": "^29.1.2", - "aws-cdk": "2.128.0", - "ts-node": "^10.9.2", - "typescript": "~5.3.3" - }, - "dependencies": { - "aws-cdk-lib": "2.128.0", - "constructs": "^10.0.0", - "source-map-support": "^0.5.21" - } -} diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/test/deploy.test.ts b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/test/deploy.test.ts deleted file mode 100644 index e54577d6c..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/test/deploy.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -// import * as cdk from 'aws-cdk-lib'; -// import { Template } from 'aws-cdk-lib/assertions'; -// import * as Deploy from '../lib/deploy-stack'; - -// example test. To run these tests, uncomment this file along with the -// example resource in lib/cttso_v2_launch_stack.ts -test('SQS Queue Created', () => { - // const app = new cdk.App(); - // // WHEN - // const stack = new Deploy.DeployStack(app, 'MyTestStack'); - // // THEN - // const template = Template.fromStack(stack); - // template.hasResourceProperties('AWS::SQS::Queue', { - // VisibilityTimeout: 300 - // }); -}); diff --git a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/tsconfig.json b/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/tsconfig.json deleted file mode 100644 index aaa7dc510..000000000 --- a/lib/workload/stateful/cttso_v2_pipeline_dynamo_db/deploy/tsconfig.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2020", - "module": "commonjs", - "lib": [ - "es2020", - "dom" - ], - "declaration": true, - "strict": true, - "noImplicitAny": true, - "strictNullChecks": true, - "noImplicitThis": true, - "alwaysStrict": true, - "noUnusedLocals": false, - "noUnusedParameters": false, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": false, - "inlineSourceMap": true, - "inlineSources": true, - "experimentalDecorators": true, - "strictPropertyInitialization": false, - "typeRoots": [ - "./node_modules/@types" - ] - }, - "exclude": [ - "node_modules", - "cdk.out" - ] -} diff --git a/lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/Readme.md b/lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/Readme.md new file mode 100644 index 000000000..737d1ce96 --- /dev/null +++ b/lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/Readme.md @@ -0,0 +1,7 @@ +# ctTSOV2 Pipeline Dynamo Db + +The dynamo db service that stores icav2 cttsov2 analyses + +Analysis are stored by their portal_run_id, icav2_analysis_id and their database uuid. + + diff --git a/lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/deploy/stack.ts b/lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/deploy/stack.ts new file mode 100644 index 000000000..d88de79fc --- /dev/null +++ b/lib/workload/stateful/stacks/cttso-v2-pipeline-dynamo-db/deploy/stack.ts @@ -0,0 +1,53 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import { DynamodbIcav2PipelineConstruct } from '../../../../components/dynamodb-icav2-table'; + +export interface Cttsov2Icav2PipelineTableConfig { + dynamodb_table_name: string; + cttsov2_icav2_dynamodb_table_arn_ssm_parameter_path: string; + cttsov2_icav2_dynamodb_table_name_ssm_parameter_path: string; +} + +export type Cttsov2Icav2PipelineTableStackProps = Cttsov2Icav2PipelineTableConfig & cdk.StackProps; + +export class Cttsov2Icav2PipelineTable extends cdk.Stack { + public readonly cttsov2_icav2_dynamodb_table_arn_ssm_parameter_path: string; + public readonly cttsov2_icav2_dynamodb_table_name_ssm_parameter_path: string; + + constructor(scope: Construct, id: string, props: Cttsov2Icav2PipelineTableStackProps) { + super(scope, id, props); + + /* + Initialise dynamodb table, where portal_run_id is the primary sort key + */ + const dynamodb_table = new DynamodbIcav2PipelineConstruct( + this, + 'cttsov2_icav2_pipeline_table', + { + table_name: props.dynamodb_table_name, + } + ); + + /* + Generate a ssm parameter to store the table arn so it can be referred to be other stacks + */ + this.cttsov2_icav2_dynamodb_table_arn_ssm_parameter_path = new ssm.StringParameter( + this, + 'cttsov2_icav2_pipeline_table_arn_ssm_path', + { + parameterName: props.cttsov2_icav2_dynamodb_table_arn_ssm_parameter_path, + stringValue: dynamodb_table.table_name_arn, + } + ).parameterName; + + this.cttsov2_icav2_dynamodb_table_name_ssm_parameter_path = new ssm.StringParameter( + this, + 'cttsov2_icav2_pipeline_table_name_ssm_path', + { + parameterName: props.cttsov2_icav2_dynamodb_table_name_ssm_parameter_path, + stringValue: props.dynamodb_table_name, + } + ).parameterName; + } +} diff --git a/lib/workload/stateful/statefulStackCollectionClass.ts b/lib/workload/stateful/statefulStackCollectionClass.ts index cc13ba1f2..bbc3980db 100644 --- a/lib/workload/stateful/statefulStackCollectionClass.ts +++ b/lib/workload/stateful/statefulStackCollectionClass.ts @@ -4,11 +4,16 @@ import { Stack, Environment, StackProps } from 'aws-cdk-lib'; import { SharedStack, SharedStackProps } from './stacks/shared/stack'; import { TokenServiceStackProps, TokenServiceStack } from './stacks/token-service/deploy/stack'; import { IcaEventPipeStack, IcaEventPipeStackProps } from './stacks/ica-event-pipe/stack'; +import { + Cttsov2Icav2PipelineTable, + Cttsov2Icav2PipelineTableStackProps, +} from './stacks/cttso-v2-pipeline-dynamo-db/deploy/stack'; export interface StatefulStackCollectionProps { sharedStackProps: SharedStackProps; tokenServiceStackProps: TokenServiceStackProps; icaEventPipeStackProps: IcaEventPipeStackProps; + cttsov2Icav2PipelineTableStackProps: Cttsov2Icav2PipelineTableStackProps; } export class StatefulStackCollection { @@ -17,6 +22,7 @@ export class StatefulStackCollection { readonly sharedStack: Stack; readonly tokenServiceStack: Stack; readonly icaEventPipeStack: Stack; + readonly cttsov2Icav2PipelineTableStack: Stack; constructor( scope: Construct, @@ -37,6 +43,15 @@ export class StatefulStackCollection { ...this.createTemplateProps(env, 'IcaEventPipeStack'), ...statefulConfiguration.icaEventPipeStackProps, }); + + this.cttsov2Icav2PipelineTableStack = new Cttsov2Icav2PipelineTable( + scope, + 'Cttsov2Icav2PipelineTableStack', + { + ...this.createTemplateProps(env, 'Cttsov2Icav2PipelineTableStack'), + ...statefulConfiguration.cttsov2Icav2PipelineTableStackProps, + } + ); } /** diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.gitignore b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.gitignore deleted file mode 100644 index f60797b6a..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -*.js -!jest.config.js -*.d.ts -node_modules - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.npmignore b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.npmignore deleted file mode 100644 index c1d6d45dc..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.npmignore +++ /dev/null @@ -1,6 +0,0 @@ -*.ts -!*.d.ts - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/README.md b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/README.md deleted file mode 100644 index 9315fe5b9..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Welcome to your CDK TypeScript project - -This is a blank project for CDK development with TypeScript. - -The `cdk.json` file tells the CDK Toolkit how to execute your app. - -## Useful commands - -* `npm run build` compile typescript to js -* `npm run watch` watch for changes and compile -* `npm run test` perform the jest unit tests -* `npx cdk deploy` deploy this stack to your default AWS account/region -* `npx cdk diff` compare deployed stack with current state -* `npx cdk synth` emits the synthesized CloudFormation template diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/bin/deploy.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/bin/deploy.ts deleted file mode 100644 index 0ebfe4736..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/bin/deploy.ts +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env node -import 'source-map-support/register'; -import * as cdk from 'aws-cdk-lib'; -import { ctTSOV2LaunchStateMachineStack } from '../lib/stacks/cttso_v2_launch_stack'; -import { - CTTSO_V2_LAUNCH_STATE_MACHINE_ARN_SSM_PARAMETER_PATH, - CTTSO_V2_LAUNCH_STATE_MACHINE_NAME_SSM_PARAMETER_PATH, - DYNAMODB_TABLE_NAME_SSM_PARAMETER_PATH, - ICAV2_ACCESS_TOKEN_SECRET_ID, - ICAV2_COPY_BATCH_UTILITY_STATE_MACHINE_NAME_SSM_PARAMETER_PATH, ORCABUS_EVENT_NAME, - SSM_PARAMETER_LIST_FOR_CTTSO_LAUNCH_LAMBDAS, -} from '../constants'; - -const app = new cdk.App(); -new ctTSOV2LaunchStateMachineStack(app, 'ctTSOv2LaunchStatemachineStack', { - icav2_token_secret_id: ICAV2_ACCESS_TOKEN_SECRET_ID, - ssm_parameter_list: SSM_PARAMETER_LIST_FOR_CTTSO_LAUNCH_LAMBDAS, - icav2_copy_batch_utility_state_machine_name_ssm_parameter_path: ICAV2_COPY_BATCH_UTILITY_STATE_MACHINE_NAME_SSM_PARAMETER_PATH, - cttso_v2_launch_state_machine_arn_ssm_parameter_path: CTTSO_V2_LAUNCH_STATE_MACHINE_ARN_SSM_PARAMETER_PATH, - cttso_v2_launch_state_machine_name_ssm_parameter_path: CTTSO_V2_LAUNCH_STATE_MACHINE_NAME_SSM_PARAMETER_PATH, - dynamodb_table_ssm_parameter_path: DYNAMODB_TABLE_NAME_SSM_PARAMETER_PATH, - eventbus_name: ORCABUS_EVENT_NAME, - env: { - account: process.env.CDK_DEFAULT_ACCOUNT, - region: process.env.CDK_DEFAULT_REGION - }, -}); \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/cdk.json b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/cdk.json deleted file mode 100644 index d8d9da252..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/cdk.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "app": "npx ts-node --prefer-ts-exts bin/deploy.ts", - "watch": { - "include": [ - "**" - ], - "exclude": [ - "README.md", - "cdk*.json", - "**/*.d.ts", - "**/*.js", - "tsconfig.json", - "package*.json", - "yarn.lock", - "node_modules", - "test" - ] - }, - "context": { - "@aws-cdk/aws-lambda:recognizeLayerVersion": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/core:target-partitions": [ - "aws", - "aws-cn" - ], - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/core:validateSnapshotRemovalPolicy": true, - "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, - "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, - "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, - "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, - "@aws-cdk/core:enablePartitionLiterals": true, - "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, - "@aws-cdk/aws-iam:standardizedServicePrincipals": true, - "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, - "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, - "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, - "@aws-cdk/aws-route53-patters:useCertificate": true, - "@aws-cdk/customresources:installLatestAwsSdkDefault": false, - "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, - "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, - "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, - "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, - "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, - "@aws-cdk/aws-redshift:columnId": true, - "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, - "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, - "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, - "@aws-cdk/aws-kms:aliasNameRef": true, - "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, - "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, - "@aws-cdk/aws-efs:denyAnonymousAccess": true, - "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, - "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, - "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, - "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, - "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, - "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, - "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, - "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true - } -} diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/constants.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/constants.ts deleted file mode 100644 index 622258af6..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/constants.ts +++ /dev/null @@ -1,15 +0,0 @@ - -const STATEFUL_ROOT_SSM_PARAMETER_PATH = "/umccr/orcabus/stateful/cttso_v2_pipeline_dynamo_db"; -export const DYNAMODB_TABLE_ARN_SSM_PARAMETER_PATH = `${STATEFUL_ROOT_SSM_PARAMETER_PATH}/analysis_table_arn`; -export const DYNAMODB_TABLE_NAME_SSM_PARAMETER_PATH = `${STATEFUL_ROOT_SSM_PARAMETER_PATH}/analysis_table_name`; - -export const ICAV2_ACCESS_TOKEN_SECRET_ID = "ICAv2Jwticav2-credentials-umccr-service-user-trial" -export const ICAV2_COPY_BATCH_UTILITY_STATE_MACHINE_NAME_SSM_PARAMETER_PATH = "/icav2_copy_batch_utility/state_machine_name_batch" -export const PIPELINE_ID_SSM_PARAMETER_PATH = "/icav2/umccr-prod/tso500_ctdna_2.1_pipeline_id" -export const CTTSO_V2_LAUNCH_STATE_MACHINE_ARN_SSM_PARAMETER_PATH = "/icav2/umccr-prod/cttso_launch_state_machine_arn" -export const CTTSO_V2_LAUNCH_STATE_MACHINE_NAME_SSM_PARAMETER_PATH = "/icav2/umccr-prod/cttso_launch_state_machine_name" -export const ORCABUS_EVENT_NAME = "OrcaBusMain" - -export const SSM_PARAMETER_LIST_FOR_CTTSO_LAUNCH_LAMBDAS = [ - PIPELINE_ID_SSM_PARAMETER_PATH -] \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/jest.config.js b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/jest.config.js deleted file mode 100644 index 08263b895..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/jest.config.js +++ /dev/null @@ -1,8 +0,0 @@ -module.exports = { - testEnvironment: 'node', - roots: ['/test'], - testMatch: ['**/*.test.ts'], - transform: { - '^.+\\.tsx?$': 'ts-jest' - } -}; diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/lambda_layer.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/lambda_layer.ts deleted file mode 100644 index b84793336..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/lambda_layer.ts +++ /dev/null @@ -1,44 +0,0 @@ - -import { Construct } from 'constructs' -import { PythonLayerVersion } from '@aws-cdk/aws-lambda-python-alpha'; -import * as lambda from 'aws-cdk-lib/aws-lambda'; - -export interface LambdaLayerConstructProps { - layer_directory: string -} - -export class LambdaLayerConstruct extends Construct { - - public readonly lambda_layer_arn: string; - public readonly lambda_layer_version_obj: PythonLayerVersion; - - constructor(scope: Construct, id: string, props: LambdaLayerConstructProps) { - super(scope, id); - - this.lambda_layer_version_obj = new PythonLayerVersion( - this, - 'cttso_v2_tool_layer', - { - entry: props.layer_directory, - compatibleRuntimes: [lambda.Runtime.PYTHON_3_11], - compatibleArchitectures: [lambda.Architecture.X86_64], - license: 'GPL3', - description: 'A layer to enable the cttso_v2 manager tools layer', - bundling: { - commandHooks: { - beforeBundling(inputDir: string, outputDir: string): string[] { - return []; - }, - afterBundling(inputDir: string, outputDir: string): string[] { - return [ - `python -m pip install ${inputDir} -t ${outputDir}`, - ]; - }, - }, - }, - }); - - // Set outputs - this.lambda_layer_arn = this.lambda_layer_version_obj.layerVersionArn; - } -} \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/stacks/cttso_v2_launch_stack.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/stacks/cttso_v2_launch_stack.ts deleted file mode 100644 index 06513f75b..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/stacks/cttso_v2_launch_stack.ts +++ /dev/null @@ -1,135 +0,0 @@ -import * as cdk from 'aws-cdk-lib'; -import { Construct } from 'constructs'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import * as events from 'aws-cdk-lib/aws-events'; -import { ctTSOv2LaunchStepFunctionStateMachineConstruct } from '../constructs/cttso_v2_launch_step_function'; -import { LambdaLayerConstruct } from '../constructs/lambda_layer'; - -interface ctTSOV2LaunchStateMachineStackProps extends cdk.StackProps { - icav2_token_secret_id: string; // "/icav2/umccr-prod/service-production-jwt-token-secret-arn" - ssm_parameter_list: string[]; // List of parameters the workflow session state machine will need access to - icav2_copy_batch_utility_state_machine_name_ssm_parameter_path: string; - cttso_v2_launch_state_machine_arn_ssm_parameter_path: string; - cttso_v2_launch_state_machine_name_ssm_parameter_path: string; - dynamodb_table_ssm_parameter_path: string; - eventbus_name: string; -} - -export class ctTSOV2LaunchStateMachineStack extends cdk.Stack { - - public readonly cttso_v2_launch_state_machine_arn: string; - public readonly cttso_v2_launch_state_machine_name: string; - public readonly cttso_v2_launch_state_machine_arn_ssm_parameter_path: string; - public readonly cttso_v2_launch_state_machine_name_ssm_parameter_path: string; - - constructor(scope: Construct, id: string, props: ctTSOV2LaunchStateMachineStackProps) { - super(scope, id, props); - - // Get the copy batch state machine value from lookup - // This will allow us to copy the fastq files to the correct location for launch - const icav2_copy_batch_stack_state_machine_name_obj = ssm.StringParameter.fromStringParameterName( - this, - 'icav2_copy_batch_state_machine_ssm_parameter', - props.icav2_copy_batch_utility_state_machine_name_ssm_parameter_path, - ); - const icav2_copy_batch_stack_state_machine_obj = sfn.StateMachine.fromStateMachineName( - this, - 'icav2_copy_batch_state_machine', - icav2_copy_batch_stack_state_machine_name_obj.stringValue, - ); - - // Get the dynamodb table ssm parameter - const dynamodb_table_ssm_parameter_obj = ssm.StringParameter.fromStringParameterName( - this, - 'dynamodb_table_ssm_parameter', - props.dynamodb_table_ssm_parameter_path, - ); - // Get dynamodb table for construct - const dynamodb_table_obj = dynamodb.TableV2.fromTableName( - this, - 'dynamodb_table', - dynamodb_table_ssm_parameter_obj.stringValue, - ); - - // Get ICAv2 Access token secret object for construct - const icav2_access_token_secret_obj = secretsManager.Secret.fromSecretNameV2( - this, 'Icav2SecretsObject', - props.icav2_token_secret_id, - ); - - // Get lambda layer object - const lambda_layer_obj = new LambdaLayerConstruct( - this, 'lambda_layer', { - layer_directory: __dirname + '/../../../layers/', // __dirname + '/../../../layers - }); - - // Set ssm parameter object list - const ssm_parameter_obj_list = props.ssm_parameter_list.map( - (ssm_parameter_path: string) => ssm.StringParameter.fromStringParameterName( - this, - ssm_parameter_path, - ssm_parameter_path, - ), - ); - - // Get event bus - const eventbus_obj = events.EventBus.fromEventBusName( - this, - 'eventbus', - props.eventbus_name, - ); - - const cttso_v2_launch_state_machine = new ctTSOv2LaunchStepFunctionStateMachineConstruct( - this, - id, - { - /* Stack Objects */ - dynamodb_table_obj: dynamodb_table_obj, - icav2_access_token_secret_obj: icav2_access_token_secret_obj, - lambda_layer_obj: lambda_layer_obj, - icav2_copy_batch_state_machine_obj: icav2_copy_batch_stack_state_machine_obj, - ssm_parameter_obj_list: ssm_parameter_obj_list, - eventbus_obj: eventbus_obj, - /* Lambdas paths */ - generate_db_uuid_lambda_path: __dirname + '/../../../lambdas/generate_db_uuid', // __dirname + '/../../../lambdas/generate_uuid' - generate_trimmed_samplesheet_lambda_path: __dirname + '/../../../lambdas/generate_and_trim_cttso_samplesheet_dict', // __dirname + '/../../../lambdas/generate_and_trim_cttso_samplesheet_dict' - upload_samplesheet_to_cache_dir_lambda_path: __dirname + '/../../../lambdas/upload_samplesheet_to_cache_dir', // __dirname + '/../../../lambdas/upload_samplesheet_to_cache_dir' - generate_copy_manifest_dict_lambda_path: __dirname + '/../../../lambdas/generate_copy_manifest_dict', // __dirname + '/../../../lambdas/generate_copy_manifest_dict' - launch_cttso_nextflow_pipeline_lambda_path: __dirname + '/../../../lambdas/launch_cttso_nextflow_pipeline', // __dirname + '../launch_cttso_nextflow_pipeline' - /* Step function templates */ - workflow_definition_body_path: __dirname + '/../../../step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json', // __dirname + '/../../../step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json' - }, - ); - - // Set outputs - this.cttso_v2_launch_state_machine_arn = cttso_v2_launch_state_machine.cttso_v2_launch_state_machine_arn; - this.cttso_v2_launch_state_machine_name = cttso_v2_launch_state_machine.cttso_v2_launch_state_machine_name; - this.cttso_v2_launch_state_machine_arn_ssm_parameter_path = props.cttso_v2_launch_state_machine_arn_ssm_parameter_path; - this.cttso_v2_launch_state_machine_name_ssm_parameter_path = props.cttso_v2_launch_state_machine_name_ssm_parameter_path; - - // Set SSM Parameter for the state machine arn - new ssm.StringParameter( - this, - 'state_machine_arn_ssm_parameter', - { - parameterName: this.cttso_v2_launch_state_machine_arn_ssm_parameter_path, - stringValue: this.cttso_v2_launch_state_machine_arn, - }, - ); - - // Set SSM Parameter for the state machine name - new ssm.StringParameter( - this, - 'state_machine_name_ssm_parameter', - { - parameterName: this.cttso_v2_launch_state_machine_name_ssm_parameter_path, - stringValue: this.cttso_v2_launch_state_machine_name, - }, - ); - - } - -} diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/package.json b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/package.json deleted file mode 100644 index d858db3df..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "deploy", - "version": "0.1.0", - "bin": { - "deploy": "bin/deploy.js" - }, - "scripts": { - "build": "tsc", - "watch": "tsc -w", - "test": "jest", - "cdk": "cdk" - }, - "devDependencies": { - "@types/jest": "^29.5.11", - "@types/node": "20.11.14", - "jest": "^29.7.0", - "ts-jest": "^29.1.2", - "aws-cdk": "2.126.0", - "ts-node": "^10.9.2", - "typescript": "~5.3.3" - }, - "dependencies": { - "aws-cdk-lib": "2.126.0", - "constructs": "^10.0.0", - "source-map-support": "^0.5.21" - } -} \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/test/deploy.test.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/test/deploy.test.ts deleted file mode 100644 index d3f2e9b24..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/test/deploy.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -// import * as cdk from 'aws-cdk-lib'; -// import { Template } from 'aws-cdk-lib/assertions'; -// import * as Deploy from '../lib/deploy-stack'; - -// example test. To run these tests, uncomment this file along with the -// example resource in lib/cttso_v2_launch_stack.ts -test('SQS Queue Created', () => { -// const app = new cdk.App(); -// // WHEN -// const stack = new Deploy.DeployStack(app, 'MyTestStack'); -// // THEN -// const template = Template.fromStack(stack); - -// template.hasResourceProperties('AWS::SQS::Queue', { -// VisibilityTimeout: 300 -// }); -}); diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/tsconfig.json b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/tsconfig.json deleted file mode 100644 index aaa7dc510..000000000 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/tsconfig.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2020", - "module": "commonjs", - "lib": [ - "es2020", - "dom" - ], - "declaration": true, - "strict": true, - "noImplicitAny": true, - "strictNullChecks": true, - "noImplicitThis": true, - "alwaysStrict": true, - "noUnusedLocals": false, - "noUnusedParameters": false, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": false, - "inlineSourceMap": true, - "inlineSources": true, - "experimentalDecorators": true, - "strictPropertyInitialization": false, - "typeRoots": [ - "./node_modules/@types" - ] - }, - "exclude": [ - "node_modules", - "cdk.out" - ] -} diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/Readme.md b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/Readme.md similarity index 99% rename from lib/workload/stateless/cttso_v2_pipeline_manager/Readme.md rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/Readme.md index bb0f5dad1..fc2b0a964 100644 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/Readme.md +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/Readme.md @@ -166,6 +166,5 @@ Take the fastq output locations, samplesheet location and metadata generated in ### External Parameters required by CDK ``` -/icav2_copy_batch_utility/state_machine_arn_batch /icav2/umccr-prod/tso500_ctdna_2.1_pipeline_id ``` diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/cttso_v2_launch_step_function.ts b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/constructs/cttsov2-icav2-manager/index.ts similarity index 66% rename from lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/cttso_v2_launch_step_function.ts rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/constructs/cttsov2-icav2-manager/index.ts index 72bf8479d..02132ef2a 100644 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/cttso_v2_launch_step_function.ts +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/constructs/cttsov2-icav2-manager/index.ts @@ -6,20 +6,19 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager' +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; import * as events from 'aws-cdk-lib/aws-events'; import * as events_targets from 'aws-cdk-lib/aws-events-targets'; import { DefinitionBody } from 'aws-cdk-lib/aws-stepfunctions'; import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; -import { LambdaLayerConstruct } from './lambda_layer'; -interface ctTSOv2LaunchStepFunctionConstructProps { +interface Cttsov2Icav2ManagerConstructProps { /* Stack Objects */ - dynamodb_table_obj: dynamodb.ITableV2, - icav2_access_token_secret_obj: secretsManager.ISecret, - lambda_layer_obj: LambdaLayerConstruct, - icav2_copy_batch_state_machine_obj: sfn.IStateMachine, + dynamodb_table_obj: dynamodb.ITableV2; + icav2_access_token_secret_obj: secretsManager.ISecret; + lambda_layer_obj: lambda.ILayerVersion; + icav2_copy_batch_state_machine_obj: sfn.IStateMachine; ssm_parameter_obj_list: ssm.IStringParameter[]; // List of parameters the workflow session state machine will need access to eventbus_obj: events.IEventBus; /* Lambdas paths */ @@ -32,12 +31,11 @@ interface ctTSOv2LaunchStepFunctionConstructProps { workflow_definition_body_path: string; // __dirname + '/../../../step_functions_templates/cttso_v2_launch_step_function.json' } -export class ctTSOv2LaunchStepFunctionStateMachineConstruct extends Construct { - +export class Cttsov2Icav2PipelineManagerConstruct extends Construct { public readonly cttso_v2_launch_state_machine_name: string; public readonly cttso_v2_launch_state_machine_arn: string; - constructor(scope: Construct, id: string, props: ctTSOv2LaunchStepFunctionConstructProps) { + constructor(scope: Construct, id: string, props: Cttsov2Icav2ManagerConstructProps) { super(scope, id); // launch_cttso_nextflow_pipeline lambda @@ -47,34 +45,27 @@ export class ctTSOv2LaunchStepFunctionStateMachineConstruct extends Construct { { entry: props.launch_cttso_nextflow_pipeline_lambda_path, runtime: lambda.Runtime.PYTHON_3_11, + architecture: lambda.Architecture.ARM_64, index: 'handler.py', handler: 'handler', memorySize: 1024, - // @ts-ignore - layers: [props.lambda_layer_obj.lambda_layer_version_obj], - // @ts-ignore + layers: [props.lambda_layer_obj], timeout: Duration.seconds(60), environment: { - "ICAV2_ACCESS_TOKEN_SECRET_ID": props.icav2_access_token_secret_obj.secretName - } + ICAV2_ACCESS_TOKEN_SECRET_ID: props.icav2_access_token_secret_obj.secretName, + }, } ); // Add icav2 secrets permissions to lambda props.icav2_access_token_secret_obj.grantRead( - // @ts-ignore launch_cttso_nextflow_pipeline_lambda_obj.role - ) + ); // Add each of the ssm parameters to the lambda role policy - props.ssm_parameter_obj_list.forEach( - (ssm_parameter_obj) => { - ssm_parameter_obj.grantRead( - // @ts-ignore - launch_cttso_nextflow_pipeline_lambda_obj.role - ) - }, - ); + props.ssm_parameter_obj_list.forEach((ssm_parameter_obj) => { + ssm_parameter_obj.grantRead(launch_cttso_nextflow_pipeline_lambda_obj.role); + }); // generate_db_uuid_lambda_path lambda // Doesnt need any ssm parameters @@ -84,12 +75,11 @@ export class ctTSOv2LaunchStepFunctionStateMachineConstruct extends Construct { { entry: props.generate_db_uuid_lambda_path, runtime: lambda.Runtime.PYTHON_3_11, + architecture: lambda.Architecture.ARM_64, index: 'handler.py', handler: 'handler', memorySize: 1024, - // @ts-ignore - layers: [props.lambda_layer_obj.lambda_layer_version_obj], - // @ts-ignore + layers: [props.lambda_layer_obj], timeout: Duration.seconds(60), } ); @@ -102,12 +92,11 @@ export class ctTSOv2LaunchStepFunctionStateMachineConstruct extends Construct { { entry: props.generate_copy_manifest_dict_lambda_path, runtime: lambda.Runtime.PYTHON_3_11, + architecture: lambda.Architecture.ARM_64, index: 'handler.py', handler: 'handler', memorySize: 1024, - // @ts-ignore - layers: [props.lambda_layer_obj.lambda_layer_version_obj], - // @ts-ignore + layers: [props.lambda_layer_obj], timeout: Duration.seconds(60), } ); @@ -120,12 +109,11 @@ export class ctTSOv2LaunchStepFunctionStateMachineConstruct extends Construct { { entry: props.generate_trimmed_samplesheet_lambda_path, runtime: lambda.Runtime.PYTHON_3_11, + architecture: lambda.Architecture.ARM_64, index: 'handler.py', handler: 'handler', memorySize: 1024, - // @ts-ignore - layers: [props.lambda_layer_obj.lambda_layer_version_obj], - // @ts-ignore + layers: [props.lambda_layer_obj], timeout: Duration.seconds(60), } ); @@ -137,55 +125,54 @@ export class ctTSOv2LaunchStepFunctionStateMachineConstruct extends Construct { { entry: props.upload_samplesheet_to_cache_dir_lambda_path, runtime: lambda.Runtime.PYTHON_3_11, + architecture: lambda.Architecture.ARM_64, index: 'handler.py', handler: 'handler', memorySize: 1024, - // @ts-ignore - layers: [props.lambda_layer_obj.lambda_layer_version_obj], - // @ts-ignore + layers: [props.lambda_layer_obj], timeout: Duration.seconds(60), environment: { - "ICAV2_ACCESS_TOKEN_SECRET_ID": props.icav2_access_token_secret_obj.secretName - } + ICAV2_ACCESS_TOKEN_SECRET_ID: props.icav2_access_token_secret_obj.secretName, + }, } ); // Add icav2 secrets permissions to lambda props.icav2_access_token_secret_obj.grantRead( - // @ts-ignore upload_samplesheet_to_cache_dir_lambda_obj.role - ) + ); // Add each of the ssm parameters to the lambda role policy - props.ssm_parameter_obj_list.forEach( - (ssm_parameter_obj) => { - ssm_parameter_obj.grantRead( - // @ts-ignore - upload_samplesheet_to_cache_dir_lambda_obj.role - ) - }, - ); + props.ssm_parameter_obj_list.forEach((ssm_parameter_obj) => { + ssm_parameter_obj.grantRead(upload_samplesheet_to_cache_dir_lambda_obj.role); + }); // Specify the statemachine and replace the arn placeholders with the lambda arns defined above - const stateMachine = new sfn.StateMachine(this, 'cttso_v2_launch_step_functions_state_machine', { - // defintiontemplate - definitionBody: DefinitionBody.fromFile(props.workflow_definition_body_path), - // definitionSubstitutions - definitionSubstitutions: { - /* Lambda arns */ - '__generate_db_uuid__': generate_db_uuid_lambda_obj.functionArn, - '__launch_cttso_nextflow_pipeline__': launch_cttso_nextflow_pipeline_lambda_obj.functionArn, - '__generate_copy_manifest_dict__': generate_copy_manifest_dict_lambda_obj.functionArn, - '__generate_trimmed_samplesheet__': generate_trimmed_samplesheet_lambda_obj.functionArn, - '__upload_samplesheet_to_cache_dir__': upload_samplesheet_to_cache_dir_lambda_obj.functionArn, - /* Subfunction state machines */ - '__copy_batch_data_state_machine_arn__': props.icav2_copy_batch_state_machine_obj.stateMachineArn, - /* Dynamodb tables */ - '__table_name__': props.dynamodb_table_obj.tableName, - /* Event bus to push to */ - '__eventbus_name__': props.eventbus_obj.eventBusName - }, - }); + const stateMachine = new sfn.StateMachine( + this, + 'cttso_v2_launch_step_functions_state_machine', + { + // defintiontemplate + definitionBody: DefinitionBody.fromFile(props.workflow_definition_body_path), + // definitionSubstitutions + definitionSubstitutions: { + /* Lambda arns */ + __generate_db_uuid__: generate_db_uuid_lambda_obj.functionArn, + __launch_cttso_nextflow_pipeline__: launch_cttso_nextflow_pipeline_lambda_obj.functionArn, + __generate_copy_manifest_dict__: generate_copy_manifest_dict_lambda_obj.functionArn, + __generate_trimmed_samplesheet__: generate_trimmed_samplesheet_lambda_obj.functionArn, + __upload_samplesheet_to_cache_dir__: + upload_samplesheet_to_cache_dir_lambda_obj.functionArn, + /* Subfunction state machines */ + __copy_batch_data_state_machine_arn__: + props.icav2_copy_batch_state_machine_obj.stateMachineArn, + /* Dynamodb tables */ + __table_name__: props.dynamodb_table_obj.tableName, + /* Event bus to push to */ + __eventbus_name__: props.eventbus_obj.eventBusName, + }, + } + ); // Grant lambda invoke permissions to the state machine [ @@ -194,41 +181,26 @@ export class ctTSOv2LaunchStepFunctionStateMachineConstruct extends Construct { generate_copy_manifest_dict_lambda_obj, generate_trimmed_samplesheet_lambda_obj, upload_samplesheet_to_cache_dir_lambda_obj, - ].forEach( - (lambda_obj) => { - lambda_obj.grantInvoke( - // @ts-ignore - stateMachine.role - ) - } - ) + ].forEach((lambda_obj) => { + lambda_obj.grantInvoke(stateMachine.role); + }); // Allow state machine to read/write to dynamodb table - props.dynamodb_table_obj.grantReadWriteData( - stateMachine.role - ) + props.dynamodb_table_obj.grantReadWriteData(stateMachine.role); // Because we run a nested state machine, we need to add the permissions to the state machine role // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr stateMachine.addToRolePolicy( - new iam.PolicyStatement( - { - resources: [ - `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, - ], - actions: [ - 'events:PutTargets', - 'events:PutRule', - 'events:DescribeRule', - ], - }, - ), + new iam.PolicyStatement({ + resources: [ + `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, + ], + actions: ['events:PutTargets', 'events:PutRule', 'events:DescribeRule'], + }) ); // Add state machine execution permissions to stateMachine role - props.icav2_copy_batch_state_machine_obj.grantStartExecution( - stateMachine.role - ) + props.icav2_copy_batch_state_machine_obj.grantStartExecution(stateMachine.role); // Trigger state machine on event const rule = new events.Rule(this, 'cttso_v2_launch_step_function_rule', { @@ -253,13 +225,11 @@ export class ctTSOv2LaunchStepFunctionStateMachineConstruct extends Construct { }) ); + // Allow the statemachine to submit events to the event bus props.eventbus_obj.grantPutEventsTo(stateMachine.role); - // Set outputs this.cttso_v2_launch_state_machine_name = stateMachine.stateMachineName; this.cttso_v2_launch_state_machine_arn = stateMachine.stateMachineArn; - } - } diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack.ts b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack.ts new file mode 100644 index 000000000..e1c0d9ed0 --- /dev/null +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack.ts @@ -0,0 +1,125 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; +import * as events from 'aws-cdk-lib/aws-events'; +import { Cttsov2Icav2PipelineManagerConstruct } from './constructs/cttsov2-icav2-manager'; +import { PythonLambdaLayerConstruct } from '../../../../components/python-lambda-layer'; +import path from 'path'; + +export interface Cttsov2Icav2PipelineManagerConfig { + icav2_token_secret_id: string; // "/icav2/umccr-prod/service-production-jwt-token-secret-arn" + ssm_parameter_list: string[]; // List of parameters the workflow session state machine will need access to + icav2_copy_batch_utility_state_machine_name: string; + cttso_v2_launch_state_machine_arn_ssm_parameter_path: string; + cttso_v2_launch_state_machine_name_ssm_parameter_path: string; + dynamodb_table_name: string; + eventbus_name: string; +} + +export type cttsov2Icav2PipelineManagerStackProps = Cttsov2Icav2PipelineManagerConfig & + cdk.StackProps; + +export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack { + public readonly cttso_v2_launch_state_machine_arn: string; + public readonly cttso_v2_launch_state_machine_name: string; + public readonly cttso_v2_launch_state_machine_arn_ssm_parameter_path: string; + public readonly cttso_v2_launch_state_machine_name_ssm_parameter_path: string; + + constructor(scope: Construct, id: string, props: cttsov2Icav2PipelineManagerStackProps) { + super(scope, id, props); + + // Get the copy batch state machine name + const icav2_copy_batch_stack_state_machine_obj = sfn.StateMachine.fromStateMachineName( + this, + 'icav2_copy_batch_state_machine', + props.icav2_copy_batch_utility_state_machine_name + ); + + // Get dynamodb table for construct + const dynamodb_table_obj = dynamodb.TableV2.fromTableName( + this, + 'dynamodb_table', + props.dynamodb_table_name + ); + + // Get ICAv2 Access token secret object for construct + const icav2_access_token_secret_obj = secretsManager.Secret.fromSecretNameV2( + this, + 'Icav2SecretsObject', + props.icav2_token_secret_id + ); + + // Get lambda layer object + const lambda_layer_obj = new PythonLambdaLayerConstruct(this, 'lambda_layer', { + layer_name: 'cttso-v2-launch-state-machine-layer', + layer_description: 'CTTSO v2 Launch State Machine Lambda Layer', + layer_directory: path.join(__dirname, '../layers/'), + }); + + // Set ssm parameter object list + const ssm_parameter_obj_list = props.ssm_parameter_list.map((ssm_parameter_path: string) => + ssm.StringParameter.fromStringParameterName(this, ssm_parameter_path, ssm_parameter_path) + ); + + // Get event bus + const eventbus_obj = events.EventBus.fromEventBusName(this, 'eventbus', props.eventbus_name); + + const cttso_v2_launch_state_machine = new Cttsov2Icav2PipelineManagerConstruct(this, id, { + /* Stack Objects */ + dynamodb_table_obj: dynamodb_table_obj, + icav2_access_token_secret_obj: icav2_access_token_secret_obj, + lambda_layer_obj: lambda_layer_obj.lambda_layer_version_obj, + icav2_copy_batch_state_machine_obj: icav2_copy_batch_stack_state_machine_obj, + ssm_parameter_obj_list: ssm_parameter_obj_list, + eventbus_obj: eventbus_obj, + /* Lambdas paths */ + generate_db_uuid_lambda_path: path.join(__dirname, '../lambdas/generate_db_uuid'), // __dirname + '/../../../lambdas/generate_uuid' + generate_trimmed_samplesheet_lambda_path: path.join( + __dirname, + '../lambdas/generate_and_trim_cttso_samplesheet_dict' + ), // __dirname + '/../../../lambdas/generate_and_trim_cttso_samplesheet_dict' + upload_samplesheet_to_cache_dir_lambda_path: path.join( + __dirname, + '../lambdas/upload_samplesheet_to_cache_dir' + ), // __dirname + '/../../../lambdas/upload_samplesheet_to_cache_dir' + generate_copy_manifest_dict_lambda_path: path.join( + __dirname, + '../lambdas/generate_copy_manifest_dict' + ), // __dirname + '/../../../lambdas/generate_copy_manifest_dict' + launch_cttso_nextflow_pipeline_lambda_path: path.join( + __dirname, + '../lambdas/launch_cttso_nextflow_pipeline' + ), // __dirname + '../launch_cttso_nextflow_pipeline' + /* Step function templates */ + workflow_definition_body_path: path.join( + __dirname, + '../step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json' + ), // __dirname + '/../../../step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json' + }); + + // Set outputs + this.cttso_v2_launch_state_machine_arn = + cttso_v2_launch_state_machine.cttso_v2_launch_state_machine_arn; + this.cttso_v2_launch_state_machine_name = + cttso_v2_launch_state_machine.cttso_v2_launch_state_machine_name; + this.cttso_v2_launch_state_machine_arn_ssm_parameter_path = + props.cttso_v2_launch_state_machine_arn_ssm_parameter_path; + this.cttso_v2_launch_state_machine_name_ssm_parameter_path = + props.cttso_v2_launch_state_machine_name_ssm_parameter_path; + + // Set SSM Parameter for the state machine arn + new ssm.StringParameter(this, 'state_machine_arn_ssm_parameter', { + parameterName: this.cttso_v2_launch_state_machine_arn_ssm_parameter_path, + stringValue: this.cttso_v2_launch_state_machine_arn, + }); + + // Set SSM Parameter for the state machine name + new ssm.StringParameter(this, 'state_machine_name_ssm_parameter', { + parameterName: this.cttso_v2_launch_state_machine_name_ssm_parameter_path, + stringValue: this.cttso_v2_launch_state_machine_name, + }); + } +} diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/images/step_functions_image.png b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/images/step_functions_image.png similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/images/step_functions_image.png rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/images/step_functions_image.png diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py similarity index 99% rename from lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py index 618bb797b..e6f6f54be 100644 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py @@ -165,5 +165,5 @@ def handler(event, context): # ) # # # { -# # "samplesheet_b64gz": "H4sIAIf15mUC/41R0WrCMBT9FcnzhCa1CttTUSgDV4bWhzFGiPZqw5pUk1Qm4r/vJtVtZXsYoZCcc+49veeeSQWiBEPuB2eylTXwbWOUcPwIxspGI87uBsS0mmuhAJ+ksAc9zGG9yFMWU0rj4dMcYFksOB3nzZHFBAukts60CrTj7rQPdciJJRzIxfdDUxs8/Y1TvjltavAIoQklVwVnv3GpS/joF0TfOOvj3srZJomimltwTupd5ypKsXdgeOfu1dMiK6Z4irRIp3jw6tv2hOwfwjVU4iib1idKnJHKk0pqqVrF/VtB2XWrQe9chbI48RJh37mtGuP4LZyAN7gII0v4MdhqktMXOoofHmkUvhvQm7cUTqD69UysUHtcrCx98ZzFEY1HY3TBpfqfu9K3Nc3y1KO10HBdfojWU2mGo6cZDv6VeIik8CFgLGkW8AnvltEZrmbPURSxwCR/Mpe3yyfpAP0IhwIAAA==" +# # "samplesheet_b64gz": "H4sIAIf15mUC/41R0WrCMBT9FcnzhCa1CttTUSgDV4bWhzFGiPZqw5pUk1Qm4r/vJtVtZXsYoZCcc+49veeeSQWiBEPuB2eylTXwbWOUcPwIxspGI87uBsS0mmuhAJ+ksAc9zGG9yFMWU0rj4dMcYFksOB3nzZHFBAukts60CrTj7rQPdciJJRzIxfdDUxs8/Y1TvjltavAIoQklVwVnv3GpS/joF0TfOOvj3srZJomimltwTupd5ypKsXdgeOfu1dMiK6Z4irRIp3jw6tv2hOwfwjVU4iib1idKnJHKk0pqqVrF/VtB2XWrQe9chbI48RJh37mtGuP4LZyAN7gII0v4MdhqktMXOoofHmkUvhvQm7cUTqD69UysUHtcrCx98ZzFEY1HY3TBpfqfu9K3Nc3y1KO10HBdfojWU2mGo6cZDv6VeIik8CFgLGkW8AnvltEZrmbPURSxwCR/Mpe3yyfpAP0IhwIAAA==" // pragma: allowlist secret # # } diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_copy_manifest_dict/handler.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_copy_manifest_dict/handler.py similarity index 99% rename from lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_copy_manifest_dict/handler.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_copy_manifest_dict/handler.py index 8375fc31a..371782f4a 100644 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_copy_manifest_dict/handler.py +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_copy_manifest_dict/handler.py @@ -164,7 +164,7 @@ def handler(event, context): # json.dumps( # handler( # event={ -# "cache_path": "/ilmn_cttso_fastq_cache/20241231abcd1234/L12345678_run_cache", +# "cache_path": "/ilmn_cttso_fastq_cache/20241231abcd1234/L12345678_run_cache", # pragma: allowlist secret # "project_id": "7595e8f2-32d3-4c76-a324-c6a85dae87b5", # "sample_id": "L2301346_rerun", # "fastq_list_rows_b64gz": fastq_list_rows_b64gz diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_db_uuid/handler.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_db_uuid/handler.py similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_db_uuid/handler.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_db_uuid/handler.py diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/launch_cttso_nextflow_pipeline/handler.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/launch_cttso_nextflow_pipeline/handler.py similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/launch_cttso_nextflow_pipeline/handler.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/launch_cttso_nextflow_pipeline/handler.py diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/upload_samplesheet_to_cache_dir/handler.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/upload_samplesheet_to_cache_dir/handler.py similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/upload_samplesheet_to_cache_dir/handler.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/upload_samplesheet_to_cache_dir/handler.py diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/poetry.lock b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/poetry.lock similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/layers/poetry.lock rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/poetry.lock diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/pyproject.toml b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/pyproject.toml similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/layers/pyproject.toml rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/pyproject.toml diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/__init__.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/__init__.py similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/__init__.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/__init__.py diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/__init__.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/utils/__init__.py similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/__init__.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/utils/__init__.py diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/aws_ssm_helpers.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/utils/aws_ssm_helpers.py similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/aws_ssm_helpers.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/utils/aws_ssm_helpers.py diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/compression_helpers.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/utils/compression_helpers.py similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/compression_helpers.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/utils/compression_helpers.py diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/path_helpers.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/utils/path_helpers.py similarity index 100% rename from lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/path_helpers.py rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/src/cttso_v2_pipeline_manager_tools/utils/path_helpers.py diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json similarity index 99% rename from lib/workload/stateless/cttso_v2_pipeline_manager/step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json rename to lib/workload/stateless/stacks/cttso-v2-pipeline-manager/step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json index 9a04272b5..6357f9fd8 100644 --- a/lib/workload/stateless/cttso_v2_pipeline_manager/step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/step_functions_templates/cttso_v2_launch_workflow_state_machine.asl.json @@ -48,8 +48,7 @@ "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { "FunctionName": "${__generate_db_uuid__}", - "Payload": { - } + "Payload": {} }, "Retry": [ { @@ -418,4 +417,4 @@ "End": true } } -} \ No newline at end of file +} diff --git a/lib/workload/stateless/statelessStackCollectionClass.ts b/lib/workload/stateless/statelessStackCollectionClass.ts index e9b9574ef..5822bba89 100644 --- a/lib/workload/stateless/statelessStackCollectionClass.ts +++ b/lib/workload/stateless/statelessStackCollectionClass.ts @@ -26,6 +26,10 @@ import { BsshIcav2FastqCopyManagerStack, BsshIcav2FastqCopyManagerStackProps, } from './stacks/bssh-icav2-fastq-copy-manager/deploy/stack'; +import { + cttsov2Icav2PipelineManagerStackProps, + Cttsov2Icav2PipelineManagerStack, +} from './stacks/cttso-v2-pipeline-manager/deploy/stack'; export interface StatelessStackCollectionProps { postgresManagerStackProps: PostgresManagerStackProps; @@ -35,6 +39,7 @@ export interface StatelessStackCollectionProps { bsRunsUploadManagerStackProps: BsRunsUploadManagerStackProps; icav2CopyBatchUtilityStackProps: ICAv2CopyBatchUtilityStackProps; bsshIcav2FastqCopyManagerStackProps: BsshIcav2FastqCopyManagerStackProps; + cttsov2Icav2PipelineManagerStackProps: cttsov2Icav2PipelineManagerStackProps; } export class StatelessStackCollection { @@ -46,6 +51,7 @@ export class StatelessStackCollection { readonly bsRunsUploadManagerStack: Stack; readonly icav2CopyBatchUtilityStack: Stack; readonly bsshIcav2FastqCopyManagerStack: Stack; + readonly cttsov2Icav2PipelineManagerStack: Stack; constructor( scope: Construct, @@ -98,6 +104,15 @@ export class StatelessStackCollection { ...statelessConfiguration.bsshIcav2FastqCopyManagerStackProps, } ); + + this.cttsov2Icav2PipelineManagerStack = new Cttsov2Icav2PipelineManagerStack( + scope, + 'Cttsov2Icav2PipelineManagerStack', + { + ...this.createTemplateProps(env, 'Cttsov2Icav2PipelineManagerStack'), + ...statelessConfiguration.cttsov2Icav2PipelineManagerStackProps, + } + ); } /**