chore: remove parent (of µ-app) stack in cdk.Stage and Refactor

.prettierignore

@@ -33,5 +33,5 @@ openapi/
 venv/
 
 # TODO still early days let ignore prettier them (microservice apps) for now
-lib/workload/stateless/filemanager/
-lib/workload/stateless/sequence_run_manager/
+lib/workload/stateless/stacks/filemanager/
+lib/workload/stateless/stacks/sequence-run-manager/

Makefile

@@ -30,9 +30,9 @@ test-stateless:
 # Note by running `make suite` target from repo root means your local dev env is okay with all app toolchains i.e.
 # 	Python (conda or venv), Rust and Cargo, TypeScript and Node environment, Docker and Container runtimes
 test-suite:
-	@(cd lib/workload/stateless/sequence_run_manager && $(MAKE) test)
-	@(cd lib/workload/stateless/metadata_manager && $(MAKE) test)
-	@(cd lib/workload/stateless/filemanager && $(MAKE) test)
+	@(cd lib/workload/stateless/stacks/sequence-run-manager && $(MAKE) test)
+	@(cd lib/workload/stateless/stacks/metadata-manager && $(MAKE) test)
+	@(cd lib/workload/stateless/stacks/filemanager && $(MAKE) test)
 
 # The default outer `test` target only run the top level cdk application unit tests under `./test`
 test: test-stateless test-stateful test-suite

README.md

@@ -8,7 +8,7 @@ At the top level, the Git repository root is _the CDK TypeScript project_. It is
 
 Please note; this is the _INVERSE_ of some typical standalone project setup such that the repo root of the project is your app toolchain codebase and the deployment code are arranged under some arbitrary subdirectory like `./deploy/<cdk app root>`. We **do not** do this in this repo as we anticipate that we are going to deploy multiple of closely related micro applications.
 
-In this repo, we flip this view such that the Git repo root is the TypeScript CDK project; that wraps our applications into `./lib/` directory. You may [sparse checkout](https://git-scm.com/docs/git-sparse-checkout) or directly open subdirectory to set up the application project alone if you wish; e.g. `webstorm lib/workload/stateless/metadata_manager` or `code lib/workload/stateless/metadata_manager` or `pycharm lib/workload/stateless/sequence_run_manager` or `rustrover lib/workload/stateless/filemanager`. However, `code .` is a CDK TypeScript project.
+In this repo, we flip this view such that the Git repo root is the TypeScript CDK project; that wraps our applications into `./lib/` directory. You may [sparse checkout](https://git-scm.com/docs/git-sparse-checkout) or directly open subdirectory to set up the application project alone if you wish; e.g. `webstorm lib/workload/stateless/metadata-manager` or `code lib/workload/stateless/metadata-manager` or `pycharm lib/workload/stateless/sequence-run-manager` or `rustrover lib/workload/stateless/filemanager`. However, `code .` is a CDK TypeScript project.
 
 There are 2 CDK apps here:
 

bin/stateless-pipeline.ts → bin/statelessPipeline.ts

@@ -2,7 +2,7 @@
 import 'source-map-support/register';
 
 import * as cdk from 'aws-cdk-lib';
-import { StatelessPipelineStack } from '../lib/pipeline/orcabus-stateless-pipeline-stack';
+import { StatelessPipelineStack } from '../lib/pipeline/statelessPipelineStack';
 
 const AWS_TOOLCHAIN_ACCOUNT = '383856791668'; // Bastion
 const AWS_TOOLCHAIN_REGION = 'ap-southeast-2';

config/constants.ts

@@ -1,15 +1,18 @@
 import { AuroraPostgresEngineVersion } from 'aws-cdk-lib/aws-rds';
-import { OrcaBusStatelessConfig } from '../lib/workload/orcabus-stateless-stack';
+import { VpcLookupOptions } from 'aws-cdk-lib/aws-ec2';
 import { Duration, RemovalPolicy } from 'aws-cdk-lib';
 import { EventSourceProps } from '../lib/workload/stateful/stacks/shared/constructs/event-source';
-import { DbAuthType } from '../lib/workload/stateless/postgres_manager/function/type';
+import { DbAuthType } from '../lib/workload/stateless/stacks/postgres-manager/function/type';
 import {
   FILEMANAGER_SERVICE_NAME,
   FilemanagerConfig,
-} from '../lib/workload/stateless/filemanager/deploy/lib/filemanager';
+} from '../lib/workload/stateless/stacks/filemanager/deploy/lib/filemanager';
 import { IcaEventPipeStackProps } from '../lib/workload/stateful/stacks/ica-event-pipe/stack';
 import { StatefulStackCollectionProps } from '../lib/workload/stateful/statefulStackCollectionClass';
-import { VpcLookupOptions } from 'aws-cdk-lib/aws-ec2';
+import { StatelessStackCollectionProps } from '../lib/workload/stateless/statelessStackCollectionClass';
+import { SequenceRunManagerStackProps } from '../lib/workload/stateless/stacks/sequence-run-manager/deploy/component';
+import { MetadataManagerStackProps } from '../lib/workload/stateless/stacks/metadata-manager/deploy/stack';
+import { PostgresManagerStackProps } from '../lib/workload/stateless/stacks/postgres-manager/deploy/stack';
 
 const region = 'ap-southeast-2';
 
@@ -53,7 +56,7 @@ const icaEventPipeProps: IcaEventPipeStackProps = {
 const serviceUserSecretName = 'orcabus/token-service-user'; // pragma: allowlist secret
 const jwtSecretName = 'orcabus/token-service-jwt'; // pragma: allowlist secret
 
-const orcaBusStatefulConfig = {
+const statefulConfig = {
   schemaRegistryProps: {
     registryName: regName,
     description: 'Registry for OrcaBus Events',
@@ -92,48 +95,61 @@ const orcaBusStatefulConfig = {
   },
 };
 
-const orcaBusStatelessConfig = {
-  multiSchemaConstructProps: {
-    registryName: regName,
-    schemas: [
-      {
-        schemaName: 'BclConvertWorkflowRequest',
-        schemaDescription: 'Request event for BclConvertWorkflow',
-        schemaType: 'OpenApi3',
-        schemaLocation: __dirname + '/event_schemas/BclConvertWorkflowRequest.json',
-      },
-      {
-        schemaName: 'DragenWgsQcWorkflowRequest',
-        schemaDescription: 'Request event for DragenWgsQcWorkflowRequest',
-        schemaType: 'OpenApi3',
-        schemaLocation: __dirname + '/event_schemas/DragenWgsQcWorkflowRequest.json',
-      },
-    ],
-  },
-  eventBusName: eventBusName,
-  computeSecurityGroupName: computeSecurityGroupName,
-  rdsMasterSecretName: rdsMasterSecretName,
-  postgresManagerConfig: {
-    masterSecretName: rdsMasterSecretName,
-    dbClusterIdentifier: dbClusterIdentifier,
-    clusterResourceIdParameterName: dbClusterResourceIdParameterName,
-    dbPort: databasePort,
-    microserviceDbConfig: [
-      {
-        name: 'sequence_run_manager',
-        authType: DbAuthType.USERNAME_PASSWORD,
-      },
-      {
-        name: 'metadata_manager',
-        authType: DbAuthType.USERNAME_PASSWORD,
-      },
-      { name: FILEMANAGER_SERVICE_NAME, authType: DbAuthType.RDS_IAM },
-    ],
-    secretRotationSchedule: Duration.days(7),
-  },
-  metadataManagerConfig: {},
+const sequenceRunManagerStackProps: SequenceRunManagerStackProps = {
+  vpcProps,
+  lambdaSecurityGroupName: computeSecurityGroupName,
+  mainBusName: eventBusName,
+};
+
+const metadataManagerStackProps: MetadataManagerStackProps = {
+  vpcProps,
+  lambdaSecurityGroupName: computeSecurityGroupName,
 };
 
+const postgresManagerStackProps: PostgresManagerStackProps = {
+  vpcProps,
+  lambdaSecurityGroupName: computeSecurityGroupName,
+  masterSecretName: rdsMasterSecretName,
+  dbClusterIdentifier: dbClusterIdentifier,
+  clusterResourceIdParameterName: dbClusterResourceIdParameterName,
+  dbPort: databasePort,
+  microserviceDbConfig: [
+    {
+      name: 'sequence_run_manager',
+      authType: DbAuthType.USERNAME_PASSWORD,
+    },
+    {
+      name: 'metadata_manager',
+      authType: DbAuthType.USERNAME_PASSWORD,
+    },
+    { name: FILEMANAGER_SERVICE_NAME, authType: DbAuthType.RDS_IAM },
+  ],
+  secretRotationSchedule: Duration.days(7),
+};
+
+// const statelessConfig = {
+//   multiSchemaConstructProps: {
+//     registryName: regName,
+//     schemas: [
+//       {
+//         schemaName: 'BclConvertWorkflowRequest',
+//         schemaDescription: 'Request event for BclConvertWorkflow',
+//         schemaType: 'OpenApi3',
+//         schemaLocation: __dirname + '/event_schemas/BclConvertWorkflowRequest.json',
+//       },
+//       {
+//         schemaName: 'DragenWgsQcWorkflowRequest',
+//         schemaDescription: 'Request event for DragenWgsQcWorkflowRequest',
+//         schemaType: 'OpenApi3',
+//         schemaLocation: __dirname + '/event_schemas/DragenWgsQcWorkflowRequest.json',
+//       },
+//     ],
+//   },
+//   eventBusName: eventBusName,
+//   computeSecurityGroupName: computeSecurityGroupName,
+//   rdsMasterSecretName: rdsMasterSecretName,
+// };
+
 const eventSourceConfig = (bucket: string): EventSourceProps => {
   return {
     queueName: eventSourceQueueName,
@@ -148,9 +164,11 @@ const eventSourceConfig = (bucket: string): EventSourceProps => {
 
 const filemanagerConfig = (bucket: string): FilemanagerConfig => {
   return {
+    securityGroupName: computeSecurityGroupName,
+    vpcProps,
     eventSourceQueueName: eventSourceQueueName,
     databaseClusterEndpointHostParameter:
-      orcaBusStatefulConfig.databaseProps.clusterEndpointHostParameterName,
+      statefulConfig.databaseProps.clusterEndpointHostParameterName,
     port: databasePort,
     eventSourceBuckets: [bucket],
   };
@@ -162,7 +180,7 @@ interface EnvironmentConfig {
   accountId: string;
   stackProps: {
     statefulConfig: StatefulStackCollectionProps;
-    orcaBusStatelessConfig: OrcaBusStatelessConfig;
+    statelessConfig: StatelessStackCollectionProps;
   };
 }
 
@@ -191,26 +209,28 @@ export const getEnvironmentConfig = (
           statefulConfig: {
             sharedStackProps: {
               vpcProps,
-              schemaRegistryProps: orcaBusStatefulConfig.schemaRegistryProps,
-              eventBusProps: orcaBusStatefulConfig.eventBusProps,
+              schemaRegistryProps: statefulConfig.schemaRegistryProps,
+              eventBusProps: statefulConfig.eventBusProps,
               databaseProps: {
-                ...orcaBusStatefulConfig.databaseProps,
+                ...statefulConfig.databaseProps,
                 numberOfInstance: 1,
                 minACU: 0.5,
                 maxACU: 16,
                 enhancedMonitoringInterval: Duration.seconds(60),
                 enablePerformanceInsights: true,
                 removalPolicy: RemovalPolicy.DESTROY,
               },
-              computeProps: orcaBusStatefulConfig.computeProps,
+              computeProps: statefulConfig.computeProps,
               eventSourceProps: eventSourceConfig(devBucket),
             },
-            tokenServiceStackProps: orcaBusStatefulConfig.tokenServiceProps,
-            icaEventPipeStackProps: orcaBusStatefulConfig.icaEventPipeProps,
+            tokenServiceStackProps: statefulConfig.tokenServiceProps,
+            icaEventPipeStackProps: statefulConfig.icaEventPipeProps,
           },
-          orcaBusStatelessConfig: {
-            ...orcaBusStatelessConfig,
-            filemanagerConfig: filemanagerConfig(devBucket),
+          statelessConfig: {
+            postgresManagerStackProps: postgresManagerStackProps,
+            metadataManagerStackProps: metadataManagerStackProps,
+            sequenceRunManagerStackProps: sequenceRunManagerStackProps,
+            fileManagerStackProps: filemanagerConfig(devBucket),
           },
         },
       };
@@ -225,26 +245,28 @@ export const getEnvironmentConfig = (
           statefulConfig: {
             sharedStackProps: {
               vpcProps,
-              schemaRegistryProps: orcaBusStatefulConfig.schemaRegistryProps,
-              eventBusProps: orcaBusStatefulConfig.eventBusProps,
+              schemaRegistryProps: statefulConfig.schemaRegistryProps,
+              eventBusProps: statefulConfig.eventBusProps,
               databaseProps: {
-                ...orcaBusStatefulConfig.databaseProps,
+                ...statefulConfig.databaseProps,
                 numberOfInstance: 1,
                 minACU: 0.5,
                 maxACU: 16,
                 enhancedMonitoringInterval: Duration.seconds(60),
                 enablePerformanceInsights: true,
                 removalPolicy: RemovalPolicy.DESTROY,
               },
-              computeProps: orcaBusStatefulConfig.computeProps,
+              computeProps: statefulConfig.computeProps,
               eventSourceProps: eventSourceConfig(stgBucket),
             },
-            tokenServiceStackProps: orcaBusStatefulConfig.tokenServiceProps,
-            icaEventPipeStackProps: orcaBusStatefulConfig.icaEventPipeProps,
+            tokenServiceStackProps: statefulConfig.tokenServiceProps,
+            icaEventPipeStackProps: statefulConfig.icaEventPipeProps,
           },
-          orcaBusStatelessConfig: {
-            ...orcaBusStatelessConfig,
-            filemanagerConfig: filemanagerConfig(stgBucket),
+          statelessConfig: {
+            postgresManagerStackProps: postgresManagerStackProps,
+            metadataManagerStackProps: metadataManagerStackProps,
+            sequenceRunManagerStackProps: sequenceRunManagerStackProps,
+            fileManagerStackProps: filemanagerConfig(stgBucket),
           },
         },
       };
@@ -259,31 +281,33 @@ export const getEnvironmentConfig = (
           statefulConfig: {
             sharedStackProps: {
               vpcProps,
-              schemaRegistryProps: orcaBusStatefulConfig.schemaRegistryProps,
-              eventBusProps: orcaBusStatefulConfig.eventBusProps,
+              schemaRegistryProps: statefulConfig.schemaRegistryProps,
+              eventBusProps: statefulConfig.eventBusProps,
               databaseProps: {
-                ...orcaBusStatefulConfig.databaseProps,
+                ...statefulConfig.databaseProps,
                 numberOfInstance: 1,
                 minACU: 0.5,
                 maxACU: 16,
                 removalPolicy: RemovalPolicy.RETAIN,
               },
-              computeProps: orcaBusStatefulConfig.computeProps,
+              computeProps: statefulConfig.computeProps,
               eventSourceProps: eventSourceConfig(prodBucket),
             },
-            tokenServiceStackProps: orcaBusStatefulConfig.tokenServiceProps,
-            icaEventPipeStackProps: orcaBusStatefulConfig.icaEventPipeProps,
+            tokenServiceStackProps: statefulConfig.tokenServiceProps,
+            icaEventPipeStackProps: statefulConfig.icaEventPipeProps,
           },
-          orcaBusStatelessConfig: {
-            ...orcaBusStatelessConfig,
-            filemanagerConfig: filemanagerConfig(prodBucket),
+          statelessConfig: {
+            postgresManagerStackProps: postgresManagerStackProps,
+            metadataManagerStackProps: metadataManagerStackProps,
+            sequenceRunManagerStackProps: sequenceRunManagerStackProps,
+            fileManagerStackProps: filemanagerConfig(prodBucket),
           },
         },
       };
       break;
   }
 
-  // validateSecretName(config.stackProps.orcaBusStatefulConfig.databaseProps.masterSecretName);
+  // validateSecretName(config.stackProps.statefulConfig.databaseProps.masterSecretName);
 
   return config;
 };

lib/pipeline/orcabus-stateless-pipeline-stack.ts → lib/pipeline/statelessPipelineStack.ts

@@ -7,7 +7,12 @@ import * as iam from 'aws-cdk-lib/aws-iam';
 import * as chatbot from 'aws-cdk-lib/aws-chatbot';
 import * as codepipeline from 'aws-cdk-lib/aws-codepipeline';
 import * as codestarnotifications from 'aws-cdk-lib/aws-codestarnotifications';
-import { OrcaBusStatelessConfig, OrcaBusStatelessStack } from '../workload/orcabus-stateless-stack';
+
+import {
+  StatelessStackCollection,
+  StatelessStackCollectionProps,
+} from '../workload/stateless/statelessStackCollectionClass';
+
 import { getEnvironmentConfig } from '../../config/constants';
 
 export class StatelessPipelineStack extends cdk.Stack {
@@ -99,9 +104,15 @@ export class StatelessPipelineStack extends cdk.Stack {
     const betaConfig = getEnvironmentConfig('beta');
     if (!betaConfig) throw new Error(`No 'Beta' account configuration`);
     pipeline.addStage(
-      new OrcaBusStatelessDeploymentStage(this, 'BetaDeployment', betaConfig.stackProps, {
-        account: betaConfig.accountId,
-      })
+      new OrcaBusStatelessDeploymentStage(
+        this,
+        'BetaDeployment',
+        betaConfig.stackProps.statelessConfig,
+        {
+          account: betaConfig.accountId,
+          region: betaConfig.region,
+        }
+      )
     );
 
     // Since the stateless stack might need to reference the stateful resources (e.g. db, sg), we might comment this out
@@ -117,9 +128,10 @@ export class StatelessPipelineStack extends cdk.Stack {
     //   new OrcaBusStatelessDeploymentStage(
     //     this,
     //     'GammaDeployment',
-    //     gammaConfig.stackProps,
+    //     gammaConfig.stackProps.statelessConfig,
     //     {
     //       account: gammaConfig.accountId,
+    //       region: gammaConfig.region,
     //     }
     //   ),
     //   { pre: [new pipelines.ManualApprovalStep('PromoteToGamma')] }
@@ -131,9 +143,15 @@ export class StatelessPipelineStack extends cdk.Stack {
     // const prodConfig = getEnvironmentConfig('prod');
     // if (!prodConfig) throw new Error(`No 'Prod' account configuration`);
     // pipeline.addStage(
-    //   new OrcaBusStatelessDeploymentStage(this, 'ProdDeployment', prodConfig.stackProps, {
-    //     account: prodConfig?.accountId,
-    //   }),
+    //   new OrcaBusStatelessDeploymentStage(
+    //     this,
+    //     'ProdDeployment',
+    //     prodConfig.stackProps.statelessConfig,
+    //     {
+    //       account: prodConfig.accountId,
+    //       region: prodConfig.region,
+    //     }
+    //   ),
     //   { pre: [new pipelines.ManualApprovalStep('PromoteToProd')] }
     // );
 
@@ -166,13 +184,11 @@ class OrcaBusStatelessDeploymentStage extends cdk.Stage {
   constructor(
     scope: Construct,
     environmentName: string,
-    stackProps: {
-      orcaBusStatelessConfig: OrcaBusStatelessConfig;
-    },
-    env?: cdk.Environment
+    statelessStackCollectionProps: StatelessStackCollectionProps,
+    env: cdk.Environment
   ) {
-    super(scope, environmentName, { env: { account: env?.account, region: 'ap-southeast-2' } });
+    super(scope, environmentName, { env: env });
 
-    new OrcaBusStatelessStack(this, 'OrcaBusStatelessStack', stackProps.orcaBusStatelessConfig);
+    new StatelessStackCollection(this, env, statelessStackCollectionProps);
   }
 }