chore: remove parent (of µ-app) stack in cdk.Stage and Refactor (#221)

umccr · Apr 17, 2024 · 4ea72cb · 4ea72cb
1 parent 5240ec3
commit 4ea72cb
Show file tree

Hide file tree

Showing 203 changed files with 405 additions and 404 deletions.
diff --git a/.prettierignore b/.prettierignore
@@ -33,5 +33,5 @@ openapi/
 venv/
 
 # TODO still early days let ignore prettier them (microservice apps) for now
-lib/workload/stateless/filemanager/
-lib/workload/stateless/sequence_run_manager/
+lib/workload/stateless/stacks/filemanager/
+lib/workload/stateless/stacks/sequence-run-manager/
diff --git a/Makefile b/Makefile
@@ -30,9 +30,9 @@ test-stateless:
 # Note by running `make suite` target from repo root means your local dev env is okay with all app toolchains i.e.
 # 	Python (conda or venv), Rust and Cargo, TypeScript and Node environment, Docker and Container runtimes
 test-suite:
-	@(cd lib/workload/stateless/sequence_run_manager && $(MAKE) test)
-	@(cd lib/workload/stateless/metadata_manager && $(MAKE) test)
-	@(cd lib/workload/stateless/filemanager && $(MAKE) test)
+	@(cd lib/workload/stateless/stacks/sequence-run-manager && $(MAKE) test)
+	@(cd lib/workload/stateless/stacks/metadata-manager && $(MAKE) test)
+	@(cd lib/workload/stateless/stacks/filemanager && $(MAKE) test)
 
 # The default outer `test` target only run the top level cdk application unit tests under `./test`
 test: test-stateless test-stateful test-suite

diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@ At the top level, the Git repository root is _the CDK TypeScript project_. It is
 
 Please note; this is the _INVERSE_ of some typical standalone project setup such that the repo root of the project is your app toolchain codebase and the deployment code are arranged under some arbitrary subdirectory like `./deploy/<cdk app root>`. We **do not** do this in this repo as we anticipate that we are going to deploy multiple of closely related micro applications.
 
-In this repo, we flip this view such that the Git repo root is the TypeScript CDK project; that wraps our applications into `./lib/` directory. You may [sparse checkout](https://git-scm.com/docs/git-sparse-checkout) or directly open subdirectory to set up the application project alone if you wish; e.g. `webstorm lib/workload/stateless/metadata_manager` or `code lib/workload/stateless/metadata_manager` or `pycharm lib/workload/stateless/sequence_run_manager` or `rustrover lib/workload/stateless/filemanager`. However, `code .` is a CDK TypeScript project.
+In this repo, we flip this view such that the Git repo root is the TypeScript CDK project; that wraps our applications into `./lib/` directory. You may [sparse checkout](https://git-scm.com/docs/git-sparse-checkout) or directly open subdirectory to set up the application project alone if you wish; e.g. `webstorm lib/workload/stateless/stacks/metadata-manager` or `code lib/workload/stateless/stacks/metadata-manager` or `pycharm lib/workload/stateless/stacks/sequence-run-manager` or `rustrover lib/workload/stateless/stacks/filemanager`. However, `code .` is a CDK TypeScript project.
 
 There are 2 CDK apps here:
 
@@ -85,7 +85,7 @@ You could list the CDK stacks with the `cdk ls` command to look at the stackId g
 yarn cdk-stateless ls
 
 OrcaBusStatelessPipeline
-OrcaBusStatelessPipeline/BetaDeployment/MetadataManager
+OrcaBusStatelessPipeline/BetaDeployment/MetadataManagerStack
 ...
 ```
 

diff --git a/bin/stateless-pipeline.ts → bin/statelessPipeline.ts b/bin/stateless-pipeline.ts → bin/statelessPipeline.ts
@@ -2,7 +2,7 @@
 import 'source-map-support/register';
 
 import * as cdk from 'aws-cdk-lib';
-import { StatelessPipelineStack } from '../lib/pipeline/orcabus-stateless-pipeline-stack';
+import { StatelessPipelineStack } from '../lib/pipeline/statelessPipelineStack';
 
 const AWS_TOOLCHAIN_ACCOUNT = '383856791668'; // Bastion
 const AWS_TOOLCHAIN_REGION = 'ap-southeast-2';

diff --git a/config/constants.ts b/config/constants.ts
@@ -1,15 +1,18 @@
 import { AuroraPostgresEngineVersion } from 'aws-cdk-lib/aws-rds';
-import { OrcaBusStatelessConfig } from '../lib/workload/orcabus-stateless-stack';
+import { VpcLookupOptions } from 'aws-cdk-lib/aws-ec2';
 import { Duration, RemovalPolicy } from 'aws-cdk-lib';
 import { EventSourceProps } from '../lib/workload/stateful/stacks/shared/constructs/event-source';
-import { DbAuthType } from '../lib/workload/stateless/postgres_manager/function/type';
+import { DbAuthType } from '../lib/workload/stateless/stacks/postgres-manager/function/type';
 import {
   FILEMANAGER_SERVICE_NAME,
   FilemanagerConfig,
-} from '../lib/workload/stateless/filemanager/deploy/lib/filemanager';
+} from '../lib/workload/stateless/stacks/filemanager/deploy/lib/filemanager';
 import { IcaEventPipeStackProps } from '../lib/workload/stateful/stacks/ica-event-pipe/stack';
 import { StatefulStackCollectionProps } from '../lib/workload/stateful/statefulStackCollectionClass';
-import { VpcLookupOptions } from 'aws-cdk-lib/aws-ec2';
+import { StatelessStackCollectionProps } from '../lib/workload/stateless/statelessStackCollectionClass';
+import { SequenceRunManagerStackProps } from '../lib/workload/stateless/stacks/sequence-run-manager/deploy/component';
+import { MetadataManagerStackProps } from '../lib/workload/stateless/stacks/metadata-manager/deploy/stack';
+import { PostgresManagerStackProps } from '../lib/workload/stateless/stacks/postgres-manager/deploy/stack';
 
 const region = 'ap-southeast-2';
 
@@ -53,7 +56,7 @@ const icaEventPipeProps: IcaEventPipeStackProps = {
 const serviceUserSecretName = 'orcabus/token-service-user'; // pragma: allowlist secret
 const jwtSecretName = 'orcabus/token-service-jwt'; // pragma: allowlist secret
 
-const orcaBusStatefulConfig = {
+const statefulConfig = {
   schemaRegistryProps: {
     registryName: regName,
     description: 'Registry for OrcaBus Events',
@@ -92,48 +95,61 @@ const orcaBusStatefulConfig = {
   },
 };
 
-const orcaBusStatelessConfig = {
-  multiSchemaConstructProps: {
-    registryName: regName,
-    schemas: [
-      {
-        schemaName: 'BclConvertWorkflowRequest',
-        schemaDescription: 'Request event for BclConvertWorkflow',
-        schemaType: 'OpenApi3',
-        schemaLocation: __dirname + '/event_schemas/BclConvertWorkflowRequest.json',
-      },
-      {
-        schemaName: 'DragenWgsQcWorkflowRequest',
-        schemaDescription: 'Request event for DragenWgsQcWorkflowRequest',
-        schemaType: 'OpenApi3',
-        schemaLocation: __dirname + '/event_schemas/DragenWgsQcWorkflowRequest.json',
-      },
-    ],
-  },
-  eventBusName: eventBusName,
-  computeSecurityGroupName: computeSecurityGroupName,
-  rdsMasterSecretName: rdsMasterSecretName,
-  postgresManagerConfig: {
-    masterSecretName: rdsMasterSecretName,
-    dbClusterIdentifier: dbClusterIdentifier,
-    clusterResourceIdParameterName: dbClusterResourceIdParameterName,
-    dbPort: databasePort,
-    microserviceDbConfig: [
-      {
-        name: 'sequence_run_manager',
-        authType: DbAuthType.USERNAME_PASSWORD,
-      },
-      {
-        name: 'metadata_manager',
-        authType: DbAuthType.USERNAME_PASSWORD,
-      },
-      { name: FILEMANAGER_SERVICE_NAME, authType: DbAuthType.RDS_IAM },
-    ],
-    secretRotationSchedule: Duration.days(7),
-  },
-  metadataManagerConfig: {},
+const sequenceRunManagerStackProps: SequenceRunManagerStackProps = {
+  vpcProps,
+  lambdaSecurityGroupName: computeSecurityGroupName,
+  mainBusName: eventBusName,
+};
+
+const metadataManagerStackProps: MetadataManagerStackProps = {
+  vpcProps,
+  lambdaSecurityGroupName: computeSecurityGroupName,
 };
 
+const postgresManagerStackProps: PostgresManagerStackProps = {
+  vpcProps,
+  lambdaSecurityGroupName: computeSecurityGroupName,
+  masterSecretName: rdsMasterSecretName,
+  dbClusterIdentifier: dbClusterIdentifier,
+  clusterResourceIdParameterName: dbClusterResourceIdParameterName,
+  dbPort: databasePort,
+  microserviceDbConfig: [
+    {
+      name: 'sequence_run_manager',
+      authType: DbAuthType.USERNAME_PASSWORD,
+    },
+    {
+      name: 'metadata_manager',
+      authType: DbAuthType.USERNAME_PASSWORD,
+    },
+    { name: FILEMANAGER_SERVICE_NAME, authType: DbAuthType.RDS_IAM },
+  ],
+  secretRotationSchedule: Duration.days(7),
+};
+
+// const statelessConfig = {
+//   multiSchemaConstructProps: {
+//     registryName: regName,
+//     schemas: [
+//       {
+//         schemaName: 'BclConvertWorkflowRequest',
+//         schemaDescription: 'Request event for BclConvertWorkflow',
+//         schemaType: 'OpenApi3',
+//         schemaLocation: __dirname + '/event_schemas/BclConvertWorkflowRequest.json',
+//       },
+//       {
+//         schemaName: 'DragenWgsQcWorkflowRequest',
+//         schemaDescription: 'Request event for DragenWgsQcWorkflowRequest',
+//         schemaType: 'OpenApi3',
+//         schemaLocation: __dirname + '/event_schemas/DragenWgsQcWorkflowRequest.json',
+//       },
+//     ],
+//   },
+//   eventBusName: eventBusName,
+//   computeSecurityGroupName: computeSecurityGroupName,
+//   rdsMasterSecretName: rdsMasterSecretName,
+// };
+
 const eventSourceConfig = (bucket: string): EventSourceProps => {
   return {
     queueName: eventSourceQueueName,
@@ -148,11 +164,14 @@ const eventSourceConfig = (bucket: string): EventSourceProps => {
 
 const filemanagerConfig = (bucket: string): FilemanagerConfig => {
   return {
+    securityGroupName: computeSecurityGroupName,
+    vpcProps,
     eventSourceQueueName: eventSourceQueueName,
     databaseClusterEndpointHostParameter:
-      orcaBusStatefulConfig.databaseProps.clusterEndpointHostParameterName,
+      statefulConfig.databaseProps.clusterEndpointHostParameterName,
     port: databasePort,
     eventSourceBuckets: [bucket],
+    migrateDatabase: true,
   };
 };
 
@@ -162,7 +181,7 @@ interface EnvironmentConfig {
   accountId: string;
   stackProps: {
     statefulConfig: StatefulStackCollectionProps;
-    orcaBusStatelessConfig: OrcaBusStatelessConfig;
+    statelessConfig: StatelessStackCollectionProps;
   };
 }
 
@@ -191,26 +210,28 @@ export const getEnvironmentConfig = (
           statefulConfig: {
             sharedStackProps: {
               vpcProps,
-              schemaRegistryProps: orcaBusStatefulConfig.schemaRegistryProps,
-              eventBusProps: orcaBusStatefulConfig.eventBusProps,
+              schemaRegistryProps: statefulConfig.schemaRegistryProps,
+              eventBusProps: statefulConfig.eventBusProps,
               databaseProps: {
-                ...orcaBusStatefulConfig.databaseProps,
+                ...statefulConfig.databaseProps,
                 numberOfInstance: 1,
                 minACU: 0.5,
                 maxACU: 16,
                 enhancedMonitoringInterval: Duration.seconds(60),
                 enablePerformanceInsights: true,
                 removalPolicy: RemovalPolicy.DESTROY,
               },
-              computeProps: orcaBusStatefulConfig.computeProps,
+              computeProps: statefulConfig.computeProps,
               eventSourceProps: eventSourceConfig(devBucket),
             },
-            tokenServiceStackProps: orcaBusStatefulConfig.tokenServiceProps,
-            icaEventPipeStackProps: orcaBusStatefulConfig.icaEventPipeProps,
+            tokenServiceStackProps: statefulConfig.tokenServiceProps,
+            icaEventPipeStackProps: statefulConfig.icaEventPipeProps,
           },
-          orcaBusStatelessConfig: {
-            ...orcaBusStatelessConfig,
-            filemanagerConfig: filemanagerConfig(devBucket),
+          statelessConfig: {
+            postgresManagerStackProps: postgresManagerStackProps,
+            metadataManagerStackProps: metadataManagerStackProps,
+            sequenceRunManagerStackProps: sequenceRunManagerStackProps,
+            fileManagerStackProps: filemanagerConfig(devBucket),
           },
         },
       };
@@ -225,26 +246,28 @@ export const getEnvironmentConfig = (
           statefulConfig: {
             sharedStackProps: {
               vpcProps,
-              schemaRegistryProps: orcaBusStatefulConfig.schemaRegistryProps,
-              eventBusProps: orcaBusStatefulConfig.eventBusProps,
+              schemaRegistryProps: statefulConfig.schemaRegistryProps,
+              eventBusProps: statefulConfig.eventBusProps,
               databaseProps: {
-                ...orcaBusStatefulConfig.databaseProps,
+                ...statefulConfig.databaseProps,
                 numberOfInstance: 1,
                 minACU: 0.5,
                 maxACU: 16,
                 enhancedMonitoringInterval: Duration.seconds(60),
                 enablePerformanceInsights: true,
                 removalPolicy: RemovalPolicy.DESTROY,
               },
-              computeProps: orcaBusStatefulConfig.computeProps,
+              computeProps: statefulConfig.computeProps,
               eventSourceProps: eventSourceConfig(stgBucket),
             },
-            tokenServiceStackProps: orcaBusStatefulConfig.tokenServiceProps,
-            icaEventPipeStackProps: orcaBusStatefulConfig.icaEventPipeProps,
+            tokenServiceStackProps: statefulConfig.tokenServiceProps,
+            icaEventPipeStackProps: statefulConfig.icaEventPipeProps,
           },
-          orcaBusStatelessConfig: {
-            ...orcaBusStatelessConfig,
-            filemanagerConfig: filemanagerConfig(stgBucket),
+          statelessConfig: {
+            postgresManagerStackProps: postgresManagerStackProps,
+            metadataManagerStackProps: metadataManagerStackProps,
+            sequenceRunManagerStackProps: sequenceRunManagerStackProps,
+            fileManagerStackProps: filemanagerConfig(stgBucket),
           },
         },
       };
@@ -259,31 +282,33 @@ export const getEnvironmentConfig = (
           statefulConfig: {
             sharedStackProps: {
               vpcProps,
-              schemaRegistryProps: orcaBusStatefulConfig.schemaRegistryProps,
-              eventBusProps: orcaBusStatefulConfig.eventBusProps,
+              schemaRegistryProps: statefulConfig.schemaRegistryProps,
+              eventBusProps: statefulConfig.eventBusProps,
               databaseProps: {
-                ...orcaBusStatefulConfig.databaseProps,
+                ...statefulConfig.databaseProps,
                 numberOfInstance: 1,
                 minACU: 0.5,
                 maxACU: 16,
                 removalPolicy: RemovalPolicy.RETAIN,
               },
-              computeProps: orcaBusStatefulConfig.computeProps,
+              computeProps: statefulConfig.computeProps,
               eventSourceProps: eventSourceConfig(prodBucket),
             },
-            tokenServiceStackProps: orcaBusStatefulConfig.tokenServiceProps,
-            icaEventPipeStackProps: orcaBusStatefulConfig.icaEventPipeProps,
+            tokenServiceStackProps: statefulConfig.tokenServiceProps,
+            icaEventPipeStackProps: statefulConfig.icaEventPipeProps,
           },
-          orcaBusStatelessConfig: {
-            ...orcaBusStatelessConfig,
-            filemanagerConfig: filemanagerConfig(prodBucket),
+          statelessConfig: {
+            postgresManagerStackProps: postgresManagerStackProps,
+            metadataManagerStackProps: metadataManagerStackProps,
+            sequenceRunManagerStackProps: sequenceRunManagerStackProps,
+            fileManagerStackProps: filemanagerConfig(prodBucket),
           },
         },
       };
       break;
   }
 
-  // validateSecretName(config.stackProps.orcaBusStatefulConfig.databaseProps.masterSecretName);
+  // validateSecretName(config.stackProps.statefulConfig.databaseProps.masterSecretName);
 
   return config;
 };