Merge pull request #92 from umccr/feature/base-cdk-codepipeline

Initiate CDK Pipeline

.gitignore

@@ -53,3 +53,6 @@ cdk.context.json
 data/
 
 Brewfile.lock.json
+*.xml
+
+target/

Makefile

@@ -23,11 +23,17 @@ baseline:
 test:
 	@yarn test
 
+# Test only section 
+test-stateful:
+	@yarn run test ./test/stateful
+test-stateless:
+	@yarn run test ./test/stateless
+
 # Run all test suites - i.e. cdk app unit tests + each microservice app test suites
 # Each app root should have Makefile `test` target; that run your app test pipeline including compose stack up/down
 # Note by running `make suite` target from repo root means your local dev env is okay with all app toolchains i.e.
 # 	Python (conda or venv), Rust and Cargo, TypeScript and Node environment, Docker and Container runtimes
-suite: test
+suite: test-stateless
 	@(cd lib/workload/stateless/sequence_run_manager && $(MAKE) test)
 	@(cd lib/workload/stateless/metadata_manager && $(MAKE) test)
 	@#(cd lib/workload/stateless/filemanager && $(MAKE) test)   # FIXME uncomment when ready @Marko

README.md

@@ -10,28 +10,27 @@ Please note; this is the _INVERSE_ of some typical standalone project setup such
 
 In this repo, we flip this view such that the Git repo root is the TypeScript CDK project; that wraps our applications into `./lib/` directory. You may [sparse checkout](https://git-scm.com/docs/git-sparse-checkout) or directly open subdirectory to set up the application project alone if you wish; e.g. `webstorm lib/workload/stateless/metadata_manager` or `code lib/workload/stateless/metadata_manager` or `pycharm lib/workload/stateless/sequence_run_manager` or `rustrover lib/workload/stateless/filemanager`. However, `code .` is a CDK TypeScript project.
 
-This root level CDK app contains 3 major stacks: `pipeline`, `stateful` and `stateless`. Pipeline stack is the CI/CD automation with CodePipeline setup. The `stateful` stack holds and manages some long-running AWS infrastructure resources. The `stateless` stack manages self-mutating CodePipeline reusable CDK Constructs for the [MicroService Applications](docs/developer/MICROSERVICE.md). In terms of CDK deployment point-of-view, the microservice application will be "stateless" application such that it will be changing/mutating over time; whereas "the data" its holds like PostgreSQL server infrastructure won't be changing that frequent. When updating "stateful" resources, there involves additional cares, steps and ops-procedures such as backing up database, downtime planning and so on; hence stateful. We use [configuration constants](./config) to decouple the reference between `stateful` and `stateless` AWS resources.
+This root level CDK app contains 4 major stacks: `stateful-pipeline`,`stateless-pipeline` , `stateful` and `stateless`. Pipeline stack is the CI/CD automation with CodePipeline setup. The `stateful` stack holds and manages some long-running AWS infrastructure resources. The `stateless` stack manages self-mutating CodePipeline reusable CDK Constructs for the [MicroService Applications](docs/developer/MICROSERVICE.md). In terms of CDK deployment point-of-view, the microservice application will be "stateless" application such that it will be changing/mutating over time; whereas "the data" its holds like PostgreSQL server infrastructure won't be changing that frequent. When updating "stateful" resources, there involves additional cares, steps and ops-procedures such as backing up database, downtime planning and so on; hence stateful. We use [configuration constants](./config) to decouple the reference between `stateful` and `stateless` AWS resources.
 
 In most cases, we deploy with automation across operational target environments or AWS accounts: `beta`, `gamma`, `prod`. For some particular purpose (such as onboarding procedure, isolated experimentation), we can spin up the whole infrastructure into some unique isolated AWS account. These key CDK entrypoints are documented in the following sections: Automation and Manual.
 
 ### Automation
 
 _CI/CD through CodePipeline automation from AWS toolchain account_
 
+There are 2 pipeline stacks in this project, one for the stateful and one for the stateless stack deployment. There is a
+script to access the `cdk` command for each pipeline:
+-`cdk-stateless-pipeline` - for stateless pipeline
+-`cdk-stateful-pipeline` - for stateful pipeline
+
 ```
 make install
 make check
 make test
 
-yarn cdk list
-
-yarn cdk synth <StackName>
-yarn cdk diff <StackName>
-yarn cdk deploy <StackName>
-
-yarn cdk synth
-yarn cdk diff
-yarn cdk deploy --all
+yarn cdk-stateless-pipeline synth
+yarn cdk-stateless-pipeline diff
+yarn cdk-stateless-pipeline deploy
 ```
 
 ### Manual
@@ -47,12 +46,12 @@ make test
 
 yarn orcabus --help
 
-yarn orcabus list
-yarn orcabus synth OrcaBusStatefulStack
-yarn orcabus diff OrcaBusStatefulStack
-yarn orcabus deploy OrcaBusStatefulStack
-yarn orcabus deploy --all
-yarn orcabus destroy --all
+yarn cdk-orcabus list
+yarn cdk-orcabus synth OrcaBusStatefulStack
+yarn cdk-orcabus diff OrcaBusStatefulStack
+yarn cdk-orcabus deploy OrcaBusStatefulStack
+yarn cdk-orcabus deploy --all
+yarn cdk-orcabus destroy --all
 ```
 
 ## Development

bin/orcabus.ts

@@ -15,8 +15,8 @@ import { getEnvironmentConfig } from '../config/constants';
 const app = new cdk.App();
 const props: cdk.StackProps = {
   env: {
-    account: process.env.CDK_DEFAULT_ACCOUNT,
-    region: process.env.CDK_DEFAULT_REGION,
+    account: process.env.CDK_DEPLOY_ACCOUNT || process.env.CDK_DEFAULT_ACCOUNT,
+    region: process.env.CDK_DEPLOY_REGION || process.env.CDK_DEFAULT_REGION,
   },
   tags: {
     'umccr-org:Stack': 'OrcaBusSandboxApp',

bin/pipeline.ts → bin/stateful-pipeline.ts

@@ -2,20 +2,20 @@
 import 'source-map-support/register';
 
 import * as cdk from 'aws-cdk-lib';
-import { PipelineStack } from '../lib/pipeline/orcabus-pipeline-stack';
+import { StatefulPipelineStack } from '../lib/pipeline/orcabus-stateful-pipeline-stack';
 
 const AWS_TOOLCHAIN_ACCOUNT = '383856791668'; // Bastion
 const AWS_TOOLCHAIN_REGION = 'ap-southeast-2';
 
 const app = new cdk.App();
 
-new PipelineStack(app, `OrcaBusPipeline`, {
+new StatefulPipelineStack(app, `OrcaBusStatefulPipeline`, {
   env: {
     account: AWS_TOOLCHAIN_ACCOUNT,
     region: AWS_TOOLCHAIN_REGION,
   },
   tags: {
-    'umccr-org:Stack': 'OrcaBusPipelineApp',
+    'umccr-org:Stack': 'OrcaBusStatefulPipelineApp',
     'umccr-org:Product': 'OrcaBus',
   },
 });

bin/stateless-pipeline.ts

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+
+import * as cdk from 'aws-cdk-lib';
+import { StatelessPipelineStack } from '../lib/pipeline/orcabus-stateless-pipeline-stack';
+
+const AWS_TOOLCHAIN_ACCOUNT = '383856791668'; // Bastion
+const AWS_TOOLCHAIN_REGION = 'ap-southeast-2';
+
+const app = new cdk.App();
+
+new StatelessPipelineStack(app, `OrcaBusStatelessPipeline`, {
+  env: {
+    account: AWS_TOOLCHAIN_ACCOUNT,
+    region: AWS_TOOLCHAIN_REGION,
+  },
+  tags: {
+    'umccr-org:Stack': 'OrcaBusStatelessPipeline',
+    'umccr-org:Product': 'OrcaBus',
+  },
+});

cdk.json

@@ -1,5 +1,5 @@
 {
-  "app": "npx ts-node --prefer-ts-exts bin/pipeline.ts",
+  "app": "yarn run -B ts-node --prefer-ts-exts bin/orcabus.ts",
   "watch": {
     "include": [
       "**"

config/constants.ts

@@ -1,7 +1,7 @@
 import { OrcaBusStatefulConfig } from '../lib/workload/orcabus-stateful-stack';
 import { AuroraPostgresEngineVersion } from 'aws-cdk-lib/aws-rds';
 import { OrcaBusStatelessConfig } from '../lib/workload/orcabus-stateless-stack';
-import { Duration, aws_lambda } from 'aws-cdk-lib';
+import { Duration, aws_lambda, RemovalPolicy } from 'aws-cdk-lib';
 
 const regName = 'OrcaBusSchemaRegistry';
 const eventBusName = 'OrcaBusMain';
@@ -24,7 +24,7 @@ const orcaBusStatefulConfig = {
     defaultDatabaseName: 'orcabus',
     version: AuroraPostgresEngineVersion.VER_15_4,
     parameterGroupName: 'default.aurora-postgresql15',
-    username: 'admin',
+    username: 'postgres',
     dbPort: 5432,
     masterSecretName: rdsMasterSecretName,
     monitoring: {
@@ -91,9 +91,10 @@ export const getEnvironmentConfig = (
               ...orcaBusStatefulConfig.databaseProps,
               numberOfInstance: 1,
               minACU: 0.5,
-              maxACU: 1,
+              maxACU: 16,
               enhancedMonitoringInterval: Duration.seconds(60),
               enablePerformanceInsights: true,
+              removalPolicy: RemovalPolicy.DESTROY,
             },
             securityGroupProps: {
               ...orcaBusStatefulConfig.securityGroupProps,
@@ -120,9 +121,10 @@ export const getEnvironmentConfig = (
               ...orcaBusStatefulConfig.databaseProps,
               numberOfInstance: 1,
               minACU: 0.5,
-              maxACU: 1,
+              maxACU: 16,
               enhancedMonitoringInterval: Duration.seconds(60),
               enablePerformanceInsights: true,
+              removalPolicy: RemovalPolicy.DESTROY,
             },
             securityGroupProps: {
               ...orcaBusStatefulConfig.securityGroupProps,
@@ -149,7 +151,8 @@ export const getEnvironmentConfig = (
               ...orcaBusStatefulConfig.databaseProps,
               numberOfInstance: 1,
               minACU: 0.5,
-              maxACU: 1,
+              maxACU: 16,
+              removalPolicy: RemovalPolicy.RETAIN,
             },
             securityGroupProps: {
               ...orcaBusStatefulConfig.securityGroupProps,

jest.config.js

@@ -5,4 +5,14 @@ module.exports = {
   transform: {
     '^.+\\.tsx?$': 'ts-jest',
   },
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        outputDirectory: 'target/report',
+        outputName: 'infrastructureTest.xml',
+      },
+    ],
+  ],
 };

lib/pipeline/orcabus-pipeline-stack.ts → lib/pipeline/orcabus-stateful-pipeline-stack.ts

@@ -3,12 +3,11 @@ import * as cdk from 'aws-cdk-lib';
 import * as ssm from 'aws-cdk-lib/aws-ssm';
 import * as pipelines from 'aws-cdk-lib/pipelines';
 import * as codebuild from 'aws-cdk-lib/aws-codebuild';
-import { OrcaBusStatelessConfig, OrcaBusStatelessStack } from '../workload/orcabus-stateless-stack';
+import * as iam from 'aws-cdk-lib/aws-iam';
 import { OrcaBusStatefulConfig, OrcaBusStatefulStack } from '../workload/orcabus-stateful-stack';
 import { getEnvironmentConfig } from '../../config/constants';
-import { PolicyStatement } from 'aws-cdk-lib/aws-iam';
 
-export class PipelineStack extends cdk.Stack {
+export class StatefulPipelineStack extends cdk.Stack {
   constructor(scope: Construct, id: string, props: cdk.StackProps) {
     super(scope, id, props);
 
@@ -18,34 +17,61 @@ export class PipelineStack extends cdk.Stack {
       connectionArn: codeStarArn,
     });
 
-    const synthAction = new pipelines.CodeBuildStep('Synth', {
+    const unitTest = new pipelines.CodeBuildStep('UnitTest', {
+      commands: ['yarn install --frozen-lockfile', 'make test-stateful'],
       input: sourceFile,
-      commands: ['yarn install --frozen-lockfile', 'yarn cdk synth -v'],
+      primaryOutputDirectory: '.',
+      buildEnvironment: {
+        privileged: true,
+        computeType: codebuild.ComputeType.LARGE,
+        buildImage: codebuild.LinuxArmBuildImage.AMAZON_LINUX_2_STANDARD_3_0,
+        environmentVariables: {
+          NODE_OPTIONS: {
+            value: '--max-old-space-size=8192',
+          },
+        },
+      },
+      partialBuildSpec: codebuild.BuildSpec.fromObject({
+        reports: {
+          'orcabus-infrastructureStatefulReports': {
+            files: ['target/report/*.xml'],
+            'file-format': 'JUNITXML',
+          },
+        },
+        version: '0.2',
+      }),
+    });
+
+    const synthAction = new pipelines.CodeBuildStep('Synth', {
+      commands: ['yarn install --frozen-lockfile', 'yarn run cdk-stateful-pipeline synth'],
+      input: unitTest,
       primaryOutputDirectory: 'cdk.out',
       rolePolicyStatements: [
-        new PolicyStatement({
+        new iam.PolicyStatement({
+          effect: iam.Effect.ALLOW,
           actions: ['sts:AssumeRole'],
           resources: ['*'],
-          conditions: {
-            StringEquals: {
-              'iam:ResourceTag/aws-cdk:bootstrap-role': 'lookup',
-            },
-          },
         }),
       ],
     });
-    synthAction.addStepDependency(new OrcaBusTestStep('OrcaBusUnitTest', { source: sourceFile }));
 
     const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {
       synth: synthAction,
       selfMutation: true,
       crossAccountKeys: true,
+      dockerEnabledForSynth: true,
+      dockerEnabledForSelfMutation: true,
       codeBuildDefaults: {
         buildEnvironment: {
-          buildImage: codebuild.LinuxBuildImage.STANDARD_6_0,
+          computeType: codebuild.ComputeType.LARGE,
+          buildImage: codebuild.LinuxArmBuildImage.AMAZON_LINUX_2_STANDARD_3_0,
+          environmentVariables: {
+            NODE_OPTIONS: {
+              value: '--max-old-space-size=8192',
+            },
+          },
         },
       },
-      dockerEnabledForSelfMutation: true,
     });
 
     /**
@@ -54,7 +80,7 @@ export class PipelineStack extends cdk.Stack {
     const betaConfig = getEnvironmentConfig('beta');
     if (!betaConfig) throw new Error(`No 'Beta' account configuration`);
     pipeline.addStage(
-      new OrcaBusDeploymentStage(this, 'BetaDeployment', betaConfig.stackProps, {
+      new OrcaBusStatefulDeploymentStage(this, 'BetaDeployment', betaConfig.stackProps, {
         account: betaConfig.accountId,
       })
     );
@@ -65,7 +91,7 @@ export class PipelineStack extends cdk.Stack {
     const gammaConfig = getEnvironmentConfig('gamma');
     if (!gammaConfig) throw new Error(`No 'Gamma' account configuration`);
     pipeline.addStage(
-      new OrcaBusDeploymentStage(this, 'GammaDeployment', gammaConfig.stackProps, {
+      new OrcaBusStatefulDeploymentStage(this, 'GammaDeployment', gammaConfig.stackProps, {
         account: gammaConfig.accountId,
       }),
       { pre: [new pipelines.ManualApprovalStep('PromoteToGamma')] }
@@ -74,65 +100,28 @@ export class PipelineStack extends cdk.Stack {
     /**
      * Deployment to Prod account (DISABLED)
      */
-    // const prodConfig = getEnvironmentConfig('prod');
-    // if (!prodConfig) throw new Error(`No 'Prod' account configuration`);
-    // pipeline.addStage(
-    //   new OrcaBusDeploymentStage(this, 'prodDeployment', prodConfig.stackProps, {
-    //     account: gammaConfig?.accountId,
-    //   }),
-    //   { pre: [new pipelines.ManualApprovalStep('PromoteToProd')] }
-    // );
+    const prodConfig = getEnvironmentConfig('prod');
+    if (!prodConfig) throw new Error(`No 'Prod' account configuration`);
+    pipeline.addStage(
+      new OrcaBusStatefulDeploymentStage(this, 'prodDeployment', prodConfig.stackProps, {
+        account: gammaConfig?.accountId,
+      }),
+      { pre: [new pipelines.ManualApprovalStep('PromoteToProd')] }
+    );
   }
 }
 
-class OrcaBusDeploymentStage extends cdk.Stage {
+class OrcaBusStatefulDeploymentStage extends cdk.Stage {
   constructor(
     scope: Construct,
     environmentName: string,
     stackProps: {
       orcaBusStatefulConfig: OrcaBusStatefulConfig;
-      orcaBusStatelessConfig: OrcaBusStatelessConfig;
     },
     env?: cdk.Environment
   ) {
     super(scope, environmentName, { env: { account: env?.account, region: 'ap-southeast-2' } });
 
     new OrcaBusStatefulStack(this, 'OrcaBusStatefulStack', stackProps.orcaBusStatefulConfig);
-    new OrcaBusStatelessStack(this, 'OrcaBusStatelessStack', stackProps.orcaBusStatelessConfig);
-  }
-}
-
-interface OrcaBusTestStepProps {
-  source: pipelines.CodePipelineSource;
-}
-class OrcaBusTestStep extends pipelines.CodeBuildStep {
-  constructor(id: string, props: OrcaBusTestStepProps) {
-    const stepProps: pipelines.CodeBuildStepProps = {
-      input: props.source,
-      commands: [],
-      partialBuildSpec: codebuild.BuildSpec.fromObject({
-        env: {
-          shell: 'bash',
-        },
-        phases: {
-          install: {
-            commands: [
-              'wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py310_23.3.1-0-Linux-x86_64.sh',
-              'bash Miniconda3-py310_23.3.1-0-Linux-x86_64.sh -b',
-              'export PATH=/root/miniconda3/bin:$PATH',
-              'conda init bash',
-              'source activate',
-              'conda create -q -n orcabus python=3.10',
-              'conda run -n orcabus yarn install',
-            ],
-          },
-          build: {
-            commands: ['conda run -n orcabus make test'],
-          },
-        },
-        version: '0.2',
-      }),
-    };
-    super(id, stepProps);
   }
 }