Merge branch 'main' of https://github.com/umccr/orcabus into objects_…

…apigw_api

.github/workflows/prbuild.yml

@@ -31,7 +31,7 @@ jobs:
       # TODO see whether we can leverage https://github.com/pre-commit/action
       - name: Install system-wide tools dependencies
         run: |
-          pip3 install pre-commit detect-secrets black ggshield awscli-local
+          pip3 install pre-commit detect-secrets black ggshield
 
       - name: Checkout code
         uses: actions/checkout@v3

config/constants.ts

@@ -6,11 +6,17 @@ import {
 } from '../lib/workload/orcabus-stateless-stack';
 import { Duration, aws_lambda, RemovalPolicy } from 'aws-cdk-lib';
 import { EventSourceProps } from '../lib/workload/stateful/event_source/component';
+import { DbAuthType } from '../lib/workload/stateless/postgres_manager/function/type';
 
 const regName = 'OrcaBusSchemaRegistry';
 const eventBusName = 'OrcaBusMain';
 const lambdaSecurityGroupName = 'OrcaBusLambdaSecurityGroup';
-const rdsMasterSecretName = 'orcabus/rds-master'; // pragma: allowlist secret
+const dbClusterIdentifier = 'orcabus-db';
+const dbClusterResourceIdParameterName = '/orcabus/db-cluster-resource-id';
+
+// Note, this should not end with a hyphen and 6 characters, otherwise secrets manager won't be
+// able to find the secret using a partial ARN.
+const rdsMasterSecretName = 'orcabus/master-rds'; // pragma: allowlist secret
 
 const orcaBusStatefulConfig = {
   schemaRegistryProps: {
@@ -24,7 +30,7 @@ const orcaBusStatefulConfig = {
     archiveRetention: 365,
   },
   databaseProps: {
-    clusterIdentifier: 'orcabus-db',
+    clusterIdentifier: dbClusterIdentifier,
     defaultDatabaseName: 'orcabus',
     version: AuroraPostgresEngineVersion.VER_15_4,
     parameterGroupName: 'default.aurora-postgresql15',
@@ -34,6 +40,7 @@ const orcaBusStatefulConfig = {
     monitoring: {
       cloudwatchLogsExports: ['orcabus-postgresql'],
     },
+    clusterResourceIdParameterName: dbClusterResourceIdParameterName,
   },
   securityGroupProps: {
     securityGroupName: lambdaSecurityGroupName,
@@ -64,6 +71,18 @@ const orcaBusStatelessConfig = {
   lambdaRuntimePythonVersion: aws_lambda.Runtime.PYTHON_3_10,
   bclConvertFunctionName: 'orcabus_bcl_convert',
   rdsMasterSecretName: rdsMasterSecretName,
+  postgresManagerConfig: {
+    masterSecretName: rdsMasterSecretName,
+    dbClusterIdentifier: dbClusterIdentifier,
+    clusterResourceIdParameterName: dbClusterResourceIdParameterName,
+    microserviceDbConfig: [
+      {
+        name: 'metadata_manager',
+        authType: DbAuthType.USERNAME_PASSWORD,
+      },
+      { name: 'filemanager', authType: DbAuthType.RDS_IAM },
+    ],
+  },
 };
 
 const eventSourceConfig: EventSourceProps = {
@@ -90,12 +109,25 @@ interface EnvironmentConfig {
     orcaBusStatelessConfig: OrcaBusStatelessConfig;
   };
 }
+
+/**
+ * Validate the secret name so that it doesn't end with 6 characters and a hyphen.
+ */
+export const validateSecretName = (secretName: string) => {
+  // If there are more config validation requirements like this it might be good to use
+  // a dedicated library like zod.
+  if (/-(.){6}$/.test(secretName)) {
+    throw new Error('the secret name should not end with a hyphen and 6 characters');
+  }
+};
+
 export const getEnvironmentConfig = (
   accountName: 'beta' | 'gamma' | 'prod'
 ): EnvironmentConfig | null => {
+  let config = null;
   switch (accountName) {
     case 'beta':
-      return {
+      config = {
         name: 'beta',
         accountId: '843407916570', // umccr_development
         stackProps: {
@@ -126,9 +158,10 @@ export const getEnvironmentConfig = (
           },
         },
       };
+      break;
 
     case 'gamma':
-      return {
+      config = {
         name: 'gamma',
         accountId: '455634345446', // umccr_staging
         stackProps: {
@@ -155,9 +188,10 @@ export const getEnvironmentConfig = (
           orcaBusStatelessConfig: orcaBusStatelessConfig,
         },
       };
+      break;
 
     case 'prod':
-      return {
+      config = {
         name: 'prod',
         accountId: '472057503814', // umccr_production
         stackProps: {
@@ -182,8 +216,10 @@ export const getEnvironmentConfig = (
           orcaBusStatelessConfig: orcaBusStatelessConfig,
         },
       };
-
-    default:
-      return null;
+      break;
   }
+
+  validateSecretName(config.stackProps.orcaBusStatefulConfig.databaseProps.masterSecretName);
+
+  return config;
 };

lib/pipeline/orcabus-stateful-pipeline-stack.ts

@@ -4,6 +4,9 @@ import * as ssm from 'aws-cdk-lib/aws-ssm';
 import * as pipelines from 'aws-cdk-lib/pipelines';
 import * as codebuild from 'aws-cdk-lib/aws-codebuild';
 import * as iam from 'aws-cdk-lib/aws-iam';
+import * as codepipeline from 'aws-cdk-lib/aws-codepipeline';
+import * as chatbot from 'aws-cdk-lib/aws-chatbot';
+import * as codestarnotifications from 'aws-cdk-lib/aws-codestarnotifications';
 import { OrcaBusStatefulConfig, OrcaBusStatefulStack } from '../workload/orcabus-stateful-stack';
 import { getEnvironmentConfig } from '../../config/constants';
 
@@ -18,7 +21,7 @@ export class StatefulPipelineStack extends cdk.Stack {
     });
 
     const unitTest = new pipelines.CodeBuildStep('UnitTest', {
-      commands: ['yarn install --frozen-lockfile', 'make test-stateful'],
+      commands: ['yarn install --immutable', 'make test-stateful'],
       input: sourceFile,
       primaryOutputDirectory: '.',
       buildEnvironment: {
@@ -43,7 +46,7 @@ export class StatefulPipelineStack extends cdk.Stack {
     });
 
     const synthAction = new pipelines.CodeBuildStep('Synth', {
-      commands: ['yarn install --frozen-lockfile', 'yarn run cdk-stateful-pipeline synth'],
+      commands: ['yarn install --immutable', 'yarn run cdk-stateful-pipeline synth'],
       input: unitTest,
       primaryOutputDirectory: 'cdk.out',
       rolePolicyStatements: [
@@ -108,6 +111,29 @@ export class StatefulPipelineStack extends cdk.Stack {
       }),
       { pre: [new pipelines.ManualApprovalStep('PromoteToProd')] }
     );
+
+    // need to build pipeline so we could add notification at the pipeline construct
+    pipeline.buildPipeline();
+
+    // notification for success/failure
+    const arteriaDevSlackConfigArn = ssm.StringParameter.valueForStringParameter(
+      this,
+      '/chatbot_arn/slack/arteria-dev'
+    );
+    const target = chatbot.SlackChannelConfiguration.fromSlackChannelConfigurationArn(
+      this,
+      'SlackChannelConfiguration',
+      arteriaDevSlackConfigArn
+    );
+
+    pipeline.pipeline.notifyOn('PipelineSlackNotification', target, {
+      events: [
+        codepipeline.PipelineNotificationEvents.PIPELINE_EXECUTION_FAILED,
+        codepipeline.PipelineNotificationEvents.PIPELINE_EXECUTION_SUCCEEDED,
+      ],
+      detailType: codestarnotifications.DetailType.BASIC,
+      notificationRuleName: 'orcabus_stateful_pipeline_notification',
+    });
   }
 }
 

lib/pipeline/orcabus-stateless-pipeline-stack.ts

@@ -4,6 +4,9 @@ import * as ssm from 'aws-cdk-lib/aws-ssm';
 import * as pipelines from 'aws-cdk-lib/pipelines';
 import * as codebuild from 'aws-cdk-lib/aws-codebuild';
 import * as iam from 'aws-cdk-lib/aws-iam';
+import * as chatbot from 'aws-cdk-lib/aws-chatbot';
+import * as codepipeline from 'aws-cdk-lib/aws-codepipeline';
+import * as codestarnotifications from 'aws-cdk-lib/aws-codestarnotifications';
 import { OrcaBusStatelessConfig, OrcaBusStatelessStack } from '../workload/orcabus-stateless-stack';
 import { getEnvironmentConfig } from '../../config/constants';
 
@@ -18,7 +21,13 @@ export class StatelessPipelineStack extends cdk.Stack {
     });
 
     const unitTest = new pipelines.CodeBuildStep('UnitTest', {
-      commands: ['yarn install --frozen-lockfile', 'make suite'],
+      installCommands: [
+        //  RUST installation
+        `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y`,
+        `source $HOME/.cargo/env`,
+        `pip3 install cargo-lambda`,
+      ],
+      commands: ['yarn install --immutable', 'make suite'],
       input: sourceFile,
       primaryOutputDirectory: '.',
       buildEnvironment: {
@@ -47,7 +56,13 @@ export class StatelessPipelineStack extends cdk.Stack {
     });
 
     const synthAction = new pipelines.CodeBuildStep('Synth', {
-      commands: ['yarn install --frozen-lockfile', 'yarn run cdk-stateless-pipeline synth'],
+      installCommands: [
+        //  RUST installation
+        `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y`,
+        `source $HOME/.cargo/env`,
+        `pip3 install cargo-lambda`,
+      ],
+      commands: ['yarn install --immutable', 'yarn run cdk-stateless-pipeline synth'],
       input: unitTest,
       primaryOutputDirectory: 'cdk.out',
       rolePolicyStatements: [
@@ -89,34 +104,61 @@ export class StatelessPipelineStack extends cdk.Stack {
       })
     );
 
-    /**
-     * Deployment to Gamma (Staging) account
-     */
-    const gammaConfig = getEnvironmentConfig('gamma');
-    if (!gammaConfig) throw new Error(`No 'Gamma' account configuration`);
-    pipeline.addStage(
-      new OrcaBusStatelessDeploymentStage(
-        this,
-        'GammaStatelessDeployment',
-        gammaConfig.stackProps,
-        {
-          account: gammaConfig.accountId,
-        }
-      ),
-      { pre: [new pipelines.ManualApprovalStep('PromoteToGamma')] }
-    );
+    // Since the stateless stack might need to reference the stateful resources (e.g. db, sg), we might comment this out
+    // to prevent cdk from looking up for non existence resource. Currently the stateful resource is only deployed in
+    // dev
 
-    /**
-     * Deployment to Prod account
-     */
-    const prodConfig = getEnvironmentConfig('prod');
-    if (!prodConfig) throw new Error(`No 'Prod' account configuration`);
-    pipeline.addStage(
-      new OrcaBusStatelessDeploymentStage(this, 'ProdStatelessDeployment', prodConfig.stackProps, {
-        account: gammaConfig?.accountId,
-      }),
-      { pre: [new pipelines.ManualApprovalStep('PromoteToProd')] }
+    // /**
+    //  * Deployment to Gamma (Staging) account
+    //  */
+    // const gammaConfig = getEnvironmentConfig('gamma');
+    // if (!gammaConfig) throw new Error(`No 'Gamma' account configuration`);
+    // pipeline.addStage(
+    //   new OrcaBusStatelessDeploymentStage(
+    //     this,
+    //     'GammaStatelessDeployment',
+    //     gammaConfig.stackProps,
+    //     {
+    //       account: gammaConfig.accountId,
+    //     }
+    //   ),
+    //   { pre: [new pipelines.ManualApprovalStep('PromoteToGamma')] }
+    // );
+
+    // /**
+    //  * Deployment to Prod account
+    //  */
+    // const prodConfig = getEnvironmentConfig('prod');
+    // if (!prodConfig) throw new Error(`No 'Prod' account configuration`);
+    // pipeline.addStage(
+    //   new OrcaBusStatelessDeploymentStage(this, 'ProdStatelessDeployment', prodConfig.stackProps, {
+    //     account: prodConfig?.accountId,
+    //   }),
+    //   { pre: [new pipelines.ManualApprovalStep('PromoteToProd')] }
+    // );
+
+    // need to build pipeline so we could add notification at the pipeline construct
+    pipeline.buildPipeline();
+
+    // notification for success/failure
+    const arteriaDevSlackConfigArn = ssm.StringParameter.valueForStringParameter(
+      this,
+      '/chatbot_arn/slack/arteria-dev'
     );
+    const target = chatbot.SlackChannelConfiguration.fromSlackChannelConfigurationArn(
+      this,
+      'SlackChannelConfiguration',
+      arteriaDevSlackConfigArn
+    );
+
+    pipeline.pipeline.notifyOn('PipelineSlackNotification', target, {
+      events: [
+        codepipeline.PipelineNotificationEvents.PIPELINE_EXECUTION_FAILED,
+        codepipeline.PipelineNotificationEvents.PIPELINE_EXECUTION_SUCCEEDED,
+      ],
+      detailType: codestarnotifications.DetailType.BASIC,
+      notificationRuleName: 'orcabus_stateless_pipeline_notification',
+    });
   }
 }