Merge pull request #180 from umccr/refactor/filemanager-stack

refactor: filemanager stack
umccr · Mar 22, 2024 · 33abe19 · 33abe19
2 parents 06dbbee + 50e7d85
commit 33abe19
Show file tree

Hide file tree

Showing 16 changed files with 223 additions and 567 deletions.
diff --git a/config/constants.ts b/config/constants.ts
@@ -1,19 +1,22 @@
 import { OrcaBusStatefulConfig } from '../lib/workload/orcabus-stateful-stack';
 import { AuroraPostgresEngineVersion } from 'aws-cdk-lib/aws-rds';
-import {
-  FilemanagerDependencies,
-  OrcaBusStatelessConfig,
-} from '../lib/workload/orcabus-stateless-stack';
+import { OrcaBusStatelessConfig } from '../lib/workload/orcabus-stateless-stack';
 import { Duration, RemovalPolicy } from 'aws-cdk-lib';
 import { EventSourceProps } from '../lib/workload/stateful/event_source/component';
 import { DbAuthType } from '../lib/workload/stateless/postgres_manager/function/type';
+import { FilemanagerConfig } from '../lib/workload/stateless/filemanager/deploy/lib/filemanager';
 
 const regName = 'OrcaBusSchemaRegistry';
 const eventBusName = 'OrcaBusMain';
 const lambdaSecurityGroupName = 'OrcaBusLambdaSecurityGroup';
 const dbClusterIdentifier = 'orcabus-db';
 const dbClusterResourceIdParameterName = '/orcabus/db-cluster-resource-id';
 
+const eventSourceQueueName = 'orcabus-event-source-queue';
+const devBucket = 'umccr-temp-dev';
+const stgBucket = 'umccr-temp-stg';
+const prodBucket = 'org.umccr.data.oncoanalyser';
+
 // Note, this should not end with a hyphen and 6 characters, otherwise secrets manager won't be
 // able to find the secret using a partial ARN.
 const rdsMasterSecretName = 'orcabus/master-rds'; // pragma: allowlist secret
@@ -87,20 +90,24 @@ const orcaBusStatelessConfig = {
   },
 };
 
-const eventSourceConfig: EventSourceProps = {
-  queueName: 'orcabus-event-source-queue',
-  maxReceiveCount: 3,
-  rules: [
-    {
-      bucket: 'umccr-temp-dev',
-    },
-  ],
+const eventSourceConfig = (bucket: string): EventSourceProps => {
+  return {
+    queueName: eventSourceQueueName,
+    maxReceiveCount: 3,
+    rules: [
+      {
+        bucket,
+      },
+    ],
+  };
 };
 
-const filemanagerDependencies: FilemanagerDependencies = {
-  eventSourceBuckets: ['umccr-temp-dev'],
-  eventSourceQueueName: eventSourceConfig.queueName,
-  databaseSecretName: orcaBusStatefulConfig.databaseProps.masterSecretName,
+const filemanagerConfig = (bucket: string): FilemanagerConfig => {
+  return {
+    eventSourceQueueName: eventSourceQueueName,
+    databaseSecretName: orcaBusStatefulConfig.databaseProps.masterSecretName,
+    eventSourceBuckets: [bucket],
+  };
 };
 
 interface EnvironmentConfig {
@@ -152,11 +159,11 @@ export const getEnvironmentConfig = (
             securityGroupProps: {
               ...orcaBusStatefulConfig.securityGroupProps,
             },
-            eventSourceProps: eventSourceConfig,
+            eventSourceProps: eventSourceConfig(devBucket),
           },
           orcaBusStatelessConfig: {
             ...orcaBusStatelessConfig,
-            filemanagerDependencies: filemanagerDependencies,
+            filemanagerConfig: filemanagerConfig(devBucket),
           },
         },
       };
@@ -186,8 +193,12 @@ export const getEnvironmentConfig = (
             securityGroupProps: {
               ...orcaBusStatefulConfig.securityGroupProps,
             },
+            eventSourceProps: eventSourceConfig(stgBucket),
+          },
+          orcaBusStatelessConfig: {
+            ...orcaBusStatelessConfig,
+            filemanagerConfig: filemanagerConfig(stgBucket),
           },
-          orcaBusStatelessConfig: orcaBusStatelessConfig,
         },
       };
       break;
@@ -214,8 +225,12 @@ export const getEnvironmentConfig = (
             securityGroupProps: {
               ...orcaBusStatefulConfig.securityGroupProps,
             },
+            eventSourceProps: eventSourceConfig(prodBucket),
+          },
+          orcaBusStatelessConfig: {
+            ...orcaBusStatelessConfig,
+            filemanagerConfig: filemanagerConfig(prodBucket),
           },
-          orcaBusStatelessConfig: orcaBusStatelessConfig,
         },
       };
       break;

diff --git a/...tateless/functions/cdk_resource_invoke.ts → ...orkload/components/cdk_resource_invoke.ts b/...tateless/functions/cdk_resource_invoke.ts → ...orkload/components/cdk_resource_invoke.ts
@@ -36,7 +36,7 @@ export type FunctionName = {
    * Function name.
    */
   functionName: string;
-}
+};
 
 /**
  * Props for the resource invoke construct.
@@ -101,7 +101,7 @@ export class CdkResourceInvoke<P, F extends InvokeFunction> extends Construct {
       action: 'invoke',
       parameters: {
         FunctionName: this.function.functionName,
-        ...(props.payload && { Payload: props.payload })
+        ...(props.payload && { Payload: props.payload }),
       },
       physicalResourceId: PhysicalResourceId.of(
         `${id}-AwsSdkCall-${this.function.currentVersion + this.hashValue(props.payload)}`
@@ -111,13 +111,10 @@ export class CdkResourceInvoke<P, F extends InvokeFunction> extends Construct {
     const role = new Role(this, 'AwsCustomResourceRole', {
       assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
     });
+    const lambdaResource = `arn:aws:lambda:${stack.region}:${stack.account}:function:${stackHash}-ResourceInvokeFunction-${props.id}`;
     role.addToPolicy(
       new PolicyStatement({
-        resources: [
-          // This needs to have permissions to run any `ResourceInvokeFunction` because it is deployed as a
-          // singleton Lambda function.
-          `arn:aws:lambda:${stack.region}:${stack.account}:function:${stackHash}-ResourceInvokeFunction-*`,
-        ],
+        resources: [lambdaResource],
         actions: ['lambda:InvokeFunction'],
       })
     );
@@ -128,11 +125,12 @@ export class CdkResourceInvoke<P, F extends InvokeFunction> extends Construct {
 
     this._customResource = new AwsCustomResource(this, 'AwsCustomResource', {
       policy: AwsCustomResourcePolicy.fromSdkCalls({
-        resources: AwsCustomResourcePolicy.ANY_RESOURCE,
+        resources: [lambdaResource],
       }),
       onUpdate: sdkCall,
       role: role,
       vpc: props.vpc,
+      installLatestAwsSdk: true,
       vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS },
     });
 

diff --git a/lib/workload/orcabus-stateless-stack.ts b/lib/workload/orcabus-stateless-stack.ts
@@ -4,7 +4,7 @@ import { Construct } from 'constructs';
 import { getVpc } from './stateful/vpc/component';
 import { MultiSchemaConstructProps } from './stateless/schema/component';
 import { IVpc, ISecurityGroup, SecurityGroup } from 'aws-cdk-lib/aws-ec2';
-import { Filemanager } from './stateless/filemanager/deploy/lib/filemanager';
+import { Filemanager, FilemanagerConfig } from './stateless/filemanager/deploy/lib/filemanager';
 import { Queue } from 'aws-cdk-lib/aws-sqs';
 import { Secret } from 'aws-cdk-lib/aws-secretsmanager';
 import {
@@ -20,22 +20,7 @@ export interface OrcaBusStatelessConfig {
   lambdaSecurityGroupName: string;
   rdsMasterSecretName: string;
   postgresManagerConfig: PostgresManagerConfig;
-  filemanagerDependencies?: FilemanagerDependencies;
-}
-
-export interface FilemanagerDependencies {
-  /**
-   * Queue name used by the EventSource construct.
-   */
-  eventSourceQueueName: string;
-  /**
-   * Buckets defined by the EventSource construct.
-   */
-  eventSourceBuckets: string[];
-  /**
-   * Database secret name for the filemanager.
-   */
-  databaseSecretName: string;
+  filemanagerConfig: FilemanagerConfig;
 }
 
 export class OrcaBusStatelessStack extends cdk.Stack {
@@ -70,13 +55,7 @@ export class OrcaBusStatelessStack extends cdk.Stack {
 
     this.microserviceStackArray.push(this.createSequenceRunManager(props));
     this.microserviceStackArray.push(this.createPostgresManager(props.postgresManagerConfig));
-
-    if (props.filemanagerDependencies) {
-      this.createFilemanager({
-        ...props.filemanagerDependencies,
-        lambdaSecurityGroupName: props.lambdaSecurityGroupName,
-      });
-    }
+    this.microserviceStackArray.push(this.createFilemanager(props.filemanagerConfig));
   }
 
   private createSequenceRunManager(props: cdk.StackProps) {
@@ -96,38 +75,30 @@ export class OrcaBusStatelessStack extends cdk.Stack {
     });
   }
 
-  private createFilemanager(
-    dependencies: FilemanagerDependencies & { lambdaSecurityGroupName: string }
-  ) {
+  private createFilemanager(config: FilemanagerConfig) {
     // Opting to reconstruct the dependencies here, and pass them into the service as constructs.
     const queue = Queue.fromQueueArn(
       this,
       'FilemanagerQueue',
       Arn.format(
         {
-          resource: dependencies.eventSourceQueueName,
+          resource: config.eventSourceQueueName,
           service: 'sqs',
         },
         this
       )
     );
-    const databaseSecurityGroup = SecurityGroup.fromLookupByName(
-      this,
-      'FilemanagerDatabaseSecurityGroup',
-      dependencies.lambdaSecurityGroupName,
-      this.vpc
-    );
     const databaseSecret = Secret.fromSecretNameV2(
       this,
       'FilemanagerDatabaseSecret',
-      dependencies.databaseSecretName
+      config.databaseSecretName
     );
 
     return new Filemanager(this, 'Filemanager', {
-      buckets: dependencies.eventSourceBuckets,
+      buckets: config.eventSourceBuckets,
       buildEnvironment: {},
       databaseSecret,
-      databaseSecurityGroup,
+      databaseSecurityGroup: this.lambdaSecurityGroup,
       eventSources: [queue],
       migrateDatabase: true,
       vpc: this.vpc,

diff --git a/lib/workload/stateful/event_source/component.ts b/lib/workload/stateful/event_source/component.ts
@@ -55,9 +55,10 @@ export class EventSource extends Construct {
   constructor(scope: Construct, id: string, props: EventSourceProps) {
     super(scope, id);
 
-    this.deadLetterQueue = new Queue(this, 'DeadLetterQueue');
+    this.deadLetterQueue = new Queue(this, 'DeadLetterQueue', { enforceSSL: true });
     this.queue = new Queue(this, 'Queue', {
       queueName: props.queueName,
+      enforceSSL: true,
       deadLetterQueue: {
         maxReceiveCount: props.maxReceiveCount,
         queue: this.deadLetterQueue,