Merge pull request #98 from umccr/refactor/filemanager

refactor: filemanager deployment code and Makefile

.secrets.baseline

@@ -75,6 +75,10 @@
     {
       "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
     },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
     {
       "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
       "min_level": 2
@@ -109,10 +113,10 @@
     {
       "path": "detect_secrets.filters.regex.should_exclude_file",
       "pattern": [
-        "^(yarn.lock|.yarn/|.local/|openapi/)"
+        "^(yarn.lock|.yarn/|.local/|openapi/)|.sqlx/"
       ]
     }
   ],
   "results": {},
-  "generated_at": "2023-05-24T11:39:46Z"
+  "generated_at": "2024-02-18T03:27:13Z"
 }

Makefile

@@ -36,7 +36,7 @@ test-stateless:
 suite: test-stateless
 	@(cd lib/workload/stateless/sequence_run_manager && $(MAKE) test)
 	@(cd lib/workload/stateless/metadata_manager && $(MAKE) test)
-	@#(cd lib/workload/stateless/filemanager && $(MAKE) test)   # FIXME uncomment when ready @Marko
+	@(cd lib/workload/stateless/filemanager && $(MAKE) test)
 
 clean:
 	@yarn clean

config/constants.ts

@@ -1,6 +1,9 @@
 import { OrcaBusStatefulConfig } from '../lib/workload/orcabus-stateful-stack';
 import { AuroraPostgresEngineVersion } from 'aws-cdk-lib/aws-rds';
-import { OrcaBusStatelessConfig } from '../lib/workload/orcabus-stateless-stack';
+import {
+  FilemanagerDependencies,
+  OrcaBusStatelessConfig,
+} from '../lib/workload/orcabus-stateless-stack';
 import { Duration, aws_lambda, RemovalPolicy } from 'aws-cdk-lib';
 import { EventSourceProps } from '../lib/workload/stateful/event_source/component';
 
@@ -73,6 +76,12 @@ const eventSourceConfig: EventSourceProps = {
   ],
 };
 
+const filemanagerDependencies: FilemanagerDependencies = {
+  eventSourceBuckets: ['umccr-temp-dev'],
+  eventSourceQueueName: eventSourceConfig.queueName,
+  databaseSecretName: orcaBusStatefulConfig.databaseProps.masterSecretName,
+};
+
 interface EnvironmentConfig {
   name: string;
   accountId: string;
@@ -113,7 +122,7 @@ export const getEnvironmentConfig = (
           },
           orcaBusStatelessConfig: {
             ...orcaBusStatelessConfig,
-            eventSourceQueueName: eventSourceConfig.queueName,
+            filemanagerDependencies: filemanagerDependencies,
           },
         },
       };

lib/workload/orcabus-stateful-stack.ts

@@ -2,22 +2,22 @@ import * as cdk from 'aws-cdk-lib';
 import { Construct } from 'constructs';
 import { getVpc } from './stateful/vpc/component';
 import { EventBusConstruct, EventBusProps } from './stateful/eventbridge/component';
-import { DatabaseConstruct, DatabaseProps } from './stateful/database/component';
+import { Database, ConfigurableDatabaseProps } from './stateful/database/component';
 import { SecurityGroupConstruct, SecurityGroupProps } from './stateful/securitygroup/component';
 import { SchemaRegistryConstruct, SchemaRegistryProps } from './stateful/schemaregistry/component';
 import { EventSource, EventSourceProps } from './stateful/event_source/component';
 
 export interface OrcaBusStatefulConfig {
   schemaRegistryProps: SchemaRegistryProps;
   eventBusProps: EventBusProps;
-  databaseProps: DatabaseProps;
+  databaseProps: ConfigurableDatabaseProps;
   securityGroupProps: SecurityGroupProps;
   eventSourceProps?: EventSourceProps;
 }
 
 export class OrcaBusStatefulStack extends cdk.Stack {
   readonly eventBus: EventBusConstruct;
-  readonly database: DatabaseConstruct;
+  readonly database: Database;
   readonly securityGroup: SecurityGroupConstruct;
   readonly schemaRegistry: SchemaRegistryConstruct;
   readonly eventSource?: EventSource;
@@ -40,7 +40,9 @@ export class OrcaBusStatefulStack extends cdk.Stack {
       props.securityGroupProps
     );
 
-    this.database = new DatabaseConstruct(this, 'OrcaBusDatabaseConstruct', vpc, {
+    this.database = new Database(this, 'OrcaBusDatabaseConstruct', {
+      vpc,
+      allowedInboundSG: this.securityGroup.computeSecurityGroup,
       ...props.databaseProps,
     });
 

lib/workload/orcabus-stateless-stack.ts

@@ -1,9 +1,12 @@
 import * as cdk from 'aws-cdk-lib';
-import { aws_lambda } from 'aws-cdk-lib';
+import { Arn, aws_lambda } from 'aws-cdk-lib';
 import { Construct } from 'constructs';
 import { getVpc } from './stateful/vpc/component';
 import { MultiSchemaConstructProps } from './stateless/schema/component';
-import { IVpc } from 'aws-cdk-lib/aws-ec2';
+import { IVpc, SecurityGroup } from 'aws-cdk-lib/aws-ec2';
+import { Filemanager } from './stateless/filemanager/deploy/lib/filemanager';
+import { Queue } from 'aws-cdk-lib/aws-sqs';
+import { Secret } from 'aws-cdk-lib/aws-secretsmanager';
 
 export interface OrcaBusStatelessConfig {
   multiSchemaConstructProps: MultiSchemaConstructProps;
@@ -12,7 +15,22 @@ export interface OrcaBusStatelessConfig {
   lambdaRuntimePythonVersion: aws_lambda.Runtime;
   bclConvertFunctionName: string;
   rdsMasterSecretName: string;
-  eventSourceQueueName?: string;
+  filemanagerDependencies?: FilemanagerDependencies;
+}
+
+export interface FilemanagerDependencies {
+  /**
+   * Queue name used by the EventSource construct.
+   */
+  eventSourceQueueName: string;
+  /**
+   * Buckets defined by the EventSource construct.
+   */
+  eventSourceBuckets: string[];
+  /**
+   * Database secret name for the filemanager.
+   */
+  databaseSecretName: string;
 }
 
 export class OrcaBusStatelessStack extends cdk.Stack {
@@ -41,10 +59,55 @@ export class OrcaBusStatelessStack extends cdk.Stack {
 
     // hook microservice construct components here
     this.createSequenceRunManager();
+
+    if (props.filemanagerDependencies) {
+      this.createFilemanager({
+        ...props.filemanagerDependencies,
+        lambdaSecurityGroupName: props.lambdaSecurityGroupName,
+      });
+    }
   }
 
   private createSequenceRunManager() {
     // TODO new SequenceRunManagerConstruct() from lib/workload/stateless/sequence_run_manager/deploy/component.ts
     //   However, the implementation is still incomplete...
   }
+
+  private createFilemanager(
+    dependencies: FilemanagerDependencies & { lambdaSecurityGroupName: string }
+  ) {
+    // Opting to reconstruct the dependencies here, and pass them into the service as constructs.
+    const queue = Queue.fromQueueArn(
+      this,
+      'FilemanagerQueue',
+      Arn.format(
+        {
+          resource: dependencies.eventSourceQueueName,
+          service: 'Queue',
+        },
+        this
+      )
+    );
+    const databaseSecurityGroup = SecurityGroup.fromLookupByName(
+      this,
+      'FilemanagerDatabaseSecurityGroup',
+      dependencies.lambdaSecurityGroupName,
+      this.vpc
+    );
+    const databaseSecret = Secret.fromSecretNameV2(
+      this,
+      'FilemanagerDatabaseSecret',
+      dependencies.databaseSecretName
+    );
+
+    new Filemanager(this, 'Filemanager', {
+      buckets: dependencies.eventSourceBuckets,
+      buildEnvironment: {},
+      databaseSecret,
+      databaseSecurityGroup,
+      eventSources: [queue],
+      migrateDatabase: true,
+      vpc: this.vpc,
+    });
+  }
 }

lib/workload/stateful/database/component.ts

@@ -2,6 +2,8 @@ import { Construct } from 'constructs';
 import { RemovalPolicy, Duration } from 'aws-cdk-lib';
 import * as rds from 'aws-cdk-lib/aws-rds';
 import * as ec2 from 'aws-cdk-lib/aws-ec2';
+import { SecurityGroup } from 'aws-cdk-lib/aws-ec2';
+import { DatabaseCluster } from 'aws-cdk-lib/aws-rds';
 
 /**
  * Props for enabling enhanced monitoring.
@@ -16,59 +18,108 @@ type MonitoringProps = {
    */
   readonly enablePerformanceInsights?: boolean;
   /**
-   * performance insights retention period
+   * performance insights retention period.
    */
   readonly performanceInsightsRetention?: rds.PerformanceInsightRetention;
   /**
-   * Enable enhanced monitoring by specifying the interval
+   * Enable enhanced monitoring by specifying the interval.
    */
   readonly enhancedMonitoringInterval?: Duration;
 };
 
-export type DatabaseProps = MonitoringProps & {
+/**
+ * Database props without a VPC.
+ */
+export type ConfigurableDatabaseProps = MonitoringProps & {
+  /**
+   * The cluster identifier.
+   */
   clusterIdentifier: string;
+  /**
+   * The initial database name created.
+   */
   defaultDatabaseName: string;
+  /**
+   * Parameter group for the database.
+   */
   parameterGroupName: string;
+  /**
+   * Database username.
+   */
   username: string;
+  /**
+   * Database secret name.
+   */
   masterSecretName: string;
+  /**
+   * Database engine version.
+   */
   version: rds.AuroraPostgresEngineVersion;
+  /**
+   * Number of database instances.
+   */
   numberOfInstance: number;
+  /**
+   * Min ACU for serverless database.
+   */
   minACU: number;
+  /**
+   * Max ACU for serverless database.
+   */
   maxACU: number;
+  /**
+   * Port to run the database on.
+   */
   dbPort: number;
+  /**
+   * The database removal policy.
+   */
   removalPolicy: RemovalPolicy;
+};
+
+/**
+ * Database props with vpc and inbound security group.
+ */
+export type DatabaseProps = ConfigurableDatabaseProps & {
+  /**
+   * The database VPC.
+   */
+  vpc: ec2.IVpc;
+  /**
+   * Inbound security group for the database.
+   */
   allowedInboundSG?: ec2.SecurityGroup;
 };
 
-export class DatabaseConstruct extends Construct {
-  readonly dbSecurityGroup: ec2.SecurityGroup;
-  readonly dbCluster: rds.DatabaseCluster;
+export class Database extends Construct {
+  readonly securityGroup: SecurityGroup;
+  readonly cluster: DatabaseCluster;
 
-  constructor(scope: Construct, id: string, vpc: ec2.IVpc, props: DatabaseProps) {
+  constructor(scope: Construct, id: string, props: DatabaseProps) {
     super(scope, id);
 
     const dbSecret = new rds.DatabaseSecret(this, id + 'DbSecret', {
       username: props.username,
       secretName: props.masterSecretName,
     });
 
-    this.dbSecurityGroup = new ec2.SecurityGroup(this, 'DbSecurityGroup', {
-      vpc: vpc,
+    this.securityGroup = new ec2.SecurityGroup(this, 'DbSecurityGroup', {
+      vpc: props.vpc,
       allowAllOutbound: false,
       allowAllIpv6Outbound: false,
       description: 'security group for OrcaBus RDS',
     });
 
     // give compute sg to access the rds
     if (props.allowedInboundSG) {
-      this.dbSecurityGroup.addIngressRule(
+      this.securityGroup.addIngressRule(
         props.allowedInboundSG,
         ec2.Port.tcp(props.dbPort),
         'allow the OrcaBus compute sg to access db'
       );
     }
 
-    this.dbCluster = new rds.DatabaseCluster(this, id + 'Cluster', {
+    this.cluster = new rds.DatabaseCluster(this, id + 'Cluster', {
       engine: rds.DatabaseClusterEngine.auroraPostgres({ version: props.version }),
       clusterIdentifier: props.clusterIdentifier,
       credentials: rds.Credentials.fromSecret(dbSecret),
@@ -82,10 +133,10 @@ export class DatabaseConstruct extends Construct {
       storageEncrypted: true,
       iamAuthentication: true,
       removalPolicy: props.removalPolicy,
-      securityGroups: [this.dbSecurityGroup],
+      securityGroups: [this.securityGroup],
       serverlessV2MaxCapacity: props.maxACU,
       serverlessV2MinCapacity: props.minACU,
-      vpc: vpc,
+      vpc: props.vpc,
       vpcSubnets: {
         subnetType: ec2.SubnetType.PRIVATE_ISOLATED,
       },