Merge pull request #79 from umccr/deploy/filemanager

filemanager: deploy changes and fixes

lib/workload/stateful/filemanager/Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "filemanager",
     "filemanager-http-lambda",
     "filemanager-ingest-lambda",
+    "filemanager-migrate-lambda"
 ]
 
 [workspace.package]

lib/workload/stateful/filemanager/README.md

@@ -28,10 +28,21 @@ this might interfere and complain about non-existing roles and users.
 
 ### Tooling prerequisites, testing and building the code
 
-For development of the rust workspace, install a build cache (sccache) and build manually:
+For development of the rust workspace, it's recommended to install a build cache (sccache) to improve compilation speeds:
 
 ```sh
 brew install sccache && export RUSTC_WRAPPER=`which sccache`
+```
+
+or 
+
+```sh
+cargo install sccache && export RUSTC_WRAPPER=`which sccache`
+```
+
+Then install build prerequisites to build:
+
+```sh
 cargo install cargo-watch sqlx-cli
 cargo build --all-targets --all-features
 ```

lib/workload/stateful/filemanager/build.rs

@@ -1,5 +1,5 @@
 // generated by `sqlx migrate build-script`
 fn main() {
     // trigger recompilation when a new migration is added
-    println!("cargo:rerun-if-changed=migrations");
-}
+    println!("cargo:rerun-if-changed=database/migrations");
+}

lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql

@@ -7,13 +7,13 @@ create table object (
     -- The name of the object.
     key varchar(1024) not null,
     -- The size of the object.
-    size int not null,
+    size int default null,
     -- A unique identifier for the object, if it is present.
     hash varchar(255) default null,
     -- When this object was created.
-    created_date timestamptz not null,
+    created_date timestamptz not null default now(),
     -- When this object was last modified.
-    last_modified_date timestamptz not null,
+    last_modified_date timestamptz not null default now(),
     -- When this object was deleted, a null value means that the object has not yet been deleted.
     deleted_date timestamptz default null,
     -- The date of the object and its id combined.

lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql

@@ -5,5 +5,5 @@ create table s3_object(
     -- The object id.
     object_id uuid references object (object_id) primary key,
     -- The S3 storage class of the object.
-    storage_class storage_class default null
+    storage_class storage_class not null
 );

lib/workload/stateful/filemanager/deploy/README.md

@@ -2,9 +2,20 @@
 
 This folder contains CDK deployment code for filemanager. The CDK code can be deployed using `cdk`:
 
-
 ```sh
 npm install
 cdk bootstrap
 cdk deploy
 ```
+
+By default, the stack does not perform database migration. To migrate the database, use the script inside `package.json`:
+
+```sh
+npm run migrate -- cdk deploy
+```
+
+or set `FILEMANAGER_DEPLOY_DATABASE_MIGRATION`:
+
+```sh
+export FILEMANAGER_DEPLOY_DATABASE_MIGRATION="true"
+```

lib/workload/stateful/filemanager/deploy/bin/filemanager.ts

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+
+import 'source-map-support/register';
+import * as cdk from 'aws-cdk-lib';
+import { FilemanagerStack } from '../lib/filemanager_stack';
+import { Tags } from 'aws-cdk-lib';
+
+export const STACK_NAME = 'FilemanagerStack';
+const STACK_DESCRIPTION = 'A stack deploying filemanager to dev.';
+
+const app = new cdk.App();
+new FilemanagerStack(
+  app,
+  STACK_NAME,
+  {
+    stackName: STACK_NAME,
+    description: STACK_DESCRIPTION,
+    tags: {
+      Stack: STACK_NAME,
+    },
+    env: {
+      region: 'ap-southeast-2',
+    },
+  },
+  {
+    destroyOnRemove: true,
+    enableMonitoring: {
+      enablePerformanceInsights: true,
+    },
+    public: [
+      // Put your IP here if you want the database to be reachable.
+    ],
+    migrateDatabase: process.env.FILEMANAGER_DEPLOY_MIGRATE_DATABASE == 'true',
+  }
+);
+
+Tags.of(app).add('Stack', STACK_NAME);

lib/workload/stateful/filemanager/deploy/cdk.json

@@ -1,5 +1,5 @@
 {
-  "app": "npx ts-node --prefer-ts-exts stack/stack.ts",
+  "app": "npx ts-node --prefer-ts-exts bin/filemanager.ts",
   "watch": {
     "include": ["**"],
     "exclude": [

lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts

@@ -0,0 +1,136 @@
+import { Construct, IDependable } from 'constructs';
+import {
+  AwsCustomResource,
+  AwsCustomResourcePolicy,
+  AwsSdkCall,
+  PhysicalResourceId,
+} from 'aws-cdk-lib/custom-resources';
+import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2';
+import * as fn from './functions/function';
+import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
+import { CfnOutput, Stack, Token } from 'aws-cdk-lib';
+
+/**
+ * Props for the resource invoke construct.
+ */
+export type CdkResourceInvokeProps = {
+  /**
+   * Vpc for the function.
+   */
+  vpc: IVpc;
+  /**
+   * The function to create. This will override the function name to ensure that it remains
+   * callable using the singleton function created by `AwsCustomResource`. See
+   * https://github.com/aws-samples/amazon-rds-init-cdk/blob/239626632f399ebe4928410a49d5ac5d009a6502/lib/resource-initializer.ts#L69-L71.
+   */
+  createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => fn.Function;
+  /**
+   * Function props when creating the Lambda function.
+   */
+  functionProps: fn.FunctionPropsNoPackage;
+  /**
+   * Name to use when creating the function.
+   */
+  id: string;
+  /**
+   * Dependencies for this resource.
+   */
+  dependencies?: IDependable[];
+};
+
+/**
+ * A construct for invoking a Lambda function for resource initialization.
+ */
+export class CdkResourceInvoke extends Construct {
+  private readonly _response: string;
+  private readonly _customResource: AwsCustomResource;
+  private readonly _function: fn.Function;
+
+  constructor(scope: Construct, id: string, props: CdkResourceInvokeProps) {
+    super(scope, id);
+
+    const stack = Stack.of(this);
+    this._function = props.createFunction(this, props.id, {
+      ...props.functionProps,
+      functionName: `${stack.stackName}-ResourceInvokeFunction-${props.id}`,
+    });
+
+    // Call another lambda function with no arguments.
+    const sdkCall: AwsSdkCall = {
+      service: 'Lambda',
+      action: 'invoke',
+      parameters: {
+        FunctionName: this.function.functionName(),
+      },
+      physicalResourceId: PhysicalResourceId.of(
+        `${id}-AwsSdkCall-${this.function.currentVersion()}`
+      ),
+    };
+
+    const role = new Role(this, 'AwsCustomResourceRole', {
+      assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
+    });
+    role.addToPolicy(
+      new PolicyStatement({
+        resources: [
+          // This needs to have permissions to run any `ResourceInvokeFunction` because it is deployed as a
+          // singleton Lambda function.
+          `arn:aws:lambda:${stack.region}:${stack.account}:function:${stack.stackName}-ResourceInvokeFunction-*`,
+        ],
+        actions: ['lambda:InvokeFunction'],
+      })
+    );
+    // Also require VPC access for a Lambda function within the VPC.
+    role.addManagedPolicy(
+      ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole')
+    );
+
+    this._customResource = new AwsCustomResource(this, 'AwsCustomResource', {
+      policy: AwsCustomResourcePolicy.fromSdkCalls({
+        resources: AwsCustomResourcePolicy.ANY_RESOURCE,
+      }),
+      onUpdate: sdkCall,
+      role: role,
+      vpc: props.vpc,
+      vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS },
+    });
+
+    this._response = this.customResource.getResponseField('Payload');
+
+    // Add any dependencies.
+    props.dependencies?.forEach((dependency) => this.addDependency(dependency));
+
+    // Output the result.
+    new CfnOutput(this, 'MigrateDatabaseResponse', {
+      value: Token.asString(this.response),
+    });
+  }
+
+  /**
+   * Add a dependency to this resource.
+   */
+  addDependency(dependency: IDependable) {
+    this.customResource.node.addDependency(dependency);
+  }
+
+  /**
+   * Get the function response.
+   */
+  get response(): string {
+    return this._response;
+  }
+
+  /**
+   * Get the custom resource.
+   */
+  get customResource(): AwsCustomResource {
+    return this._customResource;
+  }
+
+  /**
+   * Get the function.
+   */
+  get function(): fn.Function {
+    return this._function;
+  }
+}