From 491960a65d20cd89d0d2e7179e45cc0c6f55c24f Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 15 Dec 2023 07:23:18 +1100 Subject: [PATCH 01/24] feat(filemanager): cross platform scripts --- lib/workload/stateful/filemanager/deploy/stack/stack.ts | 3 +-- lib/workload/stateful/filemanager/scripts/deploy.sh | 5 +++-- lib/workload/stateful/filemanager/scripts/logs.sh | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/workload/stateful/filemanager/deploy/stack/stack.ts b/lib/workload/stateful/filemanager/deploy/stack/stack.ts index 3cc4f175e..2e4ed12db 100644 --- a/lib/workload/stateful/filemanager/deploy/stack/stack.ts +++ b/lib/workload/stateful/filemanager/deploy/stack/stack.ts @@ -25,8 +25,7 @@ new FilemanagerStack( // Additionally speed up builds by removing debug info. Please enable this if required. CARGO_PROFILE_RELEASE_DEBUG: 'false', - // Add SCCACHE to speed up compilation. FIXME: Not cross-platform right now as it's defined here :/ - RUSTC_WRAPPER: '/opt/homebrew/bin/sccache', + RUSTC_WRAPPER: `${process.env.HOME}/.cargo/bin/sccache`, }, }, { diff --git a/lib/workload/stateful/filemanager/scripts/deploy.sh b/lib/workload/stateful/filemanager/scripts/deploy.sh index 385fa3c87..35138da61 100755 --- a/lib/workload/stateful/filemanager/scripts/deploy.sh +++ b/lib/workload/stateful/filemanager/scripts/deploy.sh @@ -1,15 +1,16 @@ #!/bin/sh -x # TODO: Takes too long for re-deploy, find further shortcuts -export AWS_ENDPOINT_URL=http://localhost:4566 export FM_BUCKET=filemanager-test-ingest docker compose down docker compose up --wait --wait-timeout 20 -d -cd deploy + +cd deploy || exit npm install yes | npx cdklocal destroy yes | npx cdklocal bootstrap + cd ../database && sqlx migrate run && cd .. cd deploy && yes | npx cdklocal deploy --require-approval never && cd .. diff --git a/lib/workload/stateful/filemanager/scripts/logs.sh b/lib/workload/stateful/filemanager/scripts/logs.sh index b018b5882..bdd068b33 100755 --- a/lib/workload/stateful/filemanager/scripts/logs.sh +++ b/lib/workload/stateful/filemanager/scripts/logs.sh @@ -1,7 +1,5 @@ #!/bin/sh -x -export AWS_ENDPOINT_URL=http://localhost:4566 - # Don't die when the whole stack re-deploys group_name=$(aws logs describe-log-groups --query 'logGroups[*].logGroupName' --output text) aws logs tail "$group_name" --follow From e6629e573204cca7de1acc6149e99b545ce694f5 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 15 Dec 2023 07:38:54 +1100 Subject: [PATCH 02/24] refactor(filemanager): make sccache optional --- lib/workload/stateful/filemanager/README.md | 13 ++++++++++++- .../stateful/filemanager/deploy/stack/stack.ts | 3 ++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/workload/stateful/filemanager/README.md b/lib/workload/stateful/filemanager/README.md index 906324651..0cc85da94 100644 --- a/lib/workload/stateful/filemanager/README.md +++ b/lib/workload/stateful/filemanager/README.md @@ -28,10 +28,21 @@ this might interfere and complain about non-existing roles and users. ### Tooling prerequisites, testing and building the code -For development of the rust workspace, install a build cache (sccache) and build manually: +For development of the rust workspace, it's recommended to install a build cache (sccache) to improve compilation speeds: ```sh brew install sccache && export RUSTC_WRAPPER=`which sccache` +``` + +or + +```sh +cargo install sccache && export RUSTC_WRAPPER=`which sccache` +``` + +Then install build prerequisites to build: + +```sh cargo install cargo-watch sqlx-cli cargo build --all-targets --all-features ``` diff --git a/lib/workload/stateful/filemanager/deploy/stack/stack.ts b/lib/workload/stateful/filemanager/deploy/stack/stack.ts index 2e4ed12db..c6ed020c7 100644 --- a/lib/workload/stateful/filemanager/deploy/stack/stack.ts +++ b/lib/workload/stateful/filemanager/deploy/stack/stack.ts @@ -25,7 +25,8 @@ new FilemanagerStack( // Additionally speed up builds by removing debug info. Please enable this if required. CARGO_PROFILE_RELEASE_DEBUG: 'false', - RUSTC_WRAPPER: `${process.env.HOME}/.cargo/bin/sccache`, + // Use sccache if it is present in the wrapper env. + RUSTC_WRAPPER: process.env.RUSTC_WRAPPER, }, }, { From d9b4a42f84278042501b8add42b1ce0f92f0cf52 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 15 Dec 2023 14:38:28 +1100 Subject: [PATCH 03/24] refactor(filemanager-deploy): add database construct --- .../stateful/filemanager/deploy/README.md | 1 - .../{stack/stack.ts => bin/filemanager.ts} | 2 +- .../stateful/filemanager/deploy/cdk.json | 2 +- .../filemanager/deploy/constructs/database.ts | 168 ++ .../{stack => lib}/filemanager_stack.ts | 14 +- .../filemanager/deploy/package-lock.json | 1397 ++++++++++++++++- .../stateful/filemanager/deploy/package.json | 5 +- 7 files changed, 1541 insertions(+), 48 deletions(-) rename lib/workload/stateful/filemanager/deploy/{stack/stack.ts => bin/filemanager.ts} (95%) create mode 100644 lib/workload/stateful/filemanager/deploy/constructs/database.ts rename lib/workload/stateful/filemanager/deploy/{stack => lib}/filemanager_stack.ts (90%) diff --git a/lib/workload/stateful/filemanager/deploy/README.md b/lib/workload/stateful/filemanager/deploy/README.md index 33425be40..d7fd11fde 100644 --- a/lib/workload/stateful/filemanager/deploy/README.md +++ b/lib/workload/stateful/filemanager/deploy/README.md @@ -2,7 +2,6 @@ This folder contains CDK deployment code for filemanager. The CDK code can be deployed using `cdk`: - ```sh npm install cdk bootstrap diff --git a/lib/workload/stateful/filemanager/deploy/stack/stack.ts b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts similarity index 95% rename from lib/workload/stateful/filemanager/deploy/stack/stack.ts rename to lib/workload/stateful/filemanager/deploy/bin/filemanager.ts index c6ed020c7..db9739c79 100644 --- a/lib/workload/stateful/filemanager/deploy/stack/stack.ts +++ b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts @@ -2,7 +2,7 @@ import 'source-map-support/register'; import * as cdk from 'aws-cdk-lib'; -import { FilemanagerStack } from './filemanager_stack'; +import { FilemanagerStack } from '../lib/filemanager_stack'; export const STACK_NAME = 'FilemanagerStack'; const STACK_DESCRIPTION = 'A stack deploying filemanager to dev.'; diff --git a/lib/workload/stateful/filemanager/deploy/cdk.json b/lib/workload/stateful/filemanager/deploy/cdk.json index 7447948b9..a795fe39a 100644 --- a/lib/workload/stateful/filemanager/deploy/cdk.json +++ b/lib/workload/stateful/filemanager/deploy/cdk.json @@ -1,5 +1,5 @@ { - "app": "npx ts-node --prefer-ts-exts stack/stack.ts", + "app": "npx ts-node --prefer-ts-exts bin/filemanager.ts", "watch": { "include": ["**"], "exclude": [ diff --git a/lib/workload/stateful/filemanager/deploy/constructs/database.ts b/lib/workload/stateful/filemanager/deploy/constructs/database.ts new file mode 100644 index 000000000..d58175849 --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/database.ts @@ -0,0 +1,168 @@ +import { Construct } from 'constructs'; +import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; +import { + AuroraPostgresEngineVersion, + ClusterInstance, + DatabaseCluster, + DatabaseClusterEngine, + PostgresEngineVersion, + ServerlessCluster, +} from 'aws-cdk-lib/aws-rds'; +import { aws_ec2 as ec2, aws_rds as rds, Duration, RemovalPolicy } from 'aws-cdk-lib'; +import { ManagedPolicy, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; + +/** + * Props for enabling enhanced monitoring. + */ +type enableMonitoringProps = { + /** + * Add cloud watch exports. + */ + readonly cloudwatchLogsExports?: string[]; + /** + * Enable performance insights. + */ + readonly enablePerformanceInsights?: boolean; + /** + * The interval for monitoring, defaults to 60 seconds. + */ + readonly monitoringInterval?: Duration; +}; + +/** + * Props for the database + */ +type DatabaseProps = { + /** + * Vpc for the database. + */ + readonly vpc: IVpc; + /** + * The name of the database initially created. + */ + readonly databaseName: string; + /** + * Secret for database credentials + */ + readonly secret: ISecret; + /** + * Whether to destroy the database on stack removal. Defaults to keeping a snapshot. + */ + readonly destroyOnRemove?: boolean; + /** + * Enable enhanced monitoring. + */ + readonly enableMonitoring?: enableMonitoringProps; + /** + * Minimum ACU capacity, defaults to 0.5. + */ + readonly minCapacity?: number; + /** + * Maximum ACU capacity, defaults to 4. + */ + readonly maxCapacity?: number; + /** + * Port to use for the database. The default for the engine is used if not specified. + */ + readonly port?: number; +}; + +/** + * A construct for the postgres database used with filemanager. + */ +export class Database extends Construct { + private readonly _securityGroup: SecurityGroup; + private readonly _cluster: DatabaseCluster; + private readonly _unsafeConnection: string; + + constructor(scope: Construct, id: string, props: DatabaseProps) { + super(scope, id); + + // Create security group with no outbound connections, because outbound connections + // shouldn't be very useful for a database anyway. + this._securityGroup = new SecurityGroup(this, 'SecurityGroup', { + vpc: props.vpc, + allowAllOutbound: false, + allowAllIpv6Outbound: false, + description: 'Security group for communicating with the filemanager RDS instance', + }); + + // Creates roles for enhanced RDS monitoring, if enabled. + let enableMonitoring; + if (props.enableMonitoring) { + const monitoringRole = new Role(this, 'DatabaseMonitoringRole', { + assumedBy: new ServicePrincipal('monitoring.rds.amazonaws.com'), + }); + monitoringRole.addManagedPolicy( + ManagedPolicy.fromAwsManagedPolicyName('service-role/AmazonRDSEnhancedMonitoringRole') + ); + + enableMonitoring = { + enablePerformanceInsights: props.enableMonitoring.enablePerformanceInsights, + cloudwatchLogsExports: props.enableMonitoring.cloudwatchLogsExports, + monitoringInterval: props.enableMonitoring.monitoringInterval?.toSeconds() ?? 60, + monitoringRoleArn: monitoringRole.roleArn, + }; + } + + // Serverless V2 Cluster. + this._cluster = new DatabaseCluster(this, 'Cluster', { + vpc: props.vpc, + vpcSubnets: { + subnetType: SubnetType.PRIVATE_ISOLATED, + }, + securityGroups: [this._securityGroup], + credentials: rds.Credentials.fromSecret(props.secret), + removalPolicy: props.destroyOnRemove ? RemovalPolicy.DESTROY : RemovalPolicy.SNAPSHOT, + defaultDatabaseName: props.databaseName, + port: props.port, + engine: DatabaseClusterEngine.auroraPostgres({ + version: AuroraPostgresEngineVersion.VER_15_4, + }), + serverlessV2MinCapacity: props.minCapacity ?? 0.5, + serverlessV2MaxCapacity: props.maxCapacity ?? 4, + writer: ClusterInstance.serverlessV2('Writer', { + ...(enableMonitoring && { ...enableMonitoring }), + }), + }); + + // Any inbound connections within the same security group are allowed access to the database port. + this._securityGroup.addIngressRule( + this._securityGroup, + ec2.Port.tcp(this._cluster.clusterEndpoint.port) + ); + + this._unsafeConnection = + `postgres://` + + `${props.secret.secretValueFromJson('username').unsafeUnwrap()}` + + `:` + + `${props.secret.secretValueFromJson('password').unsafeUnwrap()}` + + `@` + + `${this._cluster.clusterEndpoint.socketAddress}` + + `/` + + `${props.databaseName}`; + } + + /** + * Get the serverless cluster. + */ + get cluster(): DatabaseCluster { + return this._cluster; + } + + /** + * Get the security group for the database. + */ + get securityGroup(): SecurityGroup { + return this._securityGroup; + } + + /** + * Get the connection string. Unsafe because it contains the username and password. + * Todo might be better to use secret manager within the Lambda to fetch the connection string. + */ + get unsafeConnection(): string { + return this._unsafeConnection; + } +} diff --git a/lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts similarity index 90% rename from lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts rename to lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts index d62721802..056cf9966 100644 --- a/lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts +++ b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts @@ -4,7 +4,6 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import { RustFunction, Settings as CargoSettings } from 'rust.aws-cdk-lambda'; import { Architecture } from 'aws-cdk-lib/aws-lambda'; import * as s3 from 'aws-cdk-lib/aws-s3'; -import * as rds from 'aws-cdk-lib/aws-rds'; import * as sqs from 'aws-cdk-lib/aws-sqs'; import * as lambdaDestinations from 'aws-cdk-lib/aws-lambda-destinations'; import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; @@ -88,7 +87,7 @@ export class FilemanagerStack extends Stack { // VPC //const vpc = ec2.Vpc.fromLookup(this, 'main-vpc', { isDefault: false }); - const vpc = new ec2.Vpc(this, 'vpc', { + new ec2.Vpc(this, 'vpc', { ipAddresses: ec2.IpAddresses.cidr('10.0.0.0/24'), maxAzs: 99, // As many as there are available in the region natGateways: 1, @@ -117,16 +116,5 @@ export class FilemanagerStack extends Stack { generateStringKey: 'password', }, }); - - // RDS - new rds.ServerlessCluster(this, 'Database', { - engine: rds.DatabaseClusterEngine.auroraPostgres({ - version: rds.AuroraPostgresEngineVersion.VER_13_12, - }), - defaultDatabaseName: 'filemanager', - credentials: rds.Credentials.fromGeneratedSecret('filemanager_db_secret'), - removalPolicy: RemovalPolicy.DESTROY, - vpc, - }); } } diff --git a/lib/workload/stateful/filemanager/deploy/package-lock.json b/lib/workload/stateful/filemanager/deploy/package-lock.json index 29ffd9696..b5ed057e2 100644 --- a/lib/workload/stateful/filemanager/deploy/package-lock.json +++ b/lib/workload/stateful/filemanager/deploy/package-lock.json @@ -18,13 +18,24 @@ "source-map-support": "^0.5.21" }, "bin": { - "filemanager-stack": "stack/filemanager_stack.js" + "filemanager": "bin/filemanager.js" }, "devDependencies": { "@types/node": "^20.5.9", + "@typescript-eslint/eslint-plugin": "^6.14.0", "aws-cdk": "^2.114.1", "prettier": "3.0.3", - "typescript": "^5.2.2" + "typescript": "^5.3.3" + } + }, + "node_modules/@aashutoshrathi/word-wrap": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", + "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.10.0" } }, "node_modules/@aws-cdk/asset-awscli-v1": { @@ -80,11 +91,105 @@ "constructs": "^10.0.0" } }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", + "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^3.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", + "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", + "dev": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dev": true, + "peer": true, + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.55.0.tgz", + "integrity": "sha512-qQfo2mxH5yVom1kacMtZZJFVdW+E70mqHMJvVg6WTLo+VBuQJ4TojZlfWBjK0ve5BdEeNAVxOsl/nvNMpJOaJA==", + "dev": true, + "peer": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.11.13", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", + "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", + "dev": true, + "peer": true, + "dependencies": { + "@humanwhocodes/object-schema": "^2.0.1", + "debug": "^4.1.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", + "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", + "dev": true, + "peer": true + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "optional": true, + "devOptional": true, "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" @@ -97,7 +202,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "optional": true, + "devOptional": true, "engines": { "node": ">= 8" } @@ -106,7 +211,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "optional": true, + "devOptional": true, "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" @@ -124,6 +229,12 @@ "@types/node": "*" } }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true + }, "node_modules/@types/minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", @@ -145,12 +256,244 @@ "integrity": "sha512-PtrlVaOaI44/3pl3cvnlK+GxOM3re2526TJvPvh7W+keHIXdV4TE0ylpPBAcvFQCbGitaTXwL9u+RF7qtVeazQ==", "optional": true }, + "node_modules/@types/semver": { + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz", + "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==", + "dev": true + }, "node_modules/@types/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@types/which/-/which-2.0.2.tgz", "integrity": "sha512-113D3mDkZDjo+EeUEHCFy0qniNc1ZpecGiAU7WSo7YDoSzolZIQKpYFHrPpjkB2nuyahcKfrmLXeQlh7gqJYdw==", "optional": true }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.14.0.tgz", + "integrity": "sha512-1ZJBykBCXaSHG94vMMKmiHoL0MhNHKSVlcHVYZNw+BKxufhqQVTOawNpwwI1P5nIFZ/4jLVop0mcY6mJJDFNaw==", + "dev": true, + "dependencies": { + "@eslint-community/regexpp": "^4.5.1", + "@typescript-eslint/scope-manager": "6.14.0", + "@typescript-eslint/type-utils": "6.14.0", + "@typescript-eslint/utils": "6.14.0", + "@typescript-eslint/visitor-keys": "6.14.0", + "debug": "^4.3.4", + "graphemer": "^1.4.0", + "ignore": "^5.2.4", + "natural-compare": "^1.4.0", + "semver": "^7.5.4", + "ts-api-utils": "^1.0.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^6.0.0 || ^6.0.0-alpha", + "eslint": "^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.14.0.tgz", + "integrity": "sha512-QjToC14CKacd4Pa7JK4GeB/vHmWFJckec49FR4hmIRf97+KXole0T97xxu9IFiPxVQ1DBWrQ5wreLwAGwWAVQA==", + "dev": true, + "peer": true, + "dependencies": { + "@typescript-eslint/scope-manager": "6.14.0", + "@typescript-eslint/types": "6.14.0", + "@typescript-eslint/typescript-estree": "6.14.0", + "@typescript-eslint/visitor-keys": "6.14.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.14.0.tgz", + "integrity": "sha512-VT7CFWHbZipPncAZtuALr9y3EuzY1b1t1AEkIq2bTXUPKw+pHoXflGNG5L+Gv6nKul1cz1VH8fz16IThIU0tdg==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "6.14.0", + "@typescript-eslint/visitor-keys": "6.14.0" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.14.0.tgz", + "integrity": "sha512-x6OC9Q7HfYKqjnuNu5a7kffIYs3No30isapRBJl1iCHLitD8O0lFbRcVGiOcuyN837fqXzPZ1NS10maQzZMKqw==", + "dev": true, + "dependencies": { + "@typescript-eslint/typescript-estree": "6.14.0", + "@typescript-eslint/utils": "6.14.0", + "debug": "^4.3.4", + "ts-api-utils": "^1.0.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/types": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.14.0.tgz", + "integrity": "sha512-uty9H2K4Xs8E47z3SnXEPRNDfsis8JO27amp2GNCnzGETEW3yTqEIVg5+AI7U276oGF/tw6ZA+UesxeQ104ceA==", + "dev": true, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.14.0.tgz", + "integrity": "sha512-yPkaLwK0yH2mZKFE/bXkPAkkFgOv15GJAUzgUVonAbv0Hr4PK/N2yaA/4XQbTZQdygiDkpt5DkxPELqHguNvyw==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "6.14.0", + "@typescript-eslint/visitor-keys": "6.14.0", + "debug": "^4.3.4", + "globby": "^11.1.0", + "is-glob": "^4.0.3", + "semver": "^7.5.4", + "ts-api-utils": "^1.0.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.14.0.tgz", + "integrity": "sha512-XwRTnbvRr7Ey9a1NT6jqdKX8y/atWG+8fAIu3z73HSP8h06i3r/ClMhmaF/RGWGW1tHJEwij1uEg2GbEmPYvYg==", + "dev": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.4.0", + "@types/json-schema": "^7.0.12", + "@types/semver": "^7.5.0", + "@typescript-eslint/scope-manager": "6.14.0", + "@typescript-eslint/types": "6.14.0", + "@typescript-eslint/typescript-estree": "6.14.0", + "semver": "^7.5.4" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^7.0.0 || ^8.0.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.14.0.tgz", + "integrity": "sha512-fB5cw6GRhJUz03MrROVuj5Zm/Q+XWlVdIsFj+Zb1Hvqouc8t+XP2H5y53QYU/MGtd2dPg6/vJJlhoX3xc2ehfw==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "6.14.0", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true, + "peer": true + }, "node_modules/@ziglang/cli": { "version": "0.0.11", "resolved": "https://registry.npmjs.org/@ziglang/cli/-/cli-0.0.11.tgz", @@ -169,6 +512,88 @@ "zig-uninstall": "uninstall.js" } }, + "node_modules/acorn": { + "version": "8.11.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.2.tgz", + "integrity": "sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w==", + "dev": true, + "peer": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "peer": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "peer": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "peer": true + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/aws-cdk": { "version": "2.114.1", "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.114.1.tgz", @@ -524,11 +949,29 @@ "node": ">= 6" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "peer": true + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "peer": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/braces": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "optional": true, + "devOptional": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -541,6 +984,16 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/chalk": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", @@ -553,6 +1006,33 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "peer": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "peer": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "peer": true + }, "node_modules/constructs": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.3.0.tgz", @@ -561,6 +1041,21 @@ "node": ">= 16.14.0" } }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "peer": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -570,11 +1065,35 @@ "node": ">= 12" } }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "optional": true, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "peer": true + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "devOptional": true, "dependencies": { "path-type": "^4.0.0" }, @@ -582,6 +1101,19 @@ "node": ">=8" } }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "peer": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/dotenv": { "version": "16.3.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", @@ -599,6 +1131,198 @@ "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", "optional": true }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.55.0.tgz", + "integrity": "sha512-iyUUAM0PCKj5QpwGfmCAG9XXbZCWsqP/eWAWrG/W0umvjuLRBECwSFdt+rCntju0xEH7teIABPwXpahftIaTdA==", + "dev": true, + "peer": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.55.0", + "@humanwhocodes/config-array": "^0.11.13", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "dev": true, + "peer": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "peer": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/eslint/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "peer": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "dev": true, + "peer": true, + "dependencies": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "dev": true, + "peer": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "peer": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/event-stream": { "version": "3.3.4", "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", @@ -614,11 +1338,18 @@ "through": "~2.3.1" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "peer": true + }, "node_modules/fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "optional": true, + "devOptional": true, "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -630,11 +1361,25 @@ "node": ">=8.6.0" } }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "peer": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "peer": true + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", - "optional": true, + "devOptional": true, "dependencies": { "reusify": "^1.0.4" } @@ -662,11 +1407,24 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "peer": true, + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "optional": true, + "devOptional": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -674,6 +1432,45 @@ "node": ">=8" } }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "peer": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "dev": true, + "peer": true, + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.2.9", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.9.tgz", + "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", + "dev": true, + "peer": true + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -706,6 +1503,13 @@ "node": ">=12" } }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "peer": true + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -720,11 +1524,32 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "peer": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "optional": true, + "devOptional": true, "dependencies": { "is-glob": "^4.0.1" }, @@ -732,6 +1557,22 @@ "node": ">= 6" } }, + "node_modules/globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "dev": true, + "peer": true, + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/globby": { "version": "13.2.2", "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz", @@ -757,20 +1598,81 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "optional": true }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/ignore": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz", "integrity": "sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==", - "optional": true, + "devOptional": true, "engines": { "node": ">= 4" } }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "peer": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "peer": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "peer": true + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "optional": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -779,7 +1681,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "optional": true, + "devOptional": true, "dependencies": { "is-extglob": "^2.1.1" }, @@ -791,16 +1693,60 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "optional": true, + "devOptional": true, "engines": { "node": ">=0.12.0" } }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "optional": true + "devOptional": true + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "peer": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "peer": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "peer": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "peer": true }, "node_modules/jsonfile": { "version": "6.1.0", @@ -814,6 +1760,65 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "peer": true, + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "peer": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "peer": true + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/map-stream": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", @@ -824,7 +1829,7 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "optional": true, + "devOptional": true, "engines": { "node": ">= 8" } @@ -833,7 +1838,7 @@ "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "optional": true, + "devOptional": true, "dependencies": { "braces": "^3.0.2", "picomatch": "^2.3.1" @@ -842,6 +1847,19 @@ "node": ">=8.6" } }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "peer": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -851,6 +1869,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -888,11 +1918,114 @@ "url": "https://opencollective.com/node-fetch" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "peer": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", + "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", + "dev": true, + "peer": true, + "dependencies": { + "@aashutoshrathi/word-wrap": "^1.2.3", + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "peer": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "peer": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "peer": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "optional": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -910,7 +2043,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "optional": true, + "devOptional": true, "engines": { "node": ">=8.6" }, @@ -918,6 +2051,16 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "peer": true, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/prettier": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", @@ -948,10 +2091,21 @@ "node": ">= 0.10" } }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "devOptional": true, "funding": [ { "type": "github", @@ -965,23 +2119,49 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "optional": true + ] + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "optional": true, + "devOptional": true, "engines": { "iojs": ">=1.0.0", "node": ">=0.10.0" } }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "peer": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "devOptional": true, "funding": [ { "type": "github", @@ -996,7 +2176,6 @@ "url": "https://feross.org/support" } ], - "optional": true, "dependencies": { "queue-microtask": "^1.2.2" } @@ -1021,6 +2200,44 @@ "constructs": "^10.*" } }, + "node_modules/semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "dev": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "peer": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/slash": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", @@ -1071,6 +2288,52 @@ "duplexer": "~0.1.1" } }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "peer": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "peer": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true, + "peer": true + }, "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -1081,7 +2344,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "optional": true, + "devOptional": true, "dependencies": { "is-number": "^7.0.0" }, @@ -1094,6 +2357,44 @@ "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" }, + "node_modules/ts-api-utils": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.0.3.tgz", + "integrity": "sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==", + "dev": true, + "engines": { + "node": ">=16.13.0" + }, + "peerDependencies": { + "typescript": ">=4.2.0" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/typescript": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", @@ -1122,6 +2423,16 @@ "node": ">= 10.0.0" } }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "peer": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, "node_modules/web-streams-polyfill": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", @@ -1135,7 +2446,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "optional": true, + "devOptional": true, "dependencies": { "isexe": "^2.0.0" }, @@ -1146,6 +2457,19 @@ "node": ">= 8" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "peer": true + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/yaml": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.4.tgz", @@ -1155,6 +2479,19 @@ "node": ">= 14" } }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/zx": { "version": "6.2.5", "resolved": "https://registry.npmjs.org/zx/-/zx-6.2.5.tgz", diff --git a/lib/workload/stateful/filemanager/deploy/package.json b/lib/workload/stateful/filemanager/deploy/package.json index 6a596b27a..8b5ecf21d 100644 --- a/lib/workload/stateful/filemanager/deploy/package.json +++ b/lib/workload/stateful/filemanager/deploy/package.json @@ -2,7 +2,7 @@ "name": "filemanager", "version": "0.1", "bin": { - "filemanager-stack": "stack/filemanager_stack.js" + "filemanager": "bin/filemanager.js" }, "scripts": { "build": "tsc", @@ -11,9 +11,10 @@ }, "devDependencies": { "@types/node": "^20.5.9", + "@typescript-eslint/eslint-plugin": "^6.14.0", "aws-cdk": "^2.114.1", "prettier": "3.0.3", - "typescript": "^5.2.2" + "typescript": "^5.3.3" }, "dependencies": { "@aws-cdk/aws-apigatewayv2-alpha": "^2.114.1-alpha.0", From 334374393a6e8c941756ed54b47144d5f43bffab Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 15 Dec 2023 16:13:35 +1100 Subject: [PATCH 04/24] refactor(filemanager-deploy): add lambda function construct --- .../filemanager/deploy/constructs/database.ts | 1 - .../deploy/constructs/functions/ingest.ts | 103 ++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts diff --git a/lib/workload/stateful/filemanager/deploy/constructs/database.ts b/lib/workload/stateful/filemanager/deploy/constructs/database.ts index d58175849..ddb558f1a 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/database.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/database.ts @@ -160,7 +160,6 @@ export class Database extends Construct { /** * Get the connection string. Unsafe because it contains the username and password. - * Todo might be better to use secret manager within the Lambda to fetch the connection string. */ get unsafeConnection(): string { return this._unsafeConnection; diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts new file mode 100644 index 000000000..c0862da1a --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts @@ -0,0 +1,103 @@ +import { Construct } from 'constructs'; +import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { RustFunction } from 'rust.aws-cdk-lambda'; +import { Duration } from 'aws-cdk-lib'; +import { Architecture, Function, IDestination } from 'aws-cdk-lib/aws-lambda'; +import { Database } from '../database'; +import { IQueue } from 'aws-cdk-lib/aws-sqs'; +import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; +import * as iam from 'aws-cdk-lib/aws-iam'; + +/** + * Props for the database + */ +type IngestFunctionProps = { + /** + * Vpc for the function. + */ + readonly vpc: IVpc; + /** + * Database that the function uses. + */ + readonly database: Database; + /** + * The SQS queue URL to receive events from. + */ + readonly queue: IQueue; + /** + * The destination to post failed invocations to. + */ + readonly onFailure?: IDestination; + /** + * Additional build environment variables when building the Lambda function. + */ + readonly buildEnvironment?: { [key: string]: string | undefined }; + /** + * RUST_LOG string, defaults to trace on local crates and info everywhere else. + */ + readonly rustLog?: string; +}; + +/** + * A construct for the Lambda ingest function. + */ +export class IngestFunction extends Construct { + private readonly _function: RustFunction; + + constructor(scope: Construct, id: string, props: IngestFunctionProps) { + super(scope, id); + + // Lambda role needs SQS execution role. + const lambdaRole = new iam.Role(this, id + 'Role', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + description: 'Lambda execution role for ' + id, + }); + lambdaRole.addManagedPolicy( + iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaSQSQueueExecutionRole') + ); + + // Lambda needs to be able to reach out to access S3, security manager (eventually), etc. + // Could this use an endpoint instead? + const securityGroup = new SecurityGroup(this, 'SecurityGroup', { + vpc: props.vpc, + allowAllOutbound: true, + description: 'Security group that allows the ingest Lambda function to egress out.', + }); + + const filemanagerLambda = new RustFunction(this, 'IngestLambdaFunction', { + package: 'filemanager-ingest-lambda', + target: 'aarch64-unknown-linux-gnu', + memorySize: 128, + timeout: Duration.seconds(28), + environment: { + // Todo use security manager to get connection string rather than passing it in an environment variable + DATABASE_URL: props.database.unsafeConnection, + RUST_LOG: props.rustLog ?? 'info,filemanager_ingest_lambda=trace,filemanager=trace', + }, + buildEnvironment: props.buildEnvironment, + architecture: Architecture.ARM_64, + role: lambdaRole, + onFailure: props.onFailure, + vpc: props.vpc, + // Lambda needs to egress to (eventually) access secrets manager/other sqs endpoints. + vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, + securityGroups: [ + securityGroup, + // Allow access to database. + props.database.securityGroup, + ], + }); + + const eventSource = new lambdaEventSources.SqsEventSource(props.queue); + filemanagerLambda.addEventSource(eventSource); + + // Todo this should probably connect to an RDS proxy rather than directly to the database. + } + + /** + * Get the Lambda function. + */ + get function(): RustFunction { + return this._function; + } +} From 46a250afcf5d0faf4f22f98be9050fd857592f06 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 18 Dec 2023 08:09:41 +1100 Subject: [PATCH 05/24] refactor(filemanager): use new constructs in stack --- .../filemanager/deploy/bin/filemanager.ts | 37 ++------ .../filemanager/deploy/constructs/database.ts | 38 ++++---- .../deploy/constructs/functions/ingest.ts | 35 ++++++-- .../deploy/lib/filemanager_stack.ts | 88 +++++++------------ 4 files changed, 91 insertions(+), 107 deletions(-) diff --git a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts index db9739c79..850a6ccaa 100644 --- a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts +++ b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts @@ -3,37 +3,18 @@ import 'source-map-support/register'; import * as cdk from 'aws-cdk-lib'; import { FilemanagerStack } from '../lib/filemanager_stack'; +import { Tags } from 'aws-cdk-lib'; export const STACK_NAME = 'FilemanagerStack'; const STACK_DESCRIPTION = 'A stack deploying filemanager to dev.'; const app = new cdk.App(); -new FilemanagerStack( - app, - STACK_NAME, - { - database_url: 'postgresql://filemanager:filemanager@db:5432/filemanager', // pragma: allowlist secret - stack_name: STACK_NAME, - buildEnvironment: { - // Override release profile to match defaults for dev builds. - CARGO_PROFILE_RELEASE_OPT_LEVEL: '0', - CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS: 'true', - CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS: 'true', - CARGO_PROFILE_RELEASE_PANIC: 'unwind', - CARGO_PROFILE_RELEASE_INCREMENTAL: 'true', - CARGO_PROFILE_RELEASE_CODEGEN_UNITS: '256', - - // Additionally speed up builds by removing debug info. Please enable this if required. - CARGO_PROFILE_RELEASE_DEBUG: 'false', - // Use sccache if it is present in the wrapper env. - RUSTC_WRAPPER: process.env.RUSTC_WRAPPER, - }, +new FilemanagerStack(app, STACK_NAME, { + stackName: STACK_NAME, + description: STACK_DESCRIPTION, + tags: { + Stack: STACK_NAME, }, - { - stackName: STACK_NAME, - description: STACK_DESCRIPTION, - tags: { - Stack: STACK_NAME, - }, - } -); +}); + +Tags.of(app).add('Stack', STACK_NAME); diff --git a/lib/workload/stateful/filemanager/deploy/constructs/database.ts b/lib/workload/stateful/filemanager/deploy/constructs/database.ts index ddb558f1a..8d25e1986 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/database.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/database.ts @@ -15,7 +15,7 @@ import { ManagedPolicy, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; /** * Props for enabling enhanced monitoring. */ -type enableMonitoringProps = { +export type EnableMonitoringProps = { /** * Add cloud watch exports. */ @@ -31,21 +31,9 @@ type enableMonitoringProps = { }; /** - * Props for the database + * Settable database props that can be configured. */ -type DatabaseProps = { - /** - * Vpc for the database. - */ - readonly vpc: IVpc; - /** - * The name of the database initially created. - */ - readonly databaseName: string; - /** - * Secret for database credentials - */ - readonly secret: ISecret; +export type DatabaseSettings = { /** * Whether to destroy the database on stack removal. Defaults to keeping a snapshot. */ @@ -53,7 +41,7 @@ type DatabaseProps = { /** * Enable enhanced monitoring. */ - readonly enableMonitoring?: enableMonitoringProps; + readonly enableMonitoring?: EnableMonitoringProps; /** * Minimum ACU capacity, defaults to 0.5. */ @@ -68,6 +56,24 @@ type DatabaseProps = { readonly port?: number; }; +/** + * Props for the database. + */ +export type DatabaseProps = DatabaseSettings & { + /** + * Vpc for the database. + */ + readonly vpc: IVpc; + /** + * Secret for database credentials + */ + readonly secret: ISecret; + /** + * The name of the database initially created. + */ + readonly databaseName: string; +}; + /** * A construct for the postgres database used with filemanager. */ diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts index c0862da1a..cfc754f7b 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts @@ -7,11 +7,27 @@ import { Database } from '../database'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; import * as iam from 'aws-cdk-lib/aws-iam'; +import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; +import { IPolicy, PolicyStatement } from 'aws-cdk-lib/aws-iam'; + +/** + * Settable values for the ingest function. + */ +export type IngestFunctionSettings = { + /** + * Additional build environment variables when building the Lambda function. + */ + readonly buildEnvironment?: { [key: string]: string | undefined }; + /** + * RUST_LOG string, defaults to trace on local crates and info everywhere else. + */ + readonly rustLog?: string; +}; /** * Props for the database */ -type IngestFunctionProps = { +export type IngestFunctionProps = IngestFunctionSettings & { /** * Vpc for the function. */ @@ -29,13 +45,9 @@ type IngestFunctionProps = { */ readonly onFailure?: IDestination; /** - * Additional build environment variables when building the Lambda function. - */ - readonly buildEnvironment?: { [key: string]: string | undefined }; - /** - * RUST_LOG string, defaults to trace on local crates and info everywhere else. + * Additional policies to add to the Lambda role. */ - readonly rustLog?: string; + readonly policies?: PolicyStatement[]; }; /** @@ -55,6 +67,9 @@ export class IngestFunction extends Construct { lambdaRole.addManagedPolicy( iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaSQSQueueExecutionRole') ); + props.policies?.forEach((policy) => { + lambdaRole.addToPolicy(policy); + }); // Lambda needs to be able to reach out to access S3, security manager (eventually), etc. // Could this use an endpoint instead? @@ -64,13 +79,16 @@ export class IngestFunction extends Construct { description: 'Security group that allows the ingest Lambda function to egress out.', }); + CargoSettings.WORKSPACE_DIR = '../'; + CargoSettings.BUILD_INDIVIDUALLY = true; + const filemanagerLambda = new RustFunction(this, 'IngestLambdaFunction', { package: 'filemanager-ingest-lambda', target: 'aarch64-unknown-linux-gnu', memorySize: 128, timeout: Duration.seconds(28), environment: { - // Todo use security manager to get connection string rather than passing it in an environment variable + // Todo use security manager to get connection string rather than passing it in as an environment variable DATABASE_URL: props.database.unsafeConnection, RUST_LOG: props.rustLog ?? 'info,filemanager_ingest_lambda=trace,filemanager=trace', }, @@ -79,7 +97,6 @@ export class IngestFunction extends Construct { role: lambdaRole, onFailure: props.onFailure, vpc: props.vpc, - // Lambda needs to egress to (eventually) access secrets manager/other sqs endpoints. vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, securityGroups: [ securityGroup, diff --git a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts index 056cf9966..774a7cebc 100644 --- a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts +++ b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts @@ -10,31 +10,24 @@ import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; import * as s3n from 'aws-cdk-lib/aws-s3-notifications'; +import { IngestFunction, IngestFunctionSettings } from '../constructs/functions/ingest'; +import { Database, EnableMonitoringProps, DatabaseSettings } from '../constructs/database'; /** * Common settings for the filemanager stack. */ -interface Settings { - database_url: string; - endpoint_url?: string; - stack_name: string; - buildEnvironment?: NodeJS.ProcessEnv; -} +type Settings = DatabaseSettings & + IngestFunctionSettings & { + databaseName?: string; + }; /** * Stack used to deploy filemanager. */ export class FilemanagerStack extends Stack { - constructor(scope: Construct, id: string, settings: Settings, props?: StackProps) { + constructor(scope: Construct, id: string, props: StackProps, settings?: Settings) { super(scope, id, props); - Tags.of(this).add('Stack', settings.stack_name); - - const lambdaRole = new iam.Role(this, id + 'Role', { - assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), - description: 'Lambda execution role for ' + id, - }); - const queue = new sqs.Queue(this, id + 'Queue'); const testBucket = new s3.Bucket(this, id + 'Bucket', { @@ -44,50 +37,17 @@ export class FilemanagerStack extends Stack { enforceSSL: true, removalPolicy: RemovalPolicy.DESTROY, }); - - testBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.SqsDestination(queue)); - testBucket.addEventNotification(s3.EventType.OBJECT_REMOVED, new s3n.SqsDestination(queue)); - - lambdaRole.addManagedPolicy( - iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaSQSQueueExecutionRole') - ); - - const s3BucketPolicy = new iam.PolicyStatement({ + const testBucketPolicy = new iam.PolicyStatement({ actions: ['s3:List*', 's3:Get*'], resources: ['arn:aws:s3:::*'], }); - lambdaRole.addToPolicy(s3BucketPolicy); + testBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.SqsDestination(queue)); + testBucket.addEventNotification(s3.EventType.OBJECT_REMOVED, new s3n.SqsDestination(queue)); const deadLetterQueue = new sqs.Queue(this, id + 'DeadLetterQueue'); const deadLetterQueueDestination = new lambdaDestinations.SqsDestination(deadLetterQueue); - CargoSettings.WORKSPACE_DIR = '../'; - CargoSettings.BUILD_INDIVIDUALLY = true; - - const filemanagerLambda = new RustFunction(this, id + 'IngestLambdaFunction', { - package: 'filemanager-ingest-lambda', - target: 'aarch64-unknown-linux-gnu', - - memorySize: 128, - timeout: Duration.seconds(28), - environment: { - DATABASE_URL: settings.database_url, - ...(settings.endpoint_url && { ENDPOINT_URL: settings.endpoint_url }), - SQS_QUEUE_URL: queue.queueUrl, - RUST_LOG: 'info,filemanager_ingest_lambda=trace,filemanager=trace', - }, - buildEnvironment: settings.buildEnvironment, - architecture: Architecture.ARM_64, - role: lambdaRole, - onFailure: deadLetterQueueDestination, - }); - - const eventSource = new lambdaEventSources.SqsEventSource(queue); - filemanagerLambda.addEventSource(eventSource); - - // VPC - //const vpc = ec2.Vpc.fromLookup(this, 'main-vpc', { isDefault: false }); - new ec2.Vpc(this, 'vpc', { + const vpc = new ec2.Vpc(this, 'vpc', { ipAddresses: ec2.IpAddresses.cidr('10.0.0.0/24'), maxAzs: 99, // As many as there are available in the region natGateways: 1, @@ -107,14 +67,34 @@ export class FilemanagerStack extends Stack { ], }); - // Secret - new secretsmanager.Secret(this, 'filemanager_db_secret', { - secretName: 'filemanager_db_secret', // pragma: allowlist secret + const secret = new secretsmanager.Secret(this, 'FilemanagerDatabaseSecret', { + secretName: 'FilemanagerDatabaseSecret', // pragma: allowlist secret generateSecretString: { secretStringTemplate: JSON.stringify({ username: 'filemanager' }), excludePunctuation: true, generateStringKey: 'password', }, }); + + const database = new Database(this, 'Database', { + vpc, + databaseName: settings?.databaseName ?? 'filemanager', + secret, + destroyOnRemove: settings?.destroyOnRemove, + enableMonitoring: settings?.enableMonitoring, + minCapacity: settings?.minCapacity, + maxCapacity: settings?.maxCapacity, + port: settings?.port, + }); + + new IngestFunction(this, 'IngestLambda', { + vpc, + database, + queue, + onFailure: deadLetterQueueDestination, + policies: [testBucketPolicy], + buildEnvironment: settings?.buildEnvironment, + rustLog: settings?.rustLog, + }); } } From 3cf928b5fc5af3a89901b8073b3bcba12e914aaf Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 18 Dec 2023 09:36:29 +1100 Subject: [PATCH 06/24] refactor(filemanager): update default region --- .../filemanager/deploy/bin/filemanager.ts | 25 +++++++++--- .../filemanager/deploy/constructs/database.ts | 2 - .../deploy/constructs/functions/ingest.ts | 4 +- .../deploy/lib/filemanager_stack.ts | 40 +++++++++---------- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts index 850a6ccaa..c10acc29c 100644 --- a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts +++ b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts @@ -9,12 +9,25 @@ export const STACK_NAME = 'FilemanagerStack'; const STACK_DESCRIPTION = 'A stack deploying filemanager to dev.'; const app = new cdk.App(); -new FilemanagerStack(app, STACK_NAME, { - stackName: STACK_NAME, - description: STACK_DESCRIPTION, - tags: { - Stack: STACK_NAME, +new FilemanagerStack( + app, + STACK_NAME, + { + stackName: STACK_NAME, + description: STACK_DESCRIPTION, + tags: { + Stack: STACK_NAME, + }, + env: { + region: 'ap-southeast-2', + }, }, -}); + { + destroyOnRemove: true, + enableMonitoring: { + enablePerformanceInsights: true, + }, + } +); Tags.of(app).add('Stack', STACK_NAME); diff --git a/lib/workload/stateful/filemanager/deploy/constructs/database.ts b/lib/workload/stateful/filemanager/deploy/constructs/database.ts index 8d25e1986..f60ea6445 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/database.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/database.ts @@ -6,8 +6,6 @@ import { ClusterInstance, DatabaseCluster, DatabaseClusterEngine, - PostgresEngineVersion, - ServerlessCluster, } from 'aws-cdk-lib/aws-rds'; import { aws_ec2 as ec2, aws_rds as rds, Duration, RemovalPolicy } from 'aws-cdk-lib'; import { ManagedPolicy, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts index cfc754f7b..184932df8 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts @@ -2,13 +2,13 @@ import { Construct } from 'constructs'; import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { RustFunction } from 'rust.aws-cdk-lambda'; import { Duration } from 'aws-cdk-lib'; -import { Architecture, Function, IDestination } from 'aws-cdk-lib/aws-lambda'; +import { Architecture, IDestination } from 'aws-cdk-lib/aws-lambda'; import { Database } from '../database'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; import * as iam from 'aws-cdk-lib/aws-iam'; import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; -import { IPolicy, PolicyStatement } from 'aws-cdk-lib/aws-iam'; +import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; /** * Settable values for the ingest function. diff --git a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts index 774a7cebc..097406480 100644 --- a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts +++ b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts @@ -1,17 +1,16 @@ -import { Duration, RemovalPolicy, Stack, StackProps, Tags } from 'aws-cdk-lib'; +import { RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib'; import { Construct } from 'constructs'; -import * as iam from 'aws-cdk-lib/aws-iam'; -import { RustFunction, Settings as CargoSettings } from 'rust.aws-cdk-lambda'; -import { Architecture } from 'aws-cdk-lib/aws-lambda'; import * as s3 from 'aws-cdk-lib/aws-s3'; -import * as sqs from 'aws-cdk-lib/aws-sqs'; import * as lambdaDestinations from 'aws-cdk-lib/aws-lambda-destinations'; -import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; -import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; -import * as s3n from 'aws-cdk-lib/aws-s3-notifications'; import { IngestFunction, IngestFunctionSettings } from '../constructs/functions/ingest'; -import { Database, EnableMonitoringProps, DatabaseSettings } from '../constructs/database'; +import { Database, DatabaseSettings } from '../constructs/database'; +import { SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { SqsDestination } from 'aws-cdk-lib/aws-s3-notifications'; +import { Bucket, EventType } from 'aws-cdk-lib/aws-s3'; +import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; +import { Queue } from 'aws-cdk-lib/aws-sqs'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; /** * Common settings for the filemanager stack. @@ -28,46 +27,45 @@ export class FilemanagerStack extends Stack { constructor(scope: Construct, id: string, props: StackProps, settings?: Settings) { super(scope, id, props); - const queue = new sqs.Queue(this, id + 'Queue'); + const queue = new Queue(this, id + 'Queue'); - const testBucket = new s3.Bucket(this, id + 'Bucket', { + const testBucket = new Bucket(this, id + 'Bucket', { bucketName: 'filemanager-test-ingest', blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, encryption: s3.BucketEncryption.S3_MANAGED, enforceSSL: true, removalPolicy: RemovalPolicy.DESTROY, }); - const testBucketPolicy = new iam.PolicyStatement({ + const testBucketPolicy = new PolicyStatement({ actions: ['s3:List*', 's3:Get*'], resources: ['arn:aws:s3:::*'], }); - testBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.SqsDestination(queue)); - testBucket.addEventNotification(s3.EventType.OBJECT_REMOVED, new s3n.SqsDestination(queue)); + testBucket.addEventNotification(EventType.OBJECT_CREATED, new SqsDestination(queue)); + testBucket.addEventNotification(EventType.OBJECT_REMOVED, new SqsDestination(queue)); - const deadLetterQueue = new sqs.Queue(this, id + 'DeadLetterQueue'); + const deadLetterQueue = new Queue(this, id + 'DeadLetterQueue'); const deadLetterQueueDestination = new lambdaDestinations.SqsDestination(deadLetterQueue); - const vpc = new ec2.Vpc(this, 'vpc', { - ipAddresses: ec2.IpAddresses.cidr('10.0.0.0/24'), + const vpc = new ec2.Vpc(this, 'Vpc', { maxAzs: 99, // As many as there are available in the region natGateways: 1, subnetConfiguration: [ { name: 'ingress', - subnetType: ec2.SubnetType.PUBLIC, + subnetType: SubnetType.PUBLIC, }, { name: 'application', - subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + subnetType: SubnetType.PRIVATE_WITH_EGRESS, }, { name: 'database', - subnetType: ec2.SubnetType.PRIVATE_ISOLATED, + subnetType: SubnetType.PRIVATE_ISOLATED, }, ], }); - const secret = new secretsmanager.Secret(this, 'FilemanagerDatabaseSecret', { + const secret = new Secret(this, 'FilemanagerDatabaseSecret', { secretName: 'FilemanagerDatabaseSecret', // pragma: allowlist secret generateSecretString: { secretStringTemplate: JSON.stringify({ username: 'filemanager' }), From 64ff6cdb98932c5f7dcaf0f3d6c6cd5a21588520 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 18 Dec 2023 11:44:21 +1100 Subject: [PATCH 07/24] fix(filemanager): records field --- .../filemanager/deploy/constructs/functions/ingest.ts | 5 +++++ .../stateful/filemanager/filemanager/src/events/aws/mod.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts index 184932df8..6611259b0 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts @@ -64,9 +64,14 @@ export class IngestFunction extends Construct { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), description: 'Lambda execution role for ' + id, }); + // The Lambda needs SQS read to get events from SQS. lambdaRole.addManagedPolicy( iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaSQSQueueExecutionRole') ); + // Lambda needs VPC access if it is created in a VPC. + lambdaRole.addManagedPolicy( + iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole') + ); props.policies?.forEach((policy) => { lambdaRole.addToPolicy(policy); }); diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs index 2e85e0be1..c475eadf8 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs @@ -315,7 +315,7 @@ impl FlatS3EventMessage { #[derive(Debug, Serialize, Deserialize, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct S3EventMessage { - #[serde(alias = "Records")] + #[serde(rename = "Records")] pub records: Vec, } From dc84c20a3a98cd35eaa3f6efe4767e64e65e7b86 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 18 Dec 2023 12:54:03 +1100 Subject: [PATCH 08/24] feat(filemanager): add basic migration lambda --- lib/workload/stateful/filemanager/Cargo.lock | 16 ++++++++ lib/workload/stateful/filemanager/Cargo.toml | 1 + .../{ => aws}/0002_add_s3_object_table.sql | 0 .../filemanager-migrate-lambda/Cargo.toml | 22 ++++++++++ .../filemanager-migrate-lambda/src/main.rs | 16 ++++++++ .../filemanager/filemanager/Cargo.toml | 4 ++ .../filemanager/src/database/aws/migration.rs | 41 +++++++++++++++++++ .../filemanager/src/database/aws/mod.rs | 3 ++ 8 files changed, 103 insertions(+) rename lib/workload/stateful/filemanager/database/migrations/{ => aws}/0002_add_s3_object_table.sql (100%) create mode 100644 lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml create mode 100644 lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs create mode 100644 lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs diff --git a/lib/workload/stateful/filemanager/Cargo.lock b/lib/workload/stateful/filemanager/Cargo.lock index 69f596575..07de8eaf6 100644 --- a/lib/workload/stateful/filemanager/Cargo.lock +++ b/lib/workload/stateful/filemanager/Cargo.lock @@ -1122,6 +1122,22 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "filemanager-migrate-lambda" +version = "0.1.0" +dependencies = [ + "aws-config", + "aws-sdk-sts", + "aws_lambda_events 0.12.1", + "filemanager", + "lambda_runtime", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "finl_unicode" version = "1.2.0" diff --git a/lib/workload/stateful/filemanager/Cargo.toml b/lib/workload/stateful/filemanager/Cargo.toml index e498db2bf..40e4c7565 100644 --- a/lib/workload/stateful/filemanager/Cargo.toml +++ b/lib/workload/stateful/filemanager/Cargo.toml @@ -5,6 +5,7 @@ members = [ "filemanager", "filemanager-http-lambda", "filemanager-ingest-lambda", + "filemanager-migrate-lambda" ] [workspace.package] diff --git a/lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql b/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql similarity index 100% rename from lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql rename to lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml new file mode 100644 index 000000000..699c4e1d4 --- /dev/null +++ b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "filemanager-migrate-lambda" +version = "0.1.0" +license.workspace = true +edition.workspace = true +authors.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +aws_lambda_events = "0.12" +lambda_runtime = "0.8" +serde = "1" +tokio = { version = "1", features = ["macros"] } +tracing = { version = "0.1", features = ["log"] } +tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt"] } + +filemanager = { path = "../filemanager" } +serde_json = "1" + +aws-config = "1" +aws-sdk-sts = "1" \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs new file mode 100644 index 000000000..7c7382570 --- /dev/null +++ b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs @@ -0,0 +1,16 @@ +use lambda_runtime::Error; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{fmt, EnvFilter}; + +#[tokio::main] +async fn main() -> Result<(), Error> { + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + + tracing_subscriber::registry() + .with(fmt::layer().json().without_time()) + .with(env_filter) + .init(); + + todo!(); +} diff --git a/lib/workload/stateful/filemanager/filemanager/Cargo.toml b/lib/workload/stateful/filemanager/filemanager/Cargo.toml index 161f5c492..7c3642921 100644 --- a/lib/workload/stateful/filemanager/filemanager/Cargo.toml +++ b/lib/workload/stateful/filemanager/filemanager/Cargo.toml @@ -6,6 +6,10 @@ authors.workspace = true license.workspace = true edition.workspace = true +[features] + +migrate = ["sqlx/migrate"] + [dependencies] axum = "0.6" hyper = { version = "1", features = ["full"] } diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs new file mode 100644 index 000000000..9b0166927 --- /dev/null +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs @@ -0,0 +1,41 @@ +use crate::database::Client; +use crate::error::Error::MigrateError; +use crate::error::Result; +use sqlx::migrate; + +/// A struct to perform database migrations. +#[derive(Debug)] +pub struct Migration { + client: Client, +} + +impl Migration { + /// Create a new migration. + pub fn new(client: Client) -> Self { + Self { client } + } + + /// Create a new migration with a default database client. + pub async fn with_defaults() -> Result { + Ok(Self { + client: Client::default().await?, + }) + } + + /// Apply migrations. + pub async fn migrate(&mut self) -> Result<()> { + migrate!("../database/migrations") + .run(self.client().pool()) + .await + .map_err(|err| MigrateError(err.to_string()))?; + migrate!("../database/migrations/aws") + .run(self.client().pool()) + .await + .map_err(|err| MigrateError(err.to_string())) + } + + /// Get a reference to the database client. + pub fn client(&self) -> &Client { + &self.client + } +} diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs index 55e301256..2a3edc691 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs @@ -5,6 +5,9 @@ use aws_sdk_s3::types::StorageClass; pub mod ingester; +#[cfg(feature = "migrate")] +pub mod migration; + /// An S3 object which matches the s3 object schema. #[derive(Debug, Clone)] pub struct CloudObject { From f824c359406e88e8a9a87d03ab1281eb25c1d2d0 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 18 Dec 2023 14:12:37 +1100 Subject: [PATCH 09/24] feat(filemanager): migrate in Lambda function --- .../filemanager/filemanager-migrate-lambda/Cargo.toml | 2 +- .../filemanager/filemanager-migrate-lambda/src/main.rs | 3 ++- .../filemanager/filemanager/src/database/aws/migration.rs | 1 + .../stateful/filemanager/filemanager/src/database/mod.rs | 7 +++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml index 699c4e1d4..29ce7ad47 100644 --- a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml +++ b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml @@ -15,7 +15,7 @@ tokio = { version = "1", features = ["macros"] } tracing = { version = "0.1", features = ["log"] } tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt"] } -filemanager = { path = "../filemanager" } +filemanager = { path = "../filemanager", features = ["migrate"] } serde_json = "1" aws-config = "1" diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs index 7c7382570..647b44c76 100644 --- a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs +++ b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs @@ -2,6 +2,7 @@ use lambda_runtime::Error; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{fmt, EnvFilter}; +use filemanager::database::aws::migration::Migration; #[tokio::main] async fn main() -> Result<(), Error> { @@ -12,5 +13,5 @@ async fn main() -> Result<(), Error> { .with(env_filter) .init(); - todo!(); + Migration::with_defaults().await?.migrate() } diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs index 9b0166927..814baf139 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs @@ -24,6 +24,7 @@ impl Migration { /// Apply migrations. pub async fn migrate(&mut self) -> Result<()> { + // Note, these get compiled into the source code. migrate!("../database/migrations") .run(self.client().pool()) .await diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs index 0cec9a595..192dbb9e1 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs @@ -41,3 +41,10 @@ pub trait Ingest { /// Ingest the events. async fn ingest(&mut self, events: EventType) -> Result<()>; } + +/// Trait representing database migrations. +#[async_trait] +pub trait Migrate { + /// Migrate the database. + async fn migrate(&mut self) -> Result<()>; +} From 367663f63f80b7325fa5a5de8d08b0da7249634a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 18 Dec 2023 14:49:34 +1100 Subject: [PATCH 10/24] refactor(filemanager): merge migrations to obtain migrator struct --- lib/workload/stateful/filemanager/Cargo.lock | 1 + .../filemanager-migrate-lambda/src/main.rs | 10 ++- .../filemanager/filemanager/Cargo.toml | 3 + .../filemanager/src/database/aws/ingester.rs | 10 +-- .../filemanager/src/database/aws/migration.rs | 63 +++++++++++++++---- .../filemanager/src/database/mod.rs | 4 +- .../filemanager/src/handlers/aws.rs | 4 +- 7 files changed, 71 insertions(+), 24 deletions(-) diff --git a/lib/workload/stateful/filemanager/Cargo.lock b/lib/workload/stateful/filemanager/Cargo.lock index 07de8eaf6..a3f2f26ed 100644 --- a/lib/workload/stateful/filemanager/Cargo.lock +++ b/lib/workload/stateful/filemanager/Cargo.lock @@ -1073,6 +1073,7 @@ dependencies = [ "axum", "chrono", "dotenvy", + "filemanager", "futures", "hyper 1.0.1", "lambda_runtime", diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs index 647b44c76..66cf53f83 100644 --- a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs +++ b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs @@ -1,8 +1,9 @@ -use lambda_runtime::Error; +use filemanager::database::aws::migration::Migration; +use filemanager::database::Migrate; +use lambda_runtime::{run, service_fn, Error, LambdaEvent}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{fmt, EnvFilter}; -use filemanager::database::aws::migration::Migration; #[tokio::main] async fn main() -> Result<(), Error> { @@ -13,5 +14,8 @@ async fn main() -> Result<(), Error> { .with(env_filter) .init(); - Migration::with_defaults().await?.migrate() + run(service_fn(|_: LambdaEvent<()>| async move { + Migration::with_defaults().await?.migrate().await + })) + .await } diff --git a/lib/workload/stateful/filemanager/filemanager/Cargo.toml b/lib/workload/stateful/filemanager/filemanager/Cargo.toml index 7c3642921..a92dbda5f 100644 --- a/lib/workload/stateful/filemanager/filemanager/Cargo.toml +++ b/lib/workload/stateful/filemanager/filemanager/Cargo.toml @@ -43,3 +43,6 @@ futures = "0.3" [dev-dependencies] aws-smithy-runtime-api = "1" + +# The migrate feature is required to run sqlx tests +filemanager = { path = ".", features = ["migrate"] } diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs index 2ab8697d7..04188d929 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs @@ -29,7 +29,7 @@ impl Ingester { } /// Ingest the events into the database by calling the insert and update queries. - pub async fn ingest_events(&mut self, events: Events) -> Result<()> { + pub async fn ingest_events(&self, events: Events) -> Result<()> { let Events { object_created, object_removed, @@ -100,7 +100,7 @@ impl Ingester { #[async_trait] impl Ingest for Ingester { - async fn ingest(&mut self, events: EventType) -> Result<()> { + async fn ingest(&self, events: EventType) -> Result<()> { match events { EventType::S3(events) => self.ingest_events(events).await, } @@ -123,7 +123,7 @@ pub(crate) mod tests { let mut events = test_events(); events.object_removed = Default::default(); - let mut ingester = test_ingester(pool); + let ingester = test_ingester(pool); ingester.ingest_events(events).await.unwrap(); let result = sqlx::query("select * from object") @@ -138,7 +138,7 @@ pub(crate) mod tests { async fn ingest_object_removed(pool: PgPool) { let events = test_events(); - let mut ingester = test_ingester(pool); + let ingester = test_ingester(pool); ingester.ingest_events(events).await.unwrap(); let result = sqlx::query("select * from object") @@ -153,7 +153,7 @@ pub(crate) mod tests { async fn ingest(pool: PgPool) { let events = test_events(); - let mut ingester = test_ingester(pool); + let ingester = test_ingester(pool); ingester.ingest(EventType::S3(events)).await.unwrap(); let result = sqlx::query("select * from object") diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs index 814baf139..2fa883a9d 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs @@ -1,7 +1,9 @@ -use crate::database::Client; +use crate::database::{Client, Migrate}; use crate::error::Error::MigrateError; use crate::error::Result; +use async_trait::async_trait; use sqlx::migrate; +use sqlx::migrate::Migrator; /// A struct to perform database migrations. #[derive(Debug)] @@ -22,17 +24,17 @@ impl Migration { }) } - /// Apply migrations. - pub async fn migrate(&mut self) -> Result<()> { - // Note, these get compiled into the source code. - migrate!("../database/migrations") - .run(self.client().pool()) - .await - .map_err(|err| MigrateError(err.to_string()))?; - migrate!("../database/migrations/aws") - .run(self.client().pool()) - .await - .map_err(|err| MigrateError(err.to_string())) + /// Get the underlying sqlx migrator for the migrations. + pub fn migrator() -> Migrator { + let mut migrator = migrate!("../database/migrations"); + let aws_migrations = migrate!("../database/migrations/aws"); + + migrator + .migrations + .to_mut() + .extend(aws_migrations.migrations.into_owned()); + + migrator } /// Get a reference to the database client. @@ -40,3 +42,40 @@ impl Migration { &self.client } } + +#[async_trait] +impl Migrate for Migration { + async fn migrate(&self) -> Result<()> { + Self::migrator() + .run(self.client().pool()) + .await + .map_err(|err| MigrateError(err.to_string())) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use sqlx::{PgPool, Row}; + + use super::*; + + #[sqlx::test(migrations = false)] + async fn test_migrate(pool: PgPool) { + let migrate = Migration::new(Client::new(pool)); + + let result = sqlx::query("select * from object") + .fetch_one(migrate.client.pool()) + .await; + + assert!(result.is_err()); + + migrate.migrate().await.unwrap(); + + let result = sqlx::query("select * from object") + .fetch_one(migrate.client.pool()) + .await + .unwrap(); + + assert!(result.is_empty()); + } +} diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs index 192dbb9e1..0a3413847 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs @@ -39,12 +39,12 @@ impl Client { #[async_trait] pub trait Ingest { /// Ingest the events. - async fn ingest(&mut self, events: EventType) -> Result<()>; + async fn ingest(&self, events: EventType) -> Result<()>; } /// Trait representing database migrations. #[async_trait] pub trait Migrate { /// Migrate the database. - async fn migrate(&mut self) -> Result<()>; + async fn migrate(&self) -> Result<()>; } diff --git a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs index f3331acf9..c1a4c2824 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs @@ -32,7 +32,7 @@ pub async fn receive_and_ingest( .collect() .await?; - let mut ingester = if let Some(database_client) = database_client { + let ingester = if let Some(database_client) = database_client { Ingester::new(database_client) } else { Ingester::with_defaults().await? @@ -74,7 +74,7 @@ pub async fn ingest_event( trace!("ingesting events: {:?}", events); - let mut ingester = if let Some(database_client) = database_client { + let ingester = if let Some(database_client) = database_client { Ingester::new(database_client) } else { Ingester::with_defaults().await? From 147a893094f3beec42582a2d5c4fca5ce22e703a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 18 Dec 2023 15:49:09 +1100 Subject: [PATCH 11/24] test(filemanager): migration test --- lib/workload/stateful/filemanager/Cargo.lock | 5 ++-- .../filemanager/filemanager/Cargo.toml | 1 + .../filemanager/src/database/aws/ingester.rs | 7 +++-- .../filemanager/src/database/aws/migration.rs | 30 ++++++++++++------- .../filemanager/src/handlers/aws.rs | 5 ++-- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/lib/workload/stateful/filemanager/Cargo.lock b/lib/workload/stateful/filemanager/Cargo.lock index a3f2f26ed..e7cf0b3e5 100644 --- a/lib/workload/stateful/filemanager/Cargo.lock +++ b/lib/workload/stateful/filemanager/Cargo.lock @@ -1077,6 +1077,7 @@ dependencies = [ "futures", "hyper 1.0.1", "lambda_runtime", + "lazy_static", "mockall", "mockall_double", "serde", @@ -1963,9 +1964,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl-probe" diff --git a/lib/workload/stateful/filemanager/filemanager/Cargo.toml b/lib/workload/stateful/filemanager/filemanager/Cargo.toml index a92dbda5f..5262ca838 100644 --- a/lib/workload/stateful/filemanager/filemanager/Cargo.toml +++ b/lib/workload/stateful/filemanager/filemanager/Cargo.toml @@ -46,3 +46,4 @@ aws-smithy-runtime-api = "1" # The migrate feature is required to run sqlx tests filemanager = { path = ".", features = ["migrate"] } +lazy_static = "1.4" diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs index 04188d929..3254f890d 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs @@ -110,6 +110,7 @@ impl Ingest for Ingester { #[cfg(test)] pub(crate) mod tests { use crate::database::aws::ingester::Ingester; + use crate::database::aws::migration::tests::MIGRATOR; use crate::database::{Client, Ingest}; use crate::events::aws::tests::{expected_events, EXPECTED_E_TAG}; use crate::events::aws::{Events, StorageClass}; @@ -118,7 +119,7 @@ pub(crate) mod tests { use sqlx::postgres::PgRow; use sqlx::{PgPool, Row}; - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn ingest_object_created(pool: PgPool) { let mut events = test_events(); events.object_removed = Default::default(); @@ -134,7 +135,7 @@ pub(crate) mod tests { assert_created(result); } - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn ingest_object_removed(pool: PgPool) { let events = test_events(); @@ -149,7 +150,7 @@ pub(crate) mod tests { assert_deleted(result); } - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn ingest(pool: PgPool) { let events = test_events(); diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs index 2fa883a9d..71fbbb6c6 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs @@ -55,27 +55,37 @@ impl Migrate for Migration { #[cfg(test)] pub(crate) mod tests { - use sqlx::{PgPool, Row}; + use lazy_static::lazy_static; + use sqlx::PgPool; use super::*; + lazy_static! { + pub(crate) static ref MIGRATOR: Migrator = Migration::migrator(); + } + #[sqlx::test(migrations = false)] async fn test_migrate(pool: PgPool) { let migrate = Migration::new(Client::new(pool)); - let result = sqlx::query("select * from object") - .fetch_one(migrate.client.pool()) - .await; + let not_exists = sqlx::query!( + "select exists (select from information_schema.tables where table_name = 'object')" + ) + .fetch_one(migrate.client.pool()) + .await + .unwrap(); - assert!(result.is_err()); + assert!(!not_exists.exists.unwrap()); migrate.migrate().await.unwrap(); - let result = sqlx::query("select * from object") - .fetch_one(migrate.client.pool()) - .await - .unwrap(); + let exists = sqlx::query!( + "select exists (select from information_schema.tables where table_name = 'object')" + ) + .fetch_one(migrate.client.pool()) + .await + .unwrap(); - assert!(result.is_empty()); + assert!(exists.exists.unwrap()); } } diff --git a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs index c1a4c2824..271bedf47 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs @@ -90,13 +90,14 @@ pub async fn ingest_event( mod tests { use super::*; use crate::database::aws::ingester::tests::assert_deleted; + use crate::database::aws::migration::tests::MIGRATOR; use crate::events::aws::collecter::tests::set_s3_client_expectations; use crate::events::aws::collector_builder::tests::set_sqs_client_expectations; use crate::events::aws::tests::expected_event_record; use aws_lambda_events::sqs::SqsMessage; use sqlx::PgPool; - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn test_receive_and_ingest(pool: PgPool) { let mut sqs_client = SQSClient::default(); let mut s3_client = S3Client::default(); @@ -117,7 +118,7 @@ mod tests { assert_deleted(result); } - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn test_ingest_event(pool: PgPool) { let mut s3_client = S3Client::default(); From c235c905a1dfee3d0b9d5d43e4f1eeb3566f1060 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 18 Dec 2023 17:42:31 +1100 Subject: [PATCH 12/24] refactor(filemanager): split lambda function code --- .../deploy/constructs/functions/function.ts | 130 ++++++++++++++++++ .../deploy/constructs/functions/ingest.ts | 100 +++----------- 2 files changed, 149 insertions(+), 81 deletions(-) create mode 100644 lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts new file mode 100644 index 000000000..55f3e2e97 --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts @@ -0,0 +1,130 @@ +import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { Database } from '../database'; +import { IQueue } from 'aws-cdk-lib/aws-sqs'; +import { Architecture, IDestination } from 'aws-cdk-lib/aws-lambda'; +import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { Construct } from 'constructs'; +import { RustFunction } from 'rust.aws-cdk-lambda'; +import { Duration } from 'aws-cdk-lib'; +import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; + +/** + * Settable values for a Rust function. + */ +export type FunctionSettings = { + /** + * Additional build environment variables when building the Lambda function. + */ + readonly buildEnvironment?: { [key: string]: string | undefined }; + /** + * RUST_LOG string, defaults to trace on local crates and info everywhere else. + */ + readonly rustLog?: string; +}; + +/** + * Props for the Rust function. + */ +export type FunctionProps = FunctionSettings & { + /** + * The package to build for this function. + */ + readonly package: string; + /** + * Vpc for the function. + */ + readonly vpc: IVpc; + /** + * Database that the function uses. + */ + readonly database: Database; + /** + * The destination to post failed invocations to. + */ + readonly onFailure?: IDestination; + /** + * Additional policies to add to the Lambda role. + */ + readonly policies?: PolicyStatement[]; +}; + +/** + * A construct for a Rust Lambda function. + */ +export class Function extends Construct { + private readonly _function: RustFunction; + private readonly _role: Role; + + constructor(scope: Construct, id: string, props: FunctionProps) { + super(scope, id); + + // Lambda role needs SQS execution role. + this._role = new Role(this, 'Role', { + assumedBy: new ServicePrincipal('lambda.amazonaws.com'), + description: 'Lambda execution role for ' + id, + }); + // Lambda needs VPC access if it is created in a VPC. + this.addManagedPolicy('service-role/AWSLambdaVPCAccessExecutionRole'); + props.policies?.forEach((policy) => { + this._role.addToPolicy(policy); + }); + + // Lambda needs to be able to reach out to access S3, security manager (eventually), etc. + // Could this use an endpoint instead? + const securityGroup = new SecurityGroup(this, 'SecurityGroup', { + vpc: props.vpc, + allowAllOutbound: true, + description: 'Security group that allows the ingest Lambda function to egress out.', + }); + + CargoSettings.WORKSPACE_DIR = '../'; + CargoSettings.BUILD_INDIVIDUALLY = true; + + this._function = new RustFunction(this, id, { + package: props.package, + target: 'aarch64-unknown-linux-gnu', + memorySize: 128, + timeout: Duration.seconds(28), + environment: { + // Todo use security manager to get connection string rather than passing it in as an environment variable + DATABASE_URL: props.database.unsafeConnection, + RUST_LOG: + props.rustLog ?? `info,${props.package.replace('-', '_')}=trace,filemanager=trace`, + }, + buildEnvironment: props.buildEnvironment, + architecture: Architecture.ARM_64, + role: this._role, + onFailure: props.onFailure, + vpc: props.vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, + securityGroups: [ + securityGroup, + // Allow access to database. + props.database.securityGroup, + ], + }); + + // Todo this should probably connect to an RDS proxy rather than directly to the database. + } + + /** + * Add a managed policy to the function's role. + */ + addManagedPolicy(policyName: string) { + this._role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName(policyName)); + } + + /** + * Get the function IAM role. + */ + get role(): Role { + return this._role; + } + + /** + * Get the Lambda function. + */ + get function(): RustFunction { + return this._function; + } +} diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts index 6611259b0..c244c2868 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts @@ -1,14 +1,8 @@ import { Construct } from 'constructs'; -import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { RustFunction } from 'rust.aws-cdk-lambda'; -import { Duration } from 'aws-cdk-lib'; -import { Architecture, IDestination } from 'aws-cdk-lib/aws-lambda'; -import { Database } from '../database'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; -import * as iam from 'aws-cdk-lib/aws-iam'; -import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; -import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; +import * as fn from './function'; /** * Settable values for the ingest function. @@ -27,99 +21,43 @@ export type IngestFunctionSettings = { /** * Props for the database */ -export type IngestFunctionProps = IngestFunctionSettings & { - /** - * Vpc for the function. - */ - readonly vpc: IVpc; - /** - * Database that the function uses. - */ - readonly database: Database; - /** - * The SQS queue URL to receive events from. - */ - readonly queue: IQueue; - /** - * The destination to post failed invocations to. - */ - readonly onFailure?: IDestination; - /** - * Additional policies to add to the Lambda role. - */ - readonly policies?: PolicyStatement[]; -}; +export type IngestFunctionProps = fn.FunctionSettings & + fn.FunctionProps & { + /** + * The SQS queue URL to receive events from. + */ + readonly queue: IQueue; + }; /** * A construct for the Lambda ingest function. */ export class IngestFunction extends Construct { - private readonly _function: RustFunction; + private readonly _function: fn.Function; constructor(scope: Construct, id: string, props: IngestFunctionProps) { super(scope, id); - // Lambda role needs SQS execution role. - const lambdaRole = new iam.Role(this, id + 'Role', { - assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), - description: 'Lambda execution role for ' + id, - }); - // The Lambda needs SQS read to get events from SQS. - lambdaRole.addManagedPolicy( - iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaSQSQueueExecutionRole') - ); - // Lambda needs VPC access if it is created in a VPC. - lambdaRole.addManagedPolicy( - iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole') - ); - props.policies?.forEach((policy) => { - lambdaRole.addToPolicy(policy); - }); - - // Lambda needs to be able to reach out to access S3, security manager (eventually), etc. - // Could this use an endpoint instead? - const securityGroup = new SecurityGroup(this, 'SecurityGroup', { - vpc: props.vpc, - allowAllOutbound: true, - description: 'Security group that allows the ingest Lambda function to egress out.', - }); - - CargoSettings.WORKSPACE_DIR = '../'; - CargoSettings.BUILD_INDIVIDUALLY = true; - - const filemanagerLambda = new RustFunction(this, 'IngestLambdaFunction', { + this._function = new fn.Function(this, 'IngestLambdaFunction', { package: 'filemanager-ingest-lambda', - target: 'aarch64-unknown-linux-gnu', - memorySize: 128, - timeout: Duration.seconds(28), - environment: { - // Todo use security manager to get connection string rather than passing it in as an environment variable - DATABASE_URL: props.database.unsafeConnection, - RUST_LOG: props.rustLog ?? 'info,filemanager_ingest_lambda=trace,filemanager=trace', - }, buildEnvironment: props.buildEnvironment, - architecture: Architecture.ARM_64, - role: lambdaRole, - onFailure: props.onFailure, + rustLog: props.rustLog, vpc: props.vpc, - vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, - securityGroups: [ - securityGroup, - // Allow access to database. - props.database.securityGroup, - ], + database: props.database, + onFailure: props.onFailure, + policies: props.policies, }); - const eventSource = new lambdaEventSources.SqsEventSource(props.queue); - filemanagerLambda.addEventSource(eventSource); + this._function.addManagedPolicy('service-role/AWSLambdaSQSQueueExecutionRole'); - // Todo this should probably connect to an RDS proxy rather than directly to the database. + const eventSource = new lambdaEventSources.SqsEventSource(props.queue); + this.function.addEventSource(eventSource); } /** - * Get the Lambda function. + * Get the underlying Lambda function. */ get function(): RustFunction { - return this._function; + return this._function.function; } } From 5c6413dfcc8f6de1c577b38d367b2ffd6370db3f Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 19 Dec 2023 08:21:52 +1100 Subject: [PATCH 13/24] feat(filemanager): implement migrate lambda function cdk --- .../deploy/constructs/functions/function.ts | 21 +++++---- .../deploy/constructs/functions/ingest.ts | 43 +++---------------- .../deploy/constructs/functions/migrate.ts | 28 ++++++++++++ 3 files changed, 48 insertions(+), 44 deletions(-) create mode 100644 lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts index 55f3e2e97..50a7bf80b 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts @@ -1,6 +1,5 @@ import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { Database } from '../database'; -import { IQueue } from 'aws-cdk-lib/aws-sqs'; import { Architecture, IDestination } from 'aws-cdk-lib/aws-lambda'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { Construct } from 'constructs'; @@ -23,13 +22,9 @@ export type FunctionSettings = { }; /** - * Props for the Rust function. + * Props for a Rust function without the package. */ -export type FunctionProps = FunctionSettings & { - /** - * The package to build for this function. - */ - readonly package: string; +export type FunctionPropsNoPackage = FunctionSettings & { /** * Vpc for the function. */ @@ -48,6 +43,16 @@ export type FunctionProps = FunctionSettings & { readonly policies?: PolicyStatement[]; }; +/** + * Props for the Rust function. + */ +export type FunctionProps = FunctionPropsNoPackage & { + /** + * The package to build for this function. + */ + readonly package: string; +}; + /** * A construct for a Rust Lambda function. */ @@ -74,7 +79,7 @@ export class Function extends Construct { const securityGroup = new SecurityGroup(this, 'SecurityGroup', { vpc: props.vpc, allowAllOutbound: true, - description: 'Security group that allows the ingest Lambda function to egress out.', + description: 'Security group that allows a filemanager Lambda function to egress out.', }); CargoSettings.WORKSPACE_DIR = '../'; diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts index c244c2868..4f4426495 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts @@ -1,5 +1,4 @@ import { Construct } from 'constructs'; -import { RustFunction } from 'rust.aws-cdk-lambda'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; import * as fn from './function'; @@ -7,22 +6,13 @@ import * as fn from './function'; /** * Settable values for the ingest function. */ -export type IngestFunctionSettings = { - /** - * Additional build environment variables when building the Lambda function. - */ - readonly buildEnvironment?: { [key: string]: string | undefined }; - /** - * RUST_LOG string, defaults to trace on local crates and info everywhere else. - */ - readonly rustLog?: string; -}; +export type IngestFunctionSettings = fn.FunctionSettings; /** - * Props for the database + * Props for the ingest function. */ -export type IngestFunctionProps = fn.FunctionSettings & - fn.FunctionProps & { +export type IngestFunctionProps = IngestFunctionSettings & + fn.FunctionPropsNoPackage & { /** * The SQS queue URL to receive events from. */ @@ -32,32 +22,13 @@ export type IngestFunctionProps = fn.FunctionSettings & /** * A construct for the Lambda ingest function. */ -export class IngestFunction extends Construct { - private readonly _function: fn.Function; - +export class IngestFunction extends fn.Function { constructor(scope: Construct, id: string, props: IngestFunctionProps) { - super(scope, id); - - this._function = new fn.Function(this, 'IngestLambdaFunction', { - package: 'filemanager-ingest-lambda', - buildEnvironment: props.buildEnvironment, - rustLog: props.rustLog, - vpc: props.vpc, - database: props.database, - onFailure: props.onFailure, - policies: props.policies, - }); + super(scope, id, { package: 'filemanager-ingest-lambda', ...props }); - this._function.addManagedPolicy('service-role/AWSLambdaSQSQueueExecutionRole'); + this.addManagedPolicy('service-role/AWSLambdaSQSQueueExecutionRole'); const eventSource = new lambdaEventSources.SqsEventSource(props.queue); this.function.addEventSource(eventSource); } - - /** - * Get the underlying Lambda function. - */ - get function(): RustFunction { - return this._function.function; - } } diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts new file mode 100644 index 000000000..18bd59ed9 --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts @@ -0,0 +1,28 @@ +import { Construct } from 'constructs'; +import { IQueue } from 'aws-cdk-lib/aws-sqs'; +import * as fn from './function'; + +/** + * Settable values for the migrate function. + */ +export type IngestFunctionSettings = fn.FunctionSettings; + +/** + * Props for the migrate function. + */ +export type IngestFunctionProps = IngestFunctionSettings & + fn.FunctionPropsNoPackage & { + /** + * The SQS queue URL to receive events from. + */ + readonly queue: IQueue; + }; + +/** + * A construct for the Lambda migrate function. + */ +export class MigrateFunction extends fn.Function { + constructor(scope: Construct, id: string, props: IngestFunctionProps) { + super(scope, id, { package: 'filemanager-migrate-lambda', ...props }); + } +} From 32889e5cdfcf10abf7996850fc4274e613fe8325 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 19 Dec 2023 09:43:19 +1100 Subject: [PATCH 14/24] feat(filemanager): create custom resource construct --- .../deploy/constructs/cdk_resource_invoke.ts | 109 ++++++++++++++++++ .../deploy/constructs/functions/function.ts | 21 +++- .../deploy/constructs/functions/migrate.ts | 13 +-- 3 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts diff --git a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts new file mode 100644 index 000000000..26d44f66a --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts @@ -0,0 +1,109 @@ +import { Construct } from 'constructs'; +import { + AwsCustomResource, + AwsCustomResourcePolicy, + AwsSdkCall, + PhysicalResourceId, +} from 'aws-cdk-lib/custom-resources'; +import { IVpc } from 'aws-cdk-lib/aws-ec2'; +import * as fn from './functions/function'; +import { PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { Stack } from 'aws-cdk-lib'; +import { FunctionProps } from './functions/function'; + +/** + * Props for the resource invoke construct. + */ +export type CdkResourceInvokeProps = { + /** + * Vpc for the function. + */ + vpc: IVpc; + /** + * The function to create. This will override the function name to ensure that it remains + * callable using the singleton function created by `AwsCustomResource`. See + * https://github.com/aws-samples/amazon-rds-init-cdk/blob/239626632f399ebe4928410a49d5ac5d009a6502/lib/resource-initializer.ts#L69-L71. + */ + createFunction: (scope: Construct, id: string, props: FunctionProps) => fn.Function; + /** + * Function props when creating the Lambda function. + */ + functionProps: FunctionProps; + /** + * Name to use when creating the function. + */ + id: string; +}; + +/** + * A construct for invoking a Lambda function for resource initialization. + */ +export class CdkResourceInvoke extends Construct { + private readonly _response: string; + private readonly _customResource: AwsCustomResource; + private readonly _function: fn.Function; + + constructor(scope: Construct, id: string, props: CdkResourceInvokeProps) { + super(scope, id); + + const stack = Stack.of(this); + this._function = props.createFunction(this, props.id, { + ...props.functionProps, + functionName: `${id}-${props.id}-ResourceInvokeFunction-${stack.stackName}`, + }); + + const sdkCall: AwsSdkCall = { + service: 'Lambda', + action: 'invoke', + parameters: { + FunctionName: this.function.functionName(), + }, + physicalResourceId: PhysicalResourceId.of( + `${id}-AwsSdkCall-${this.function.currentVersion()}` + ), + }; + + const role = new Role(this, 'AwsCustomResourceRole', { + assumedBy: new ServicePrincipal('lambda.amazonaws.com'), + }); + role.addToPolicy( + new PolicyStatement({ + resources: [ + `arn:aws:lambda:${stack.region}:${stack.account}:function:*-ResourceInvokeFunction-${stack.stackName}`, + ], + actions: ['lambda:InvokeFunction'], + }) + ); + + this._customResource = new AwsCustomResource(this, 'AwsCustomResource', { + policy: AwsCustomResourcePolicy.fromSdkCalls({ + resources: AwsCustomResourcePolicy.ANY_RESOURCE, + }), + onUpdate: sdkCall, + role: role, + }); + + this._response = this.customResource.getResponseField('Payload'); + } + + /** + * Get the function response. + */ + get response(): string { + return this._response; + } + + /** + * Get the custom resource. + */ + get customResource(): AwsCustomResource { + return this._customResource; + } + + /** + * Get the function. + */ + get function(): fn.Function { + return this._function; + } +} diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts index 50a7bf80b..f3257a439 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts @@ -1,6 +1,6 @@ import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { Database } from '../database'; -import { Architecture, IDestination } from 'aws-cdk-lib/aws-lambda'; +import { Architecture, IDestination, Version } from 'aws-cdk-lib/aws-lambda'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { Construct } from 'constructs'; import { RustFunction } from 'rust.aws-cdk-lambda'; @@ -41,6 +41,10 @@ export type FunctionPropsNoPackage = FunctionSettings & { * Additional policies to add to the Lambda role. */ readonly policies?: PolicyStatement[]; + /** + * Name of the Lambda function resource. + */ + readonly functionName?: string; }; /** @@ -107,6 +111,7 @@ export class Function extends Construct { // Allow access to database. props.database.securityGroup, ], + functionName: props.functionName, }); // Todo this should probably connect to an RDS proxy rather than directly to the database. @@ -119,6 +124,20 @@ export class Function extends Construct { this._role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName(policyName)); } + /** + * Get the function name. + */ + functionName(): string { + return this.function.functionName; + } + + /** + * Get the function version. + */ + currentVersion(): Version { + return this.function.currentVersion; + } + /** * Get the function IAM role. */ diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts index 18bd59ed9..45982ca2a 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts @@ -1,28 +1,21 @@ import { Construct } from 'constructs'; -import { IQueue } from 'aws-cdk-lib/aws-sqs'; import * as fn from './function'; /** * Settable values for the migrate function. */ -export type IngestFunctionSettings = fn.FunctionSettings; +export type MigrateFunctionSettings = fn.FunctionSettings; /** * Props for the migrate function. */ -export type IngestFunctionProps = IngestFunctionSettings & - fn.FunctionPropsNoPackage & { - /** - * The SQS queue URL to receive events from. - */ - readonly queue: IQueue; - }; +export type MigrateFunctionProps = MigrateFunctionSettings & fn.FunctionPropsNoPackage; /** * A construct for the Lambda migrate function. */ export class MigrateFunction extends fn.Function { - constructor(scope: Construct, id: string, props: IngestFunctionProps) { + constructor(scope: Construct, id: string, props: MigrateFunctionProps) { super(scope, id, { package: 'filemanager-migrate-lambda', ...props }); } } From 884be2fe8b8b364cb9940f81f239c6665c05a8e8 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 19 Dec 2023 11:33:00 +1100 Subject: [PATCH 15/24] feat(filemanager): use cdk resource in stack --- .../deploy/constructs/cdk_resource_invoke.ts | 19 ++++++++++++++----- .../deploy/lib/filemanager_stack.ts | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts index 26d44f66a..ad4395518 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts @@ -1,15 +1,14 @@ -import { Construct } from 'constructs'; +import { Construct, IDependable } from 'constructs'; import { AwsCustomResource, AwsCustomResourcePolicy, AwsSdkCall, PhysicalResourceId, } from 'aws-cdk-lib/custom-resources'; -import { IVpc } from 'aws-cdk-lib/aws-ec2'; +import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; import * as fn from './functions/function'; import { PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { Stack } from 'aws-cdk-lib'; -import { FunctionProps } from './functions/function'; /** * Props for the resource invoke construct. @@ -24,11 +23,11 @@ export type CdkResourceInvokeProps = { * callable using the singleton function created by `AwsCustomResource`. See * https://github.com/aws-samples/amazon-rds-init-cdk/blob/239626632f399ebe4928410a49d5ac5d009a6502/lib/resource-initializer.ts#L69-L71. */ - createFunction: (scope: Construct, id: string, props: FunctionProps) => fn.Function; + createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => fn.Function; /** * Function props when creating the Lambda function. */ - functionProps: FunctionProps; + functionProps: fn.FunctionPropsNoPackage; /** * Name to use when creating the function. */ @@ -69,6 +68,7 @@ export class CdkResourceInvoke extends Construct { role.addToPolicy( new PolicyStatement({ resources: [ + // This needs to have permissions to run any `ResourceInvokeFunction`. `arn:aws:lambda:${stack.region}:${stack.account}:function:*-ResourceInvokeFunction-${stack.stackName}`, ], actions: ['lambda:InvokeFunction'], @@ -81,11 +81,20 @@ export class CdkResourceInvoke extends Construct { }), onUpdate: sdkCall, role: role, + vpc: props.vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, }); this._response = this.customResource.getResponseField('Payload'); } + /** + * Add a dependency to this resource. + */ + addDependency(dependency: IDependable) { + this.customResource.node.addDependency(dependency); + } + /** * Get the function response. */ diff --git a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts index 097406480..05056105d 100644 --- a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts +++ b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts @@ -11,6 +11,9 @@ import { Bucket, EventType } from 'aws-cdk-lib/aws-s3'; import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; import { Queue } from 'aws-cdk-lib/aws-sqs'; import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; +import { CdkResourceInvoke } from '../constructs/cdk_resource_invoke'; +import { MigrateFunction } from '../constructs/functions/migrate'; +import * as fn from '../constructs/functions/function'; /** * Common settings for the filemanager stack. @@ -85,6 +88,19 @@ export class FilemanagerStack extends Stack { port: settings?.port, }); + const migrate = new CdkResourceInvoke(this, 'MigrateDatabase', { + vpc, + createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => { + return new MigrateFunction(scope, id, props); + }, + functionProps: { + vpc, + database, + }, + id: 'MigrateFunction', + }); + migrate.addDependency(database.cluster); + new IngestFunction(this, 'IngestLambda', { vpc, database, From a2d661ebee925e3182dd50ae55594bc88b9a9268 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 19 Dec 2023 13:09:16 +1100 Subject: [PATCH 16/24] feat(filemanager): make migration optional in stack --- .../stateful/filemanager/deploy/README.md | 12 +++++++ .../filemanager/deploy/bin/filemanager.ts | 2 ++ .../deploy/constructs/cdk_resource_invoke.ts | 20 +++++++++-- .../filemanager/deploy/constructs/database.ts | 6 +++- .../deploy/constructs/functions/function.ts | 3 +- .../deploy/lib/filemanager_stack.ts | 35 ++++++++++++------- .../filemanager/deploy/package-lock.json | 23 +++++++++--- .../stateful/filemanager/deploy/package.json | 4 ++- .../filemanager-migrate-lambda/Cargo.toml | 2 -- 9 files changed, 83 insertions(+), 24 deletions(-) diff --git a/lib/workload/stateful/filemanager/deploy/README.md b/lib/workload/stateful/filemanager/deploy/README.md index d7fd11fde..6c30a6215 100644 --- a/lib/workload/stateful/filemanager/deploy/README.md +++ b/lib/workload/stateful/filemanager/deploy/README.md @@ -7,3 +7,15 @@ npm install cdk bootstrap cdk deploy ``` + +By default, the stack does not perform database migration. To migrate the database, use the script inside `package.json`: + +```sh +npm run migrate -- cdk deploy +``` + +or set `FILEMANAGER_DEPLOY_DATABASE_MIGRATION`: + +```sh +export FILEMANAGER_DEPLOY_DATABASE_MIGRATION="true" +``` diff --git a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts index c10acc29c..3cf86c656 100644 --- a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts +++ b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts @@ -27,6 +27,8 @@ new FilemanagerStack( enableMonitoring: { enablePerformanceInsights: true, }, + public: true, + migrateDatabase: process.env.FILEMANAGER_DEPLOY_MIGRATE_DATABASE == 'true', } ); diff --git a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts index ad4395518..50d0751dc 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts @@ -8,7 +8,7 @@ import { import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; import * as fn from './functions/function'; import { PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; -import { Stack } from 'aws-cdk-lib'; +import { CfnOutput, Stack, Token } from 'aws-cdk-lib'; /** * Props for the resource invoke construct. @@ -32,6 +32,10 @@ export type CdkResourceInvokeProps = { * Name to use when creating the function. */ id: string; + /** + * Dependencies for this resource. + */ + dependencies?: IDependable[]; }; /** @@ -48,9 +52,10 @@ export class CdkResourceInvoke extends Construct { const stack = Stack.of(this); this._function = props.createFunction(this, props.id, { ...props.functionProps, - functionName: `${id}-${props.id}-ResourceInvokeFunction-${stack.stackName}`, + functionName: `${id}-ResourceInvokeFunction-${stack.stackName}`, }); + // Call another lambda function with no arguments. const sdkCall: AwsSdkCall = { service: 'Lambda', action: 'invoke', @@ -68,7 +73,8 @@ export class CdkResourceInvoke extends Construct { role.addToPolicy( new PolicyStatement({ resources: [ - // This needs to have permissions to run any `ResourceInvokeFunction`. + // This needs to have permissions to run any `ResourceInvokeFunction` because it is deployed as a + // singleton Lambda function. `arn:aws:lambda:${stack.region}:${stack.account}:function:*-ResourceInvokeFunction-${stack.stackName}`, ], actions: ['lambda:InvokeFunction'], @@ -86,6 +92,14 @@ export class CdkResourceInvoke extends Construct { }); this._response = this.customResource.getResponseField('Payload'); + + // Add any dependencies. + props.dependencies?.forEach((dependency) => this.addDependency(dependency)); + + // Output the result. + new CfnOutput(this, 'MigrateDatabaseResponse', { + value: Token.asString(this.response), + }); } /** diff --git a/lib/workload/stateful/filemanager/deploy/constructs/database.ts b/lib/workload/stateful/filemanager/deploy/constructs/database.ts index f60ea6445..af02444ed 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/database.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/database.ts @@ -32,6 +32,10 @@ export type EnableMonitoringProps = { * Settable database props that can be configured. */ export type DatabaseSettings = { + /** + * Whether the database is publically available. + */ + readonly public?: boolean; /** * Whether to destroy the database on stack removal. Defaults to keeping a snapshot. */ @@ -114,7 +118,7 @@ export class Database extends Construct { this._cluster = new DatabaseCluster(this, 'Cluster', { vpc: props.vpc, vpcSubnets: { - subnetType: SubnetType.PRIVATE_ISOLATED, + subnetType: props.public ? SubnetType.PUBLIC : SubnetType.PRIVATE_ISOLATED, }, securityGroups: [this._securityGroup], credentials: rds.Credentials.fromSecret(props.secret), diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts index f3257a439..901e889fd 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts @@ -6,6 +6,7 @@ import { Construct } from 'constructs'; import { RustFunction } from 'rust.aws-cdk-lambda'; import { Duration } from 'aws-cdk-lib'; import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; +import * as path from 'node:path'; /** * Settable values for a Rust function. @@ -86,7 +87,6 @@ export class Function extends Construct { description: 'Security group that allows a filemanager Lambda function to egress out.', }); - CargoSettings.WORKSPACE_DIR = '../'; CargoSettings.BUILD_INDIVIDUALLY = true; this._function = new RustFunction(this, id, { @@ -101,6 +101,7 @@ export class Function extends Construct { props.rustLog ?? `info,${props.package.replace('-', '_')}=trace,filemanager=trace`, }, buildEnvironment: props.buildEnvironment, + extraBuildArgs: ['--manifest-path', `../${props.package}/Cargo.toml`], architecture: Architecture.ARM_64, role: this._role, onFailure: props.onFailure, diff --git a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts index 05056105d..3503af89b 100644 --- a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts +++ b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts @@ -1,4 +1,4 @@ -import { RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib'; +import { CfnOutput, RemovalPolicy, Stack, StackProps, Token } from 'aws-cdk-lib'; import { Construct } from 'constructs'; import * as s3 from 'aws-cdk-lib/aws-s3'; import * as lambdaDestinations from 'aws-cdk-lib/aws-lambda-destinations'; @@ -20,7 +20,14 @@ import * as fn from '../constructs/functions/function'; */ type Settings = DatabaseSettings & IngestFunctionSettings & { + /** + * The name of the database. Defaults to `filemanager`. + */ databaseName?: string; + /** + * Whether to initialize a database migration. + */ + migrateDatabase?: boolean; }; /** @@ -88,18 +95,22 @@ export class FilemanagerStack extends Stack { port: settings?.port, }); - const migrate = new CdkResourceInvoke(this, 'MigrateDatabase', { - vpc, - createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => { - return new MigrateFunction(scope, id, props); - }, - functionProps: { + if (settings?.migrateDatabase) { + new CdkResourceInvoke(this, 'MigrateDatabase', { vpc, - database, - }, - id: 'MigrateFunction', - }); - migrate.addDependency(database.cluster); + createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => { + return new MigrateFunction(scope, id, props); + }, + functionProps: { + vpc, + database, + buildEnvironment: settings?.buildEnvironment, + rustLog: settings?.rustLog, + }, + id: 'MigrateFunction', + dependencies: [database.cluster], + }); + } new IngestFunction(this, 'IngestLambda', { vpc, diff --git a/lib/workload/stateful/filemanager/deploy/package-lock.json b/lib/workload/stateful/filemanager/deploy/package-lock.json index b5ed057e2..9f8b1741f 100644 --- a/lib/workload/stateful/filemanager/deploy/package-lock.json +++ b/lib/workload/stateful/filemanager/deploy/package-lock.json @@ -24,6 +24,7 @@ "@types/node": "^20.5.9", "@typescript-eslint/eslint-plugin": "^6.14.0", "aws-cdk": "^2.114.1", + "cross-env": "^7.0.3", "prettier": "3.0.3", "typescript": "^5.3.3" } @@ -1041,12 +1042,29 @@ "node": ">= 16.14.0" } }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, - "peer": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -2016,7 +2034,6 @@ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, - "peer": true, "engines": { "node": ">=8" } @@ -2220,7 +2237,6 @@ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, - "peer": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -2233,7 +2249,6 @@ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true, - "peer": true, "engines": { "node": ">=8" } diff --git a/lib/workload/stateful/filemanager/deploy/package.json b/lib/workload/stateful/filemanager/deploy/package.json index 8b5ecf21d..f92fb23fb 100644 --- a/lib/workload/stateful/filemanager/deploy/package.json +++ b/lib/workload/stateful/filemanager/deploy/package.json @@ -7,12 +7,14 @@ "scripts": { "build": "tsc", "watch": "tsc -w", - "cdk": "cdk" + "cdk": "cdk", + "migrate": "cross-env FILEMANAGER_DEPLOY_MIGRATE_DATABASE=true" }, "devDependencies": { "@types/node": "^20.5.9", "@typescript-eslint/eslint-plugin": "^6.14.0", "aws-cdk": "^2.114.1", + "cross-env": "^7.0.3", "prettier": "3.0.3", "typescript": "^5.3.3" }, diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml index 29ce7ad47..13ee67f2b 100644 --- a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml +++ b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml @@ -5,8 +5,6 @@ license.workspace = true edition.workspace = true authors.workspace = true -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] aws_lambda_events = "0.12" lambda_runtime = "0.8" From 6749b794b662bf408b6714d54afbffd72b105dd9 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 19 Dec 2023 15:00:40 +1100 Subject: [PATCH 17/24] refactor(filemanager): allow public database --- .../deploy/constructs/cdk_resource_invoke.ts | 10 +++++++--- .../filemanager/deploy/constructs/database.ts | 18 +++++++++++++----- .../deploy/constructs/functions/function.ts | 3 +-- .../deploy/lib/filemanager_stack.ts | 9 +++++---- .../filemanager-migrate-lambda/src/main.rs | 9 ++++++--- .../filemanager/src/database/aws/migration.rs | 2 ++ 6 files changed, 34 insertions(+), 17 deletions(-) diff --git a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts index 50d0751dc..78f5fa9ea 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts @@ -7,7 +7,7 @@ import { } from 'aws-cdk-lib/custom-resources'; import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; import * as fn from './functions/function'; -import { PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { CfnOutput, Stack, Token } from 'aws-cdk-lib'; /** @@ -52,7 +52,7 @@ export class CdkResourceInvoke extends Construct { const stack = Stack.of(this); this._function = props.createFunction(this, props.id, { ...props.functionProps, - functionName: `${id}-ResourceInvokeFunction-${stack.stackName}`, + functionName: `${stack.stackName}-ResourceInvokeFunction-${props.id}`, }); // Call another lambda function with no arguments. @@ -75,11 +75,15 @@ export class CdkResourceInvoke extends Construct { resources: [ // This needs to have permissions to run any `ResourceInvokeFunction` because it is deployed as a // singleton Lambda function. - `arn:aws:lambda:${stack.region}:${stack.account}:function:*-ResourceInvokeFunction-${stack.stackName}`, + `arn:aws:lambda:${stack.region}:${stack.account}:function:${stack.stackName}-ResourceInvokeFunction-*`, ], actions: ['lambda:InvokeFunction'], }) ); + // Also require VPC access for a Lambda function within the VPC. + role.addManagedPolicy( + ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole') + ); this._customResource = new AwsCustomResource(this, 'AwsCustomResource', { policy: AwsCustomResourcePolicy.fromSdkCalls({ diff --git a/lib/workload/stateful/filemanager/deploy/constructs/database.ts b/lib/workload/stateful/filemanager/deploy/constructs/database.ts index af02444ed..c84593eb8 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/database.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/database.ts @@ -135,11 +135,19 @@ export class Database extends Construct { }), }); - // Any inbound connections within the same security group are allowed access to the database port. - this._securityGroup.addIngressRule( - this._securityGroup, - ec2.Port.tcp(this._cluster.clusterEndpoint.port) - ); + if (props.public) { + // If it's public, anyone can connect. + this._securityGroup.addIngressRule( + ec2.Peer.anyIpv4(), + ec2.Port.tcp(this._cluster.clusterEndpoint.port) + ); + } else { + // Any inbound connections within the same security group are allowed access to the database port. + this._securityGroup.addIngressRule( + this._securityGroup, + ec2.Port.tcp(this._cluster.clusterEndpoint.port) + ); + } this._unsafeConnection = `postgres://` + diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts index 901e889fd..06346026b 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts @@ -6,7 +6,6 @@ import { Construct } from 'constructs'; import { RustFunction } from 'rust.aws-cdk-lambda'; import { Duration } from 'aws-cdk-lib'; import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; -import * as path from 'node:path'; /** * Settable values for a Rust function. @@ -89,7 +88,7 @@ export class Function extends Construct { CargoSettings.BUILD_INDIVIDUALLY = true; - this._function = new RustFunction(this, id, { + this._function = new RustFunction(this, 'RustFunction', { package: props.package, target: 'aarch64-unknown-linux-gnu', memorySize: 128, diff --git a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts index 3503af89b..3f91c0816 100644 --- a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts +++ b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts @@ -1,4 +1,4 @@ -import { CfnOutput, RemovalPolicy, Stack, StackProps, Token } from 'aws-cdk-lib'; +import { RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib'; import { Construct } from 'constructs'; import * as s3 from 'aws-cdk-lib/aws-s3'; import * as lambdaDestinations from 'aws-cdk-lib/aws-lambda-destinations'; @@ -37,9 +37,9 @@ export class FilemanagerStack extends Stack { constructor(scope: Construct, id: string, props: StackProps, settings?: Settings) { super(scope, id, props); - const queue = new Queue(this, id + 'Queue'); + const queue = new Queue(this, 'Queue'); - const testBucket = new Bucket(this, id + 'Bucket', { + const testBucket = new Bucket(this, 'Bucket', { bucketName: 'filemanager-test-ingest', blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, encryption: s3.BucketEncryption.S3_MANAGED, @@ -53,7 +53,7 @@ export class FilemanagerStack extends Stack { testBucket.addEventNotification(EventType.OBJECT_CREATED, new SqsDestination(queue)); testBucket.addEventNotification(EventType.OBJECT_REMOVED, new SqsDestination(queue)); - const deadLetterQueue = new Queue(this, id + 'DeadLetterQueue'); + const deadLetterQueue = new Queue(this, 'DeadLetterQueue'); const deadLetterQueueDestination = new lambdaDestinations.SqsDestination(deadLetterQueue); const vpc = new ec2.Vpc(this, 'Vpc', { @@ -93,6 +93,7 @@ export class FilemanagerStack extends Stack { minCapacity: settings?.minCapacity, maxCapacity: settings?.maxCapacity, port: settings?.port, + public: settings?.public, }); if (settings?.migrateDatabase) { diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs index 66cf53f83..3cfb27936 100644 --- a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs +++ b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs @@ -1,6 +1,7 @@ use filemanager::database::aws::migration::Migration; use filemanager::database::Migrate; use lambda_runtime::{run, service_fn, Error, LambdaEvent}; +use std::collections::HashMap; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{fmt, EnvFilter}; @@ -14,8 +15,10 @@ async fn main() -> Result<(), Error> { .with(env_filter) .init(); - run(service_fn(|_: LambdaEvent<()>| async move { - Migration::with_defaults().await?.migrate().await - })) + run(service_fn( + |_: LambdaEvent>| async move { + Migration::with_defaults().await?.migrate().await + }, + )) .await } diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs index 71fbbb6c6..47ebb6df9 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs @@ -4,6 +4,7 @@ use crate::error::Result; use async_trait::async_trait; use sqlx::migrate; use sqlx::migrate::Migrator; +use tracing::trace; /// A struct to perform database migrations. #[derive(Debug)] @@ -46,6 +47,7 @@ impl Migration { #[async_trait] impl Migrate for Migration { async fn migrate(&self) -> Result<()> { + trace!("applying migrations"); Self::migrator() .run(self.client().pool()) .await From a5aae0e01d390d3c766d63afce74ad0a96b800c6 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 20 Dec 2023 08:03:59 +1100 Subject: [PATCH 18/24] fix(filemanager): make size and e_tag optional and remove size from delete events --- .../migrations/0001_add_object_table.sql | 2 +- .../filemanager/src/database/aws/ingester.rs | 4 +- .../filemanager/src/events/aws/collecter.rs | 12 +- .../filemanager/src/events/aws/mod.rs | 103 +++++++++++++----- 4 files changed, 90 insertions(+), 31 deletions(-) diff --git a/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql b/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql index 7973d2493..cfcbe680f 100644 --- a/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql +++ b/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql @@ -7,7 +7,7 @@ create table object ( -- The name of the object. key varchar(1024) not null, -- The size of the object. - size int not null, + size int default null, -- A unique identifier for the object, if it is present. hash varchar(255) default null, -- When this object was created. diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs index 3254f890d..c0264ad40 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs @@ -55,8 +55,8 @@ impl Ingester { &object_ids, &buckets, &keys, - &sizes, - &e_tags, + &sizes as &[Option], + &e_tags as &[Option], &event_times, &last_modified_dates as &[Option>], &portal_run_ids diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs index 3ad52587a..73d67ee47 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs @@ -74,15 +74,21 @@ impl Collecter { trace!(key = ?event.key, bucket = ?event.bucket, "updating event"); if let Some(head) = Self::head(client, &event.key, &event.bucket).await? { + trace!(head = ?head, "received head object output"); + let HeadObjectOutput { storage_class, last_modified, + content_length, + e_tag, .. } = head; - event = - event.with_storage_class(storage_class.and_then(StorageClass::from_aws)); - event = event.with_last_modified_date(Self::convert_datetime(last_modified)); + event = event + .update_storage_class(storage_class.and_then(StorageClass::from_aws)) + .update_last_modified_date(Self::convert_datetime(last_modified)) + .update_size(content_length) + .update_e_tag(e_tag); } Ok(event) diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs index c475eadf8..bb2756e54 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs @@ -67,8 +67,8 @@ pub struct TransposedS3EventMessages { pub event_names: Vec, pub buckets: Vec, pub keys: Vec, - pub sizes: Vec, - pub e_tags: Vec, + pub sizes: Vec>, + pub e_tags: Vec>, pub sequencers: Vec>, pub portal_run_ids: Vec, pub storage_classes: Vec>, @@ -289,8 +289,8 @@ pub struct FlatS3EventMessage { pub event_name: String, pub bucket: String, pub key: String, - pub size: i32, - pub e_tag: String, + pub size: Option, + pub e_tag: Option, pub sequencer: Option, pub portal_run_id: String, pub storage_class: Option, @@ -298,15 +298,31 @@ pub struct FlatS3EventMessage { } impl FlatS3EventMessage { - /// Update the storage class. - pub fn with_storage_class(mut self, storage_class: Option) -> Self { - self.storage_class = storage_class; + /// Update the storage class if not None. + pub fn update_storage_class(mut self, storage_class: Option) -> Self { + storage_class + .into_iter() + .for_each(|storage_class| self.storage_class = Some(storage_class)); self } - /// Update the last modified date. - pub fn with_last_modified_date(mut self, last_modified_date: Option>) -> Self { - self.last_modified_date = last_modified_date; + /// Update the last modified date if not None. + pub fn update_last_modified_date(mut self, last_modified_date: Option>) -> Self { + last_modified_date + .into_iter() + .for_each(|last_modified_date| self.last_modified_date = Some(last_modified_date)); + self + } + + /// Update the size if not None. + pub fn update_size(mut self, size: Option) -> Self { + size.into_iter().for_each(|size| self.size = Some(size)); + self + } + + /// Update the e_tag if not None. + pub fn update_e_tag(mut self, e_tag: Option) -> Self { + e_tag.into_iter().for_each(|e_tag| self.e_tag = Some(e_tag)); self } } @@ -344,8 +360,8 @@ pub struct BucketRecord { #[serde(rename_all = "camelCase")] pub struct ObjectRecord { pub key: String, - pub size: i32, - pub e_tag: String, + pub size: Option, + pub e_tag: Option, pub sequencer: Option, } @@ -425,13 +441,28 @@ pub(crate) mod tests { let mut result = result.into_inner().into_iter(); let first = result.next().unwrap(); - assert_flat_s3_event(first, "ObjectRemoved:Delete", EXPECTED_SEQUENCER_DELETED); + assert_flat_s3_event( + first, + "ObjectRemoved:Delete", + EXPECTED_SEQUENCER_DELETED, + None, + ); let second = result.next().unwrap(); - assert_flat_s3_event(second, "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED); + assert_flat_s3_event( + second, + "ObjectCreated:Put", + EXPECTED_SEQUENCER_CREATED, + Some(0), + ); let third = result.next().unwrap(); - assert_flat_s3_event(third, "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED); + assert_flat_s3_event( + third, + "ObjectCreated:Put", + EXPECTED_SEQUENCER_CREATED, + Some(0), + ); } #[test] @@ -440,19 +471,34 @@ pub(crate) mod tests { let mut result = result.into_inner().into_iter(); let first = result.next().unwrap(); - assert_flat_s3_event(first, "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED); + assert_flat_s3_event( + first, + "ObjectCreated:Put", + EXPECTED_SEQUENCER_CREATED, + Some(0), + ); let second = result.next().unwrap(); - assert_flat_s3_event(second, "ObjectRemoved:Delete", EXPECTED_SEQUENCER_DELETED); + assert_flat_s3_event( + second, + "ObjectRemoved:Delete", + EXPECTED_SEQUENCER_DELETED, + None, + ); } - fn assert_flat_s3_event(event: FlatS3EventMessage, event_name: &str, sequencer: &str) { + fn assert_flat_s3_event( + event: FlatS3EventMessage, + event_name: &str, + sequencer: &str, + size: Option, + ) { assert_eq!(event.event_time, DateTime::::default()); assert_eq!(event.event_name, event_name); assert_eq!(event.bucket, "bucket"); assert_eq!(event.key, "key"); - assert_eq!(event.size, 0); - assert_eq!(event.e_tag, EXPECTED_E_TAG); // pragma: allowlist secret + assert_eq!(event.size, size); + assert_eq!(event.e_tag, Some(EXPECTED_E_TAG.to_string())); // pragma: allowlist secret assert_eq!(event.sequencer, Some(sequencer.to_string())); assert!(event.portal_run_id.starts_with("19700101")); assert_eq!(event.storage_class, None); @@ -470,8 +516,11 @@ pub(crate) mod tests { assert_eq!(result.object_created.event_names[0], "ObjectCreated:Put"); assert_eq!(result.object_created.buckets[0], "bucket"); assert_eq!(result.object_created.keys[0], "key"); - assert_eq!(result.object_created.sizes[0], 0); - assert_eq!(result.object_created.e_tags[0], EXPECTED_E_TAG); + assert_eq!(result.object_created.sizes[0], Some(0)); + assert_eq!( + result.object_created.e_tags[0], + Some(EXPECTED_E_TAG.to_string()) + ); assert_eq!( result.object_created.sequencers[0], Some(EXPECTED_SEQUENCER_CREATED.to_string()) @@ -487,8 +536,11 @@ pub(crate) mod tests { assert_eq!(result.object_removed.event_names[0], "ObjectRemoved:Delete"); assert_eq!(result.object_removed.buckets[0], "bucket"); assert_eq!(result.object_removed.keys[0], "key"); - assert_eq!(result.object_removed.sizes[0], 0); - assert_eq!(result.object_removed.e_tags[0], EXPECTED_E_TAG); + assert_eq!(result.object_removed.sizes[0], None); + assert_eq!( + result.object_removed.e_tags[0], + Some(EXPECTED_E_TAG.to_string()) + ); assert_eq!( result.object_removed.sequencers[0], Some(EXPECTED_SEQUENCER_DELETED.to_string()) @@ -588,7 +640,8 @@ pub(crate) mod tests { }, "object": { "key": "key", - "size": 0, + // ObjectRemoved::Delete does not have a size, even though this isn't documented + // anywhere. "eTag": EXPECTED_E_TAG, "versionId": "096fKKXTRTtl3on89fVO.nfljtsv6qko", "sequencer": EXPECTED_SEQUENCER_DELETED From 1f29f7ed658f6e1d624f0942a6907dcb053093f7 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 20 Dec 2023 08:35:38 +1100 Subject: [PATCH 19/24] fix(filemanager): head object should not be called on removed events --- .../filemanager/src/database/aws/ingester.rs | 10 +-- .../filemanager/src/database/mod.rs | 4 +- .../filemanager/src/events/aws/collecter.rs | 82 +++++++++++-------- .../src/events/aws/collector_builder.rs | 4 +- .../filemanager/src/events/aws/mod.rs | 42 ++++++++-- .../filemanager/filemanager/src/events/mod.rs | 4 +- .../filemanager/src/handlers/aws.rs | 6 +- 7 files changed, 94 insertions(+), 58 deletions(-) diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs index c0264ad40..fadfaaf0c 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs @@ -7,7 +7,7 @@ use crate::database::{Client, Ingest}; use crate::error::Result; use crate::events::aws::StorageClass; use crate::events::aws::{Events, TransposedS3EventMessages}; -use crate::events::EventType; +use crate::events::EventSourceType; /// An ingester for S3 events. #[derive(Debug)] @@ -100,9 +100,9 @@ impl Ingester { #[async_trait] impl Ingest for Ingester { - async fn ingest(&self, events: EventType) -> Result<()> { + async fn ingest(&self, events: EventSourceType) -> Result<()> { match events { - EventType::S3(events) => self.ingest_events(events).await, + EventSourceType::S3(events) => self.ingest_events(events).await, } } } @@ -114,7 +114,7 @@ pub(crate) mod tests { use crate::database::{Client, Ingest}; use crate::events::aws::tests::{expected_events, EXPECTED_E_TAG}; use crate::events::aws::{Events, StorageClass}; - use crate::events::EventType; + use crate::events::EventSourceType; use chrono::{DateTime, Utc}; use sqlx::postgres::PgRow; use sqlx::{PgPool, Row}; @@ -155,7 +155,7 @@ pub(crate) mod tests { let events = test_events(); let ingester = test_ingester(pool); - ingester.ingest(EventType::S3(events)).await.unwrap(); + ingester.ingest(EventSourceType::S3(events)).await.unwrap(); let result = sqlx::query("select * from object") .fetch_one(ingester.client.pool()) diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs index 0a3413847..aacebdcde 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs @@ -6,7 +6,7 @@ use async_trait::async_trait; use sqlx::PgPool; use crate::error::Result; -use crate::events::EventType; +use crate::events::EventSourceType; pub mod aws; @@ -39,7 +39,7 @@ impl Client { #[async_trait] pub trait Ingest { /// Ingest the events. - async fn ingest(&self, events: EventType) -> Result<()>; + async fn ingest(&self, events: EventSourceType) -> Result<()>; } /// Trait representing database migrations. diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs index 73d67ee47..b213e0f98 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs @@ -10,8 +10,10 @@ use mockall_double::double; use tracing::trace; use crate::error::Result; -use crate::events::aws::{Events, FlatS3EventMessage, FlatS3EventMessages, StorageClass}; -use crate::events::{Collect, EventType}; +use crate::events::aws::{ + EventType, Events, FlatS3EventMessage, FlatS3EventMessages, StorageClass, +}; +use crate::events::{Collect, EventSourceType}; /// Collect raw events into the processed form which the database module accepts. #[derive(Debug)] @@ -71,6 +73,12 @@ impl Collecter { ) -> Result { Ok(FlatS3EventMessages( join_all(events.into_inner().into_iter().map(|mut event| async move { + // Only run this on created events. + match event.event_type { + EventType::Removed | EventType::Other => return Ok(event), + _ => {} + }; + trace!(key = ?event.key, bucket = ?event.bucket, "updating event"); if let Some(head) = Self::head(client, &event.key, &event.bucket).await? { @@ -102,13 +110,13 @@ impl Collecter { #[async_trait] impl Collect for Collecter { - async fn collect(self) -> Result { + async fn collect(self) -> Result { let (client, raw_events) = self.into_inner(); let raw_events = raw_events.sort_and_dedup(); let events = Self::update_events(&client, raw_events).await?; - Ok(EventType::S3(Events::from(events))) + Ok(EventSourceType::S3(Events::from(events))) } } @@ -125,6 +133,7 @@ pub(crate) mod tests { use aws_smithy_runtime_api::client::result::ServiceError; use chrono::{DateTime, Utc}; use mockall::predicate::eq; + use std::result; use super::*; @@ -142,7 +151,7 @@ pub(crate) mod tests { async fn head() { let mut collecter = test_collecter().await; - set_s3_client_expectations(&mut collecter.client, 1); + set_s3_client_expectations(&mut collecter.client, vec![|| Ok(expected_head_object())]); let result = Collecter::head(&collecter.client, "key", "bucket") .await @@ -154,19 +163,10 @@ pub(crate) mod tests { async fn head_not_found() { let mut collecter = test_collecter().await; - collecter - .client - .expect_head_object() - .with(eq("key"), eq("bucket")) - .times(1) - .returning(move |_, _| { - Err(SdkError::ServiceError( - ServiceError::builder() - .source(HeadObjectError::NotFound(NotFound::builder().build())) - .raw(HttpResponse::new(404.try_into().unwrap(), SdkBody::empty())) - .build(), - )) - }); + set_s3_client_expectations( + &mut collecter.client, + vec![|| Err(expected_head_object_not_found())], + ); let result = Collecter::head(&collecter.client, "key", "bucket") .await @@ -180,7 +180,7 @@ pub(crate) mod tests { let events = expected_flat_events().sort_and_dedup(); - set_s3_client_expectations(&mut collecter.client, 2); + set_s3_client_expectations(&mut collecter.client, vec![|| Ok(expected_head_object())]); let mut result = Collecter::update_events(&collecter.client, events) .await @@ -193,50 +193,53 @@ pub(crate) mod tests { assert_eq!(first.last_modified_date, Some(Default::default())); let second = result.next().unwrap(); - assert_eq!(second.storage_class, Some(Standard)); - assert_eq!(second.last_modified_date, Some(Default::default())); + assert_eq!(second.storage_class, None); + assert_eq!(second.last_modified_date, None); } #[tokio::test] async fn collect() { let mut collecter = test_collecter().await; - set_s3_client_expectations(&mut collecter.client, 2); + set_s3_client_expectations(&mut collecter.client, vec![|| Ok(expected_head_object())]); let result = collecter.collect().await.unwrap(); assert_collected_events(result); } - pub(crate) fn assert_collected_events(result: EventType) { - assert!(matches!(result, EventType::S3(_))); + pub(crate) fn assert_collected_events(result: EventSourceType) { + assert!(matches!(result, EventSourceType::S3(_))); match result { - EventType::S3(events) => { + EventSourceType::S3(events) => { assert_eq!(events.object_created.storage_classes[0], Some(Standard)); assert_eq!( events.object_created.last_modified_dates[0], Some(Default::default()) ); - assert_eq!(events.object_removed.storage_classes[0], Some(Standard)); - assert_eq!( - events.object_removed.last_modified_dates[0], - Some(Default::default()) - ); + assert_eq!(events.object_removed.storage_classes[0], None); + assert_eq!(events.object_removed.last_modified_dates[0], None); } } } - pub(crate) fn set_s3_client_expectations(client: &mut Client, times: usize) { - client + pub(crate) fn set_s3_client_expectations(client: &mut Client, expectations: Vec) + where + F: Fn() -> result::Result> + Send + 'static, + { + let client = client .expect_head_object() .with(eq("key"), eq("bucket")) - .times(times) - .returning(move |_, _| Ok(expected_head_object())); + .times(expectations.len()); + + for expectation in expectations { + client.returning(move |_, _| expectation()); + } } - fn expected_head_object() -> HeadObjectOutput { + pub(crate) fn expected_head_object() -> HeadObjectOutput { HeadObjectOutput::builder() .last_modified( primitives::DateTime::from_str("1970-01-01T00:00:00Z", DateTimeFormat::DateTime) @@ -246,6 +249,15 @@ pub(crate) mod tests { .build() } + pub(crate) fn expected_head_object_not_found() -> SdkError { + SdkError::ServiceError( + ServiceError::builder() + .source(HeadObjectError::NotFound(NotFound::builder().build())) + .raw(HttpResponse::new(404.try_into().unwrap(), SdkBody::empty())) + .build(), + ) + } + async fn test_collecter() -> Collecter { Collecter::new(Client::default(), expected_flat_events()) } diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs index 672e15ec2..20f88658d 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs @@ -100,7 +100,7 @@ impl CollecterBuilder { #[cfg(test)] pub(crate) mod tests { use crate::events::aws::collecter::tests::{ - assert_collected_events, set_s3_client_expectations, + assert_collected_events, expected_head_object, set_s3_client_expectations, }; use crate::events::aws::collector_builder::CollecterBuilder; use crate::events::aws::tests::{expected_event_record, expected_flat_events}; @@ -128,7 +128,7 @@ pub(crate) mod tests { let mut s3_client = S3Client::default(); set_sqs_client_expectations(&mut sqs_client); - set_s3_client_expectations(&mut s3_client, 2); + set_s3_client_expectations(&mut s3_client, vec![|| Ok(expected_head_object())]); let events = CollecterBuilder::default() .with_sqs_client(sqs_client) diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs index bb2756e54..7341236a6 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs @@ -12,6 +12,7 @@ use uuid::Uuid; use crate::error::Error; use crate::error::Error::DeserializeError; use crate::error::Result; +use crate::events::aws::EventType::{Created, Other, Removed}; pub mod collecter; pub mod collector_builder; @@ -108,6 +109,7 @@ impl TransposedS3EventMessages { portal_run_id, storage_class, last_modified_date, + .. } = message; self.object_ids.push(object_id); @@ -153,15 +155,20 @@ impl From for Events { let mut object_removed = FlatS3EventMessages::default(); let mut other = FlatS3EventMessages::default(); - messages.into_inner().into_iter().for_each(|message| { - if message.event_name.contains("ObjectCreated") { - object_created.0.push(message); - } else if message.event_name.contains("ObjectRemoved") { - object_removed.0.push(message); - } else { - other.0.push(message); - } - }); + messages + .into_inner() + .into_iter() + .for_each(|message| match message.event_type { + Created => { + object_created.0.push(message); + } + Removed => { + object_removed.0.push(message); + } + Other => { + other.0.push(message); + } + }); Self { object_created: TransposedS3EventMessages::from(object_created), @@ -281,6 +288,13 @@ impl PartialEq for FlatS3EventMessage { } } +#[derive(Debug, Eq, PartialEq)] +pub enum EventType { + Created, + Removed, + Other, +} + /// A flattened AWS S3 record #[derive(Debug, Eq)] pub struct FlatS3EventMessage { @@ -295,6 +309,7 @@ pub struct FlatS3EventMessage { pub portal_run_id: String, pub storage_class: Option, pub last_modified_date: Option>, + pub event_type: EventType, } impl FlatS3EventMessage { @@ -399,6 +414,14 @@ impl TryFrom for FlatS3EventMessages { let portal_run_id = event_time.format("%Y%m%d").to_string() + &object_id.to_string()[..8]; + let event_type = if event_name.contains("ObjectCreated") { + Created + } else if event_name.contains("ObjectRemoved") { + Removed + } else { + Other + }; + Ok(FlatS3EventMessage { object_id, event_time, @@ -412,6 +435,7 @@ impl TryFrom for FlatS3EventMessages { // Head field are optionally fetched later. storage_class: None, last_modified_date: None, + event_type, }) }) .collect::>>()?, diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/mod.rs index fad126ba0..f1165b1cf 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/mod.rs @@ -12,12 +12,12 @@ pub mod aws; #[async_trait] pub trait Collect { /// Collect into events. - async fn collect(self) -> Result; + async fn collect(self) -> Result; } /// The type of event. #[derive(Debug)] #[non_exhaustive] -pub enum EventType { +pub enum EventSourceType { S3(Events), } diff --git a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs index 271bedf47..ff9953634 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs @@ -91,7 +91,7 @@ mod tests { use super::*; use crate::database::aws::ingester::tests::assert_deleted; use crate::database::aws::migration::tests::MIGRATOR; - use crate::events::aws::collecter::tests::set_s3_client_expectations; + use crate::events::aws::collecter::tests::{expected_head_object, set_s3_client_expectations}; use crate::events::aws::collector_builder::tests::set_sqs_client_expectations; use crate::events::aws::tests::expected_event_record; use aws_lambda_events::sqs::SqsMessage; @@ -103,7 +103,7 @@ mod tests { let mut s3_client = S3Client::default(); set_sqs_client_expectations(&mut sqs_client); - set_s3_client_expectations(&mut s3_client, 2); + set_s3_client_expectations(&mut s3_client, vec![|| Ok(expected_head_object())]); let ingester = receive_and_ingest(s3_client, sqs_client, Some("url"), Some(Client::new(pool))) @@ -122,7 +122,7 @@ mod tests { async fn test_ingest_event(pool: PgPool) { let mut s3_client = S3Client::default(); - set_s3_client_expectations(&mut s3_client, 2); + set_s3_client_expectations(&mut s3_client, vec![|| Ok(expected_head_object())]); let event = SqsEvent { records: vec![SqsMessage { From a4ff71053ab6a85116692efaed05861f616fca91 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 20 Dec 2023 09:28:51 +1100 Subject: [PATCH 20/24] fix(filemanager): head object does not return storage class when it is standard --- .../migrations/0001_add_object_table.sql | 4 +-- .../aws/0002_add_s3_object_table.sql | 2 +- .../filemanager/src/database/aws/ingester.rs | 2 +- .../filemanager/src/events/aws/collecter.rs | 16 +++++++--- .../filemanager/src/events/aws/mod.rs | 31 +++++++++++++++---- 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql b/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql index cfcbe680f..3d9245e0e 100644 --- a/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql +++ b/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql @@ -11,9 +11,9 @@ create table object ( -- A unique identifier for the object, if it is present. hash varchar(255) default null, -- When this object was created. - created_date timestamptz not null, + created_date timestamptz not null default now(), -- When this object was last modified. - last_modified_date timestamptz not null, + last_modified_date timestamptz not null default now(), -- When this object was deleted, a null value means that the object has not yet been deleted. deleted_date timestamptz default null, -- The date of the object and its id combined. diff --git a/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql b/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql index f3ecce153..b77d7a8f2 100644 --- a/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql +++ b/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql @@ -5,5 +5,5 @@ create table s3_object( -- The object id. object_id uuid references object (object_id) primary key, -- The S3 storage class of the object. - storage_class storage_class default null + storage_class storage_class not null default 'Standard' ); \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs index fadfaaf0c..c9b9e727c 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs @@ -212,7 +212,7 @@ pub(crate) mod tests { events.object_created.storage_classes[0] = Some(StorageClass::Standard); events.object_removed.last_modified_dates[0] = Some(DateTime::default()); - events.object_removed.storage_classes[0] = Some(StorageClass::Standard); + events.object_removed.storage_classes[0] = None; events } diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs index b213e0f98..2338891d2 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs @@ -73,7 +73,7 @@ impl Collecter { ) -> Result { Ok(FlatS3EventMessages( join_all(events.into_inner().into_iter().map(|mut event| async move { - // Only run this on created events. + // No need to run this unnecessarily on removed events. match event.event_type { EventType::Removed | EventType::Other => return Ok(event), _ => {} @@ -81,6 +81,9 @@ impl Collecter { trace!(key = ?event.key, bucket = ?event.bucket, "updating event"); + // Race condition: it's possible that an object gets deleted so quickly that it + // occurs before calling head. This means that there may be cases where the storage + // class and other fields are not known. if let Some(head) = Self::head(client, &event.key, &event.bucket).await? { trace!(head = ?head, "received head object output"); @@ -124,7 +127,7 @@ impl Collect for Collecter { pub(crate) mod tests { use crate::events::aws::collecter::Collecter; use crate::events::aws::tests::expected_flat_events; - use crate::events::aws::StorageClass::Standard; + use crate::events::aws::StorageClass::IntelligentTiering; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::primitives::{DateTimeFormat, SdkBody}; use aws_sdk_s3::types; @@ -189,7 +192,7 @@ pub(crate) mod tests { .into_iter(); let first = result.next().unwrap(); - assert_eq!(first.storage_class, Some(Standard)); + assert_eq!(first.storage_class, Some(IntelligentTiering)); assert_eq!(first.last_modified_date, Some(Default::default())); let second = result.next().unwrap(); @@ -213,7 +216,10 @@ pub(crate) mod tests { match result { EventSourceType::S3(events) => { - assert_eq!(events.object_created.storage_classes[0], Some(Standard)); + assert_eq!( + events.object_created.storage_classes[0], + Some(IntelligentTiering) + ); assert_eq!( events.object_created.last_modified_dates[0], Some(Default::default()) @@ -245,7 +251,7 @@ pub(crate) mod tests { primitives::DateTime::from_str("1970-01-01T00:00:00Z", DateTimeFormat::DateTime) .unwrap(), ) - .storage_class(types::StorageClass::Standard) + .storage_class(types::StorageClass::IntelligentTiering) .build() } diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs index 7341236a6..2626b50dd 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs @@ -313,7 +313,7 @@ pub struct FlatS3EventMessage { } impl FlatS3EventMessage { - /// Update the storage class if not None. + /// Update the storage class if not None.` pub fn update_storage_class(mut self, storage_class: Option) -> Self { storage_class .into_iter() @@ -422,6 +422,14 @@ impl TryFrom for FlatS3EventMessages { Other }; + // S3 does not return a storage class for standard, so this is assumed to be + // the default. See https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax + let storage_class = if let Created = event_type { + Some(StorageClass::Standard) + } else { + None + }; + Ok(FlatS3EventMessage { object_id, event_time, @@ -432,8 +440,8 @@ impl TryFrom for FlatS3EventMessages { e_tag, sequencer, portal_run_id, - // Head field are optionally fetched later. - storage_class: None, + storage_class, + // Head field are fetched later. last_modified_date: None, event_type, }) @@ -451,7 +459,9 @@ impl From> for FlatS3EventMessages { #[cfg(test)] pub(crate) mod tests { - use crate::events::aws::{Events, FlatS3EventMessage, FlatS3EventMessages, S3EventMessage}; + use crate::events::aws::{ + Events, FlatS3EventMessage, FlatS3EventMessages, S3EventMessage, StorageClass, + }; use chrono::{DateTime, Utc}; use serde_json::json; @@ -470,6 +480,7 @@ pub(crate) mod tests { "ObjectRemoved:Delete", EXPECTED_SEQUENCER_DELETED, None, + None, ); let second = result.next().unwrap(); @@ -478,6 +489,7 @@ pub(crate) mod tests { "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED, Some(0), + Some(StorageClass::Standard), ); let third = result.next().unwrap(); @@ -486,6 +498,7 @@ pub(crate) mod tests { "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED, Some(0), + Some(StorageClass::Standard), ); } @@ -500,6 +513,7 @@ pub(crate) mod tests { "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED, Some(0), + Some(StorageClass::Standard), ); let second = result.next().unwrap(); @@ -508,6 +522,7 @@ pub(crate) mod tests { "ObjectRemoved:Delete", EXPECTED_SEQUENCER_DELETED, None, + None, ); } @@ -516,6 +531,7 @@ pub(crate) mod tests { event_name: &str, sequencer: &str, size: Option, + storage_class: Option, ) { assert_eq!(event.event_time, DateTime::::default()); assert_eq!(event.event_name, event_name); @@ -525,7 +541,7 @@ pub(crate) mod tests { assert_eq!(event.e_tag, Some(EXPECTED_E_TAG.to_string())); // pragma: allowlist secret assert_eq!(event.sequencer, Some(sequencer.to_string())); assert!(event.portal_run_id.starts_with("19700101")); - assert_eq!(event.storage_class, None); + assert_eq!(event.storage_class, storage_class); assert_eq!(event.last_modified_date, None); } @@ -550,7 +566,10 @@ pub(crate) mod tests { Some(EXPECTED_SEQUENCER_CREATED.to_string()) ); assert!(result.object_created.portal_run_ids[0].starts_with("19700101")); - assert_eq!(result.object_created.storage_classes[0], None); + assert_eq!( + result.object_created.storage_classes[0], + Some(StorageClass::Standard) + ); assert_eq!(result.object_created.last_modified_dates[0], None); assert_eq!( From ff9dbb0b49a6e473ce4e9f540f39919c27892152 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 20 Dec 2023 09:36:27 +1100 Subject: [PATCH 21/24] fix(filemanager): storage class can still be unknoen before a head object call --- .../aws/0002_add_s3_object_table.sql | 2 +- .../filemanager/src/events/aws/collecter.rs | 7 ++++- .../filemanager/src/events/aws/mod.rs | 27 +++---------------- 3 files changed, 11 insertions(+), 25 deletions(-) diff --git a/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql b/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql index b77d7a8f2..ba2fc4cce 100644 --- a/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql +++ b/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql @@ -5,5 +5,5 @@ create table s3_object( -- The object id. object_id uuid references object (object_id) primary key, -- The S3 storage class of the object. - storage_class storage_class not null default 'Standard' + storage_class storage_class not null ); \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs index 2338891d2..e4fbdb41c 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs @@ -4,6 +4,7 @@ use crate::error::Error::S3Error; use async_trait::async_trait; use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput}; use aws_sdk_s3::primitives; +use aws_sdk_s3::types::StorageClass::Standard; use chrono::{DateTime, NaiveDateTime, Utc}; use futures::future::join_all; use mockall_double::double; @@ -95,8 +96,12 @@ impl Collecter { .. } = head; + // S3 does not return a storage class for standard, which means this is the + // default. See https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax event = event - .update_storage_class(storage_class.and_then(StorageClass::from_aws)) + .update_storage_class(StorageClass::from_aws( + storage_class.unwrap_or(Standard), + )) .update_last_modified_date(Self::convert_datetime(last_modified)) .update_size(content_length) .update_e_tag(e_tag); diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs index 2626b50dd..854aa6bed 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs @@ -422,14 +422,6 @@ impl TryFrom for FlatS3EventMessages { Other }; - // S3 does not return a storage class for standard, so this is assumed to be - // the default. See https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax - let storage_class = if let Created = event_type { - Some(StorageClass::Standard) - } else { - None - }; - Ok(FlatS3EventMessage { object_id, event_time, @@ -440,8 +432,8 @@ impl TryFrom for FlatS3EventMessages { e_tag, sequencer, portal_run_id, - storage_class, // Head field are fetched later. + storage_class: None, last_modified_date: None, event_type, }) @@ -459,9 +451,7 @@ impl From> for FlatS3EventMessages { #[cfg(test)] pub(crate) mod tests { - use crate::events::aws::{ - Events, FlatS3EventMessage, FlatS3EventMessages, S3EventMessage, StorageClass, - }; + use crate::events::aws::{Events, FlatS3EventMessage, FlatS3EventMessages, S3EventMessage}; use chrono::{DateTime, Utc}; use serde_json::json; @@ -480,7 +470,6 @@ pub(crate) mod tests { "ObjectRemoved:Delete", EXPECTED_SEQUENCER_DELETED, None, - None, ); let second = result.next().unwrap(); @@ -489,7 +478,6 @@ pub(crate) mod tests { "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED, Some(0), - Some(StorageClass::Standard), ); let third = result.next().unwrap(); @@ -498,7 +486,6 @@ pub(crate) mod tests { "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED, Some(0), - Some(StorageClass::Standard), ); } @@ -513,7 +500,6 @@ pub(crate) mod tests { "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED, Some(0), - Some(StorageClass::Standard), ); let second = result.next().unwrap(); @@ -522,7 +508,6 @@ pub(crate) mod tests { "ObjectRemoved:Delete", EXPECTED_SEQUENCER_DELETED, None, - None, ); } @@ -531,7 +516,6 @@ pub(crate) mod tests { event_name: &str, sequencer: &str, size: Option, - storage_class: Option, ) { assert_eq!(event.event_time, DateTime::::default()); assert_eq!(event.event_name, event_name); @@ -541,7 +525,7 @@ pub(crate) mod tests { assert_eq!(event.e_tag, Some(EXPECTED_E_TAG.to_string())); // pragma: allowlist secret assert_eq!(event.sequencer, Some(sequencer.to_string())); assert!(event.portal_run_id.starts_with("19700101")); - assert_eq!(event.storage_class, storage_class); + assert_eq!(event.storage_class, None); assert_eq!(event.last_modified_date, None); } @@ -566,10 +550,7 @@ pub(crate) mod tests { Some(EXPECTED_SEQUENCER_CREATED.to_string()) ); assert!(result.object_created.portal_run_ids[0].starts_with("19700101")); - assert_eq!( - result.object_created.storage_classes[0], - Some(StorageClass::Standard) - ); + assert_eq!(result.object_created.storage_classes[0], None); assert_eq!(result.object_created.last_modified_dates[0], None); assert_eq!( From 205adb1eab3e2555b8bc712b9ee12fca58260e65 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 20 Dec 2023 12:19:28 +1100 Subject: [PATCH 22/24] fix(filemanager): make object parsing less strict to avoid errors in Lambda function --- .../filemanager/src/events/aws/collector_builder.rs | 4 +++- .../stateful/filemanager/filemanager/src/handlers/aws.rs | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs index 20f88658d..02c3d09ff 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs @@ -66,7 +66,9 @@ impl CollecterBuilder { trace!(message = ?message, "got the message"); if let Some(body) = message.body() { - serde_json::from_str(body).map_err(|err| DeserializeError(err.to_string())) + let events: Option = serde_json::from_str(body) + .map_err(|err| DeserializeError(err.to_string()))?; + Ok(events.unwrap_or_default()) } else { Err(SQSReceiveError("No body in SQS message".to_string())) } diff --git a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs index ff9953634..d75e2e2b3 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs @@ -56,8 +56,8 @@ pub async fn ingest_event( .into_iter() .filter_map(|event| { event.body.map(|body| { - let body: FlatS3EventMessages = serde_json::from_str(&body)?; - Ok(body) + let body: Option = serde_json::from_str(&body)?; + Ok(body.unwrap_or_default()) }) }) .collect::, Error>>()? From 864746456f739b158f8e3c6ce4055a1020291b33 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 2 Jan 2024 08:40:08 +1100 Subject: [PATCH 23/24] refactor(filemanager): remove aws directory from migrations as its not that useful a distinction for now --- lib/workload/stateful/filemanager/build.rs | 4 ++-- .../migrations/{aws => }/0002_add_s3_object_table.sql | 0 .../filemanager/src/database/aws/migration.rs | 10 +--------- 3 files changed, 3 insertions(+), 11 deletions(-) rename lib/workload/stateful/filemanager/database/migrations/{aws => }/0002_add_s3_object_table.sql (100%) diff --git a/lib/workload/stateful/filemanager/build.rs b/lib/workload/stateful/filemanager/build.rs index d5068697c..a8998c8ce 100644 --- a/lib/workload/stateful/filemanager/build.rs +++ b/lib/workload/stateful/filemanager/build.rs @@ -1,5 +1,5 @@ // generated by `sqlx migrate build-script` fn main() { // trigger recompilation when a new migration is added - println!("cargo:rerun-if-changed=migrations"); -} + println!("cargo:rerun-if-changed=database/migrations"); +} \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql b/lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql similarity index 100% rename from lib/workload/stateful/filemanager/database/migrations/aws/0002_add_s3_object_table.sql rename to lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs index 47ebb6df9..04c4923de 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs @@ -27,15 +27,7 @@ impl Migration { /// Get the underlying sqlx migrator for the migrations. pub fn migrator() -> Migrator { - let mut migrator = migrate!("../database/migrations"); - let aws_migrations = migrate!("../database/migrations/aws"); - - migrator - .migrations - .to_mut() - .extend(aws_migrations.migrations.into_owned()); - - migrator + migrate!("../database/migrations") } /// Get a reference to the database client. From aa023cf5f0eb730f3c4e0b7558620055049f4113 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 2 Jan 2024 10:53:20 +1100 Subject: [PATCH 24/24] refactor(filemanager): remove any ip address from public database cluster --- .../filemanager/deploy/bin/filemanager.ts | 4 +++- .../filemanager/deploy/constructs/database.ts | 16 +++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts index 3cf86c656..061692463 100644 --- a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts +++ b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts @@ -27,7 +27,9 @@ new FilemanagerStack( enableMonitoring: { enablePerformanceInsights: true, }, - public: true, + public: [ + // Put your IP here if you want the database to be reachable. + ], migrateDatabase: process.env.FILEMANAGER_DEPLOY_MIGRATE_DATABASE == 'true', } ); diff --git a/lib/workload/stateful/filemanager/deploy/constructs/database.ts b/lib/workload/stateful/filemanager/deploy/constructs/database.ts index c84593eb8..1b8530808 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/database.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/database.ts @@ -33,9 +33,9 @@ export type EnableMonitoringProps = { */ export type DatabaseSettings = { /** - * Whether the database is publically available. + * If present, specifies the database as public and adds additional inbound CIDRs to the security group. */ - readonly public?: boolean; + readonly public?: string[]; /** * Whether to destroy the database on stack removal. Defaults to keeping a snapshot. */ @@ -136,11 +136,13 @@ export class Database extends Construct { }); if (props.public) { - // If it's public, anyone can connect. - this._securityGroup.addIngressRule( - ec2.Peer.anyIpv4(), - ec2.Port.tcp(this._cluster.clusterEndpoint.port) - ); + // If it's public, set the CIDRs from the config. + props.public.forEach((cidr) => { + this._securityGroup.addIngressRule( + ec2.Peer.ipv4(cidr), + ec2.Port.tcp(this._cluster.clusterEndpoint.port) + ); + }); } else { // Any inbound connections within the same security group are allowed access to the database port. this._securityGroup.addIngressRule(