From 13925b64df2a61b0e8914b41eccdebf4be60b073 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 11 Mar 2024 12:12:55 +1100 Subject: [PATCH 1/3] refactor: swap out RustFunction and use cargo-lambda-cdk --- .../orcabus-stateless-pipeline-stack.ts | 12 - .../deploy/constructs/functions/function.ts | 20 +- package.json | 2 +- yarn.lock | 442 +++++------------- 4 files changed, 132 insertions(+), 344 deletions(-) diff --git a/lib/pipeline/orcabus-stateless-pipeline-stack.ts b/lib/pipeline/orcabus-stateless-pipeline-stack.ts index 771e676e9..100a4b620 100644 --- a/lib/pipeline/orcabus-stateless-pipeline-stack.ts +++ b/lib/pipeline/orcabus-stateless-pipeline-stack.ts @@ -21,12 +21,6 @@ export class StatelessPipelineStack extends cdk.Stack { }); const unitTest = new pipelines.CodeBuildStep('UnitTest', { - installCommands: [ - // RUST installation - `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y`, - `source $HOME/.cargo/env`, - `pip3 install cargo-lambda`, - ], commands: ['yarn install --immutable', 'make suite'], input: sourceFile, primaryOutputDirectory: '.', @@ -56,12 +50,6 @@ export class StatelessPipelineStack extends cdk.Stack { }); const synthAction = new pipelines.CodeBuildStep('Synth', { - installCommands: [ - // RUST installation - `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y`, - `source $HOME/.cargo/env`, - `pip3 install cargo-lambda`, - ], commands: ['yarn install --immutable', 'yarn run cdk-stateless-pipeline synth'], input: unitTest, primaryOutputDirectory: 'cdk.out', diff --git a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts index 67c9202fc..8e6bcbb54 100644 --- a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts @@ -1,11 +1,11 @@ import { Construct } from 'constructs'; -import { RustFunction } from 'rust.aws-cdk-lambda'; import { Duration } from 'aws-cdk-lib'; -import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; import { ISecurityGroup, IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { Architecture, Version } from 'aws-cdk-lib/aws-lambda'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; +import { RustFunction } from 'cargo-lambda-cdk'; +import path from 'path'; /** * Properties for the database. @@ -28,7 +28,7 @@ export type FunctionPropsNoPackage = DatabaseProps & { /** * Additional build environment variables when building the Lambda function. */ - readonly buildEnvironment?: { [key: string]: string | undefined }; + readonly buildEnvironment?: { [key: string]: string }; /** * RUST_LOG string, defaults to trace on local crates and info everywhere else. */ @@ -79,8 +79,6 @@ export class Function extends Construct { description: 'Security group that allows a filemanager Lambda function to egress out.', }); - CargoSettings.BUILD_INDIVIDUALLY = true; - // Todo use secrets manager for this and query for password within Lambda functions. const unsafeConnection = `postgres://` + @@ -95,8 +93,11 @@ export class Function extends Construct { `${props.databaseSecret.secretValueFromJson('dbname').unsafeUnwrap()}`; this._function = new RustFunction(this, 'RustFunction', { - package: props.package, - target: 'aarch64-unknown-linux-gnu', + manifestPath: path.join(__dirname, '..', '..', '..'), + binaryName: props.package, + bundling: { + environment: props.buildEnvironment + }, memorySize: 128, timeout: Duration.seconds(28), environment: { @@ -104,10 +105,7 @@ export class Function extends Construct { RUST_LOG: props.rustLog ?? `info,${props.package.replace('-', '_')}=trace,filemanager=trace`, }, - buildEnvironment: props.buildEnvironment, - // Todo, fix this so that it's not relying on the path. - extraBuildArgs: ['--manifest-path', `lib/workload/stateless/filemanager/${props.package}/Cargo.toml`], - architecture: Architecture.ARM_64, + architecture: Architecture.X86_64, role: this._role, vpc: props.vpc, vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, diff --git a/package.json b/package.json index 644197e23..7f53d6f54 100644 --- a/package.json +++ b/package.json @@ -29,10 +29,10 @@ "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.110.0-alpha.0", "@aws-cdk/aws-lambda-python-alpha": "2.110.0-alpha.0", "aws-cdk-lib": "^2.126.0", + "cargo-lambda-cdk": "^0.0.19", "cdk-nag": "^2.28.27", "constructs": "^10.2.69", "dotenv": "^16.3.1", - "rust.aws-cdk-lambda": "^1.2.1", "source-map-support": "^0.5.21" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index acbb8486e..2e3863022 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22,13 +22,6 @@ __metadata: languageName: node linkType: hard -"@aws-cdk/asset-awscli-v1@npm:^2.2.201": - version: 2.2.201 - resolution: "@aws-cdk/asset-awscli-v1@npm:2.2.201" - checksum: d77a7b90dee16da9c15c0a1cc2d531c100097c562f0802fc15838928a7921e417a3f4c5501ca40fdc768076f981f57f59dcb92029787be34a465e6eba9f40a2a - languageName: node - linkType: hard - "@aws-cdk/asset-awscli-v1@npm:^2.2.202": version: 2.2.202 resolution: "@aws-cdk/asset-awscli-v1@npm:2.2.202" @@ -438,6 +431,16 @@ __metadata: languageName: node linkType: hard +"@babel/runtime-corejs3@npm:^7.16.5": + version: 7.24.0 + resolution: "@babel/runtime-corejs3@npm:7.24.0" + dependencies: + core-js-pure: ^3.30.2 + regenerator-runtime: ^0.14.0 + checksum: cd4f2cde0878c911b2eeebf1cb48df819c862492da163bb9001ba690c82e9addd1d14087a8b0dd87fb1da2d916fa3b85d9a8cece4d46f8e5c51d734acda5a8bd + languageName: node + linkType: hard + "@babel/template@npm:^7.22.15, @babel/template@npm:^7.3.3": version: 7.22.15 resolution: "@babel/template@npm:7.22.15" @@ -492,6 +495,41 @@ __metadata: languageName: node linkType: hard +"@chevrotain/cst-dts-gen@npm:10.5.0": + version: 10.5.0 + resolution: "@chevrotain/cst-dts-gen@npm:10.5.0" + dependencies: + "@chevrotain/gast": 10.5.0 + "@chevrotain/types": 10.5.0 + lodash: 4.17.21 + checksum: 3ff851d5cbccc509269bb77078dafd7acfcd2e128e7d362718cde728f3fa95f4dd58eb1eea67ecf11453fba70bded97df55c5ba31ed93fb2dec4324663bd2eee + languageName: node + linkType: hard + +"@chevrotain/gast@npm:10.5.0": + version: 10.5.0 + resolution: "@chevrotain/gast@npm:10.5.0" + dependencies: + "@chevrotain/types": 10.5.0 + lodash: 4.17.21 + checksum: 35183e7067bc936db9ecfea7624ee3178634618cf1518ea3470b4ed208fb19454dc3ed990a0de2dab80794251398a857ad17d26cc552eac497a2aa974f76b86d + languageName: node + linkType: hard + +"@chevrotain/types@npm:10.5.0": + version: 10.5.0 + resolution: "@chevrotain/types@npm:10.5.0" + checksum: 72f7b48de1888ab14831108da4b0ab3ef244e1101a4094240382e4983a9e71aae6f8a87e09b819854d1028cee08f97b7d2a81fce935742c55d2bc497b7cad350 + languageName: node + linkType: hard + +"@chevrotain/utils@npm:10.5.0": + version: 10.5.0 + resolution: "@chevrotain/utils@npm:10.5.0" + checksum: f3ae9e0fea2e928a1a4930311d3ef04f45c29fa58ba4d5d2ca43c33355ac47f95ce99a98d6496706e2e7f773ef684a9a7e7cbd7b77c00af9158f08c82d88212b + languageName: node + linkType: hard + "@cspotcode/source-map-support@npm:^0.8.0": version: 0.8.1 resolution: "@cspotcode/source-map-support@npm:0.8.1" @@ -1041,15 +1079,6 @@ __metadata: languageName: node linkType: hard -"@types/fs-extra@npm:^9.0.13": - version: 9.0.13 - resolution: "@types/fs-extra@npm:9.0.13" - dependencies: - "@types/node": "*" - checksum: add79e212acd5ac76b97b9045834e03a7996aef60a814185e0459088fd290519a3c1620865d588fa36c4498bf614210d2a703af5cf80aa1dbc125db78f6edac3 - languageName: node - linkType: hard - "@types/graceful-fs@npm:^4.1.3": version: 4.1.9 resolution: "@types/graceful-fs@npm:4.1.9" @@ -1101,13 +1130,6 @@ __metadata: languageName: node linkType: hard -"@types/minimist@npm:^1.2.2": - version: 1.2.5 - resolution: "@types/minimist@npm:1.2.5" - checksum: 477047b606005058ab0263c4f58097136268007f320003c348794f74adedc3166ffc47c80ec3e94687787f2ab7f4e72c468223946e79892cf0fd9e25e9970a90 - languageName: node - linkType: hard - "@types/node@npm:*, @types/node@npm:^20.4.0": version: 20.10.5 resolution: "@types/node@npm:20.10.5" @@ -1117,20 +1139,6 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^17.0": - version: 17.0.45 - resolution: "@types/node@npm:17.0.45" - checksum: aa04366b9103b7d6cfd6b2ef64182e0eaa7d4462c3f817618486ea0422984c51fc69fd0d436eae6c9e696ddfdbec9ccaa27a917f7c2e8c75c5d57827fe3d95e8 - languageName: node - linkType: hard - -"@types/ps-tree@npm:^1.1.2": - version: 1.1.6 - resolution: "@types/ps-tree@npm:1.1.6" - checksum: bf5b7bb9bd11b8762a8302b93c335728ecb19c85a74c640a3888d476368a03733f11612b9a87b1ad9ea56f95720db23a824c78113b16024dc59264b7f9008df5 - languageName: node - linkType: hard - "@types/semver@npm:^7.5.0": version: 7.5.6 resolution: "@types/semver@npm:7.5.6" @@ -1145,13 +1153,6 @@ __metadata: languageName: node linkType: hard -"@types/which@npm:^2.0.1": - version: 2.0.2 - resolution: "@types/which@npm:2.0.2" - checksum: 8626a3c2f6db676c449142e1082e33ea0c9d88b8a2bd796366b944891e6da0088b2aa83d3fa9c79e6696f7381a851fc76d43bd353eb6c4d98a7775b4ae0a96a5 - languageName: node - linkType: hard - "@types/yargs-parser@npm:*": version: 21.0.3 resolution: "@types/yargs-parser@npm:21.0.3" @@ -1298,22 +1299,6 @@ __metadata: languageName: node linkType: hard -"@ziglang/cli@npm:^0.0.11": - version: 0.0.11 - resolution: "@ziglang/cli@npm:0.0.11" - dependencies: - zx: ^6.0.7 - bin: - zig: bin/zig - zig-cli: zig.js - zig-install: install.js - zig-postinstall: postinstall.js - zig-reinstall: reinstall.js - zig-uninstall: uninstall.js - checksum: c1286eab7cb8572d422adc0f5d90a25cbc42eacbb6818044005f13fd50c5323a5a15d80d089dfdf10120555c5eae7d0d8d77acb34b38013aab43a3cdebdf88f4 - languageName: node - linkType: hard - "abbrev@npm:^2.0.0": version: 2.0.0 resolution: "abbrev@npm:2.0.0" @@ -1491,29 +1476,6 @@ __metadata: languageName: node linkType: hard -"aws-cdk-lib@npm:^2.*": - version: 2.115.0 - resolution: "aws-cdk-lib@npm:2.115.0" - dependencies: - "@aws-cdk/asset-awscli-v1": ^2.2.201 - "@aws-cdk/asset-kubectl-v20": ^2.1.2 - "@aws-cdk/asset-node-proxy-agent-v6": ^2.0.1 - "@balena/dockerignore": ^1.0.2 - case: 1.6.3 - fs-extra: ^11.2.0 - ignore: ^5.3.0 - jsonschema: ^1.4.1 - minimatch: ^3.1.2 - punycode: ^2.3.1 - semver: ^7.5.4 - table: ^6.8.1 - yaml: 1.10.2 - peerDependencies: - constructs: ^10.0.0 - checksum: 8f518d2819096df0254d5113fdd1b3bfd09b2b2710312df3c4018e58498a0b15b7a08fd39642fe6f94ebdf52000abac21a804d9bd05d533c50860efac99cb761 - languageName: node - linkType: hard - "aws-cdk-lib@npm:^2.126.0": version: 2.126.0 resolution: "aws-cdk-lib@npm:2.126.0" @@ -1749,6 +1711,18 @@ __metadata: languageName: node linkType: hard +"cargo-lambda-cdk@npm:^0.0.19": + version: 0.0.19 + resolution: "cargo-lambda-cdk@npm:0.0.19" + dependencies: + js-toml: ^0.1.1 + peerDependencies: + aws-cdk-lib: ^2.1.0 + constructs: ^10.0.5 + checksum: a2ad75973055b5b7ab3b4f7233643677befec10ea9b6a2e4a565dd61783c4247f383f6455b19986dd2ca8bf050c656bd0ebf89161ea46e7626e875585b9d606a + languageName: node + linkType: hard + "case@npm:1.6.3": version: 1.6.3 resolution: "case@npm:1.6.3" @@ -1787,13 +1761,6 @@ __metadata: languageName: node linkType: hard -"chalk@npm:^5.0.1": - version: 5.3.0 - resolution: "chalk@npm:5.3.0" - checksum: 623922e077b7d1e9dedaea6f8b9e9352921f8ae3afe739132e0e00c275971bdd331268183b2628cf4ab1727c45ea1f28d7e24ac23ce1db1eb653c414ca8a5a80 - languageName: node - linkType: hard - "char-regex@npm:^1.0.2": version: 1.0.2 resolution: "char-regex@npm:1.0.2" @@ -1801,6 +1768,20 @@ __metadata: languageName: node linkType: hard +"chevrotain@npm:^10.4.1": + version: 10.5.0 + resolution: "chevrotain@npm:10.5.0" + dependencies: + "@chevrotain/cst-dts-gen": 10.5.0 + "@chevrotain/gast": 10.5.0 + "@chevrotain/types": 10.5.0 + "@chevrotain/utils": 10.5.0 + lodash: 4.17.21 + regexp-to-ast: 0.5.0 + checksum: b641f149f60979a29eff2434d745e9565a7c89422b601d554bcf8f047f7d8ff776b9a54b1b36085a622e3f1ed7eb4b8721b5a5348d90ae2567ce7594b10f25aa + languageName: node + linkType: hard + "chownr@npm:^2.0.0": version: 2.0.0 resolution: "chownr@npm:2.0.0" @@ -1893,7 +1874,7 @@ __metadata: languageName: node linkType: hard -"constructs@npm:^10.*, constructs@npm:^10.2.69": +"constructs@npm:^10.2.69": version: 10.3.0 resolution: "constructs@npm:10.3.0" checksum: d8d4ea4e4614914e119b1fd5fc6da0deac909a22b0dbe09423cefb3da54a2866cb9986b371649dadd90b09c56fe69ec22fe3eaab475f8914ac702ed8205e13ca @@ -1907,6 +1888,13 @@ __metadata: languageName: node linkType: hard +"core-js-pure@npm:^3.30.2": + version: 3.36.0 + resolution: "core-js-pure@npm:3.36.0" + checksum: 12a0588981efdc710426c688f6d5f6abaee76858ff32d21c6d7b81bc81c39b7cebb2733f8e822862b2a7f329f8fe37065a33ff6c4fd9253b3a1ad3cf636e483e + languageName: node + linkType: hard + "create-jest@npm:^29.7.0": version: 29.7.0 resolution: "create-jest@npm:29.7.0" @@ -1942,13 +1930,6 @@ __metadata: languageName: node linkType: hard -"data-uri-to-buffer@npm:^4.0.0": - version: 4.0.1 - resolution: "data-uri-to-buffer@npm:4.0.1" - checksum: 0d0790b67ffec5302f204c2ccca4494f70b4e2d940fea3d36b09f0bb2b8539c2e86690429eb1f1dc4bcc9e4df0644193073e63d9ee48ac9fce79ec1506e4aa4c - languageName: node - linkType: hard - "debug@npm:4, debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.1, debug@npm:^4.3.2, debug@npm:^4.3.4": version: 4.3.4 resolution: "debug@npm:4.3.4" @@ -2033,13 +2014,6 @@ __metadata: languageName: node linkType: hard -"duplexer@npm:~0.1.1": - version: 0.1.2 - resolution: "duplexer@npm:0.1.2" - checksum: 62ba61a830c56801db28ff6305c7d289b6dc9f859054e8c982abd8ee0b0a14d2e9a8e7d086ffee12e868d43e2bbe8a964be55ddbd8c8957714c87373c7a4f9b0 - languageName: node - linkType: hard - "eastasianwidth@npm:^0.2.0": version: 0.2.0 resolution: "eastasianwidth@npm:0.2.0" @@ -2284,21 +2258,6 @@ __metadata: languageName: node linkType: hard -"event-stream@npm:=3.3.4": - version: 3.3.4 - resolution: "event-stream@npm:3.3.4" - dependencies: - duplexer: ~0.1.1 - from: ~0 - map-stream: ~0.1.0 - pause-stream: 0.0.11 - split: 0.3 - stream-combiner: ~0.0.4 - through: ~2.3.1 - checksum: 80b467820b6daf824d9fb4345d2daf115a056e5c104463f2e98534e92d196a27f2df5ea2aa085624db26f4c45698905499e881d13bc7c01f7a13eac85be72a22 - languageName: node - linkType: hard - "execa@npm:^5.0.0": version: 5.1.1 resolution: "execa@npm:5.1.1" @@ -2357,7 +2316,7 @@ __metadata: languageName: node linkType: hard -"fast-glob@npm:^3.2.9, fast-glob@npm:^3.3.0": +"fast-glob@npm:^3.2.9": version: 3.3.2 resolution: "fast-glob@npm:3.3.2" dependencies: @@ -2402,16 +2361,6 @@ __metadata: languageName: node linkType: hard -"fetch-blob@npm:^3.1.2, fetch-blob@npm:^3.1.4": - version: 3.2.0 - resolution: "fetch-blob@npm:3.2.0" - dependencies: - node-domexception: ^1.0.0 - web-streams-polyfill: ^3.0.3 - checksum: f19bc28a2a0b9626e69fd7cf3a05798706db7f6c7548da657cbf5026a570945f5eeaedff52007ea35c8bcd3d237c58a20bf1543bc568ab2422411d762dd3d5bf - languageName: node - linkType: hard - "file-entry-cache@npm:^6.0.1": version: 6.0.1 resolution: "file-entry-cache@npm:6.0.1" @@ -2478,33 +2427,6 @@ __metadata: languageName: node linkType: hard -"formdata-polyfill@npm:^4.0.10": - version: 4.0.10 - resolution: "formdata-polyfill@npm:4.0.10" - dependencies: - fetch-blob: ^3.1.2 - checksum: 82a34df292afadd82b43d4a740ce387bc08541e0a534358425193017bf9fb3567875dc5f69564984b1da979979b70703aa73dee715a17b6c229752ae736dd9db - languageName: node - linkType: hard - -"from@npm:~0": - version: 0.1.7 - resolution: "from@npm:0.1.7" - checksum: b85125b7890489656eb2e4f208f7654a93ec26e3aefaf3bbbcc0d496fc1941e4405834fcc9fe7333192aa2187905510ace70417bbf9ac6f6f4784a731d986939 - languageName: node - linkType: hard - -"fs-extra@npm:^10.1.0": - version: 10.1.0 - resolution: "fs-extra@npm:10.1.0" - dependencies: - graceful-fs: ^4.2.0 - jsonfile: ^6.0.1 - universalify: ^2.0.0 - checksum: dc94ab37096f813cc3ca12f0f1b5ad6744dfed9ed21e953d72530d103cea193c2f81584a39e9dee1bea36de5ee66805678c0dddc048e8af1427ac19c00fffc50 - languageName: node - linkType: hard - "fs-extra@npm:^11.2.0": version: 11.2.0 resolution: "fs-extra@npm:11.2.0" @@ -2672,19 +2594,6 @@ __metadata: languageName: node linkType: hard -"globby@npm:^13.1.1": - version: 13.2.2 - resolution: "globby@npm:13.2.2" - dependencies: - dir-glob: ^3.0.1 - fast-glob: ^3.3.0 - ignore: ^5.2.4 - merge2: ^1.4.1 - slash: ^4.0.0 - checksum: f3d84ced58a901b4fcc29c846983108c426631fe47e94872868b65565495f7bee7b3defd68923bd480582771fd4bbe819217803a164a618ad76f1d22f666f41e - languageName: node - linkType: hard - "graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": version: 4.2.11 resolution: "graceful-fs@npm:4.2.11" @@ -3463,6 +3372,16 @@ __metadata: languageName: node linkType: hard +"js-toml@npm:^0.1.1": + version: 0.1.1 + resolution: "js-toml@npm:0.1.1" + dependencies: + chevrotain: ^10.4.1 + xregexp: ^5.1.1 + checksum: df4a817ea24849fe0e9aaa2f05d48baeba2f6d6326a3af05e26054fcd4f3be5ea3bd7b1e608c9a73b571f99f5606d18097bf912089c59346541617174d0754ee + languageName: node + linkType: hard + "js-yaml@npm:^3.13.1": version: 3.14.1 resolution: "js-yaml@npm:3.14.1" @@ -3638,6 +3557,13 @@ __metadata: languageName: node linkType: hard +"lodash@npm:4.17.21": + version: 4.17.21 + resolution: "lodash@npm:4.17.21" + checksum: eb835a2e51d381e561e508ce932ea50a8e5a68f4ebdd771ea240d3048244a8d13658acbd502cd4829768c56f2e16bdd4340b9ea141297d472517b83868e677f7 + languageName: node + linkType: hard + "lru-cache@npm:^10.0.1, lru-cache@npm:^9.1.1 || ^10.0.0": version: 10.1.0 resolution: "lru-cache@npm:10.1.0" @@ -3707,13 +3633,6 @@ __metadata: languageName: node linkType: hard -"map-stream@npm:~0.1.0": - version: 0.1.0 - resolution: "map-stream@npm:0.1.0" - checksum: 38abbe4eb883888031e6b2fc0630bc583c99396be16b8ace5794b937b682a8a081f03e8b15bfd4914d1bc88318f0e9ac73ba3512ae65955cd449f63256ddb31d - languageName: node - linkType: hard - "merge-stream@npm:^2.0.0": version: 2.0.0 resolution: "merge-stream@npm:2.0.0" @@ -3763,13 +3682,6 @@ __metadata: languageName: node linkType: hard -"minimist@npm:^1.2.6": - version: 1.2.8 - resolution: "minimist@npm:1.2.8" - checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0 - languageName: node - linkType: hard - "minipass-collect@npm:^2.0.1": version: 2.0.1 resolution: "minipass-collect@npm:2.0.1" @@ -3884,24 +3796,6 @@ __metadata: languageName: node linkType: hard -"node-domexception@npm:^1.0.0": - version: 1.0.0 - resolution: "node-domexception@npm:1.0.0" - checksum: ee1d37dd2a4eb26a8a92cd6b64dfc29caec72bff5e1ed9aba80c294f57a31ba4895a60fd48347cf17dd6e766da0ae87d75657dfd1f384ebfa60462c2283f5c7f - languageName: node - linkType: hard - -"node-fetch@npm:^3.2.5": - version: 3.3.2 - resolution: "node-fetch@npm:3.3.2" - dependencies: - data-uri-to-buffer: ^4.0.0 - fetch-blob: ^3.1.4 - formdata-polyfill: ^4.0.10 - checksum: 06a04095a2ddf05b0830a0d5302699704d59bda3102894ea64c7b9d4c865ecdff2d90fd042df7f5bc40337266961cb6183dcc808ea4f3000d024f422b462da92 - languageName: node - linkType: hard - "node-gyp@npm:latest": version: 10.0.1 resolution: "node-gyp@npm:10.0.1" @@ -4009,6 +3903,7 @@ __metadata: "@typescript-eslint/parser": ^6.19.1 aws-cdk: ^2.126.0 aws-cdk-lib: ^2.126.0 + cargo-lambda-cdk: ^0.0.19 cdk-nag: ^2.28.27 constructs: ^10.2.69 dotenv: ^16.3.1 @@ -4018,7 +3913,6 @@ __metadata: jest: ^29.6.1 jest-junit: ^16.0.0 prettier: ^3.2.4 - rust.aws-cdk-lambda: ^1.2.1 source-map-support: ^0.5.21 ts-jest: ^29.1.1 ts-node: ^10.9.1 @@ -4144,15 +4038,6 @@ __metadata: languageName: node linkType: hard -"pause-stream@npm:0.0.11": - version: 0.0.11 - resolution: "pause-stream@npm:0.0.11" - dependencies: - through: ~2.3 - checksum: 3c4a14052a638b92e0c96eb00c0d7977df7f79ea28395250c525d197f1fc02d34ce1165d5362e2e6ebbb251524b94a76f3f0d4abc39ab8b016d97449fe15583c - languageName: node - linkType: hard - "picocolors@npm:^1.0.0": version: 1.0.0 resolution: "picocolors@npm:1.0.0" @@ -4246,17 +4131,6 @@ __metadata: languageName: node linkType: hard -"ps-tree@npm:^1.2.0": - version: 1.2.0 - resolution: "ps-tree@npm:1.2.0" - dependencies: - event-stream: =3.3.4 - bin: - ps-tree: ./bin/ps-tree.js - checksum: e635dd00f53d30d31696cf5f95b3a8dbdf9b1aeb36d4391578ce8e8cd22949b7c5536c73b0dc18c78615ea3ddd4be96101166be59ca2e3e3cb1e2f79ba3c7f98 - languageName: node - linkType: hard - "punycode@npm:^2.1.0, punycode@npm:^2.3.1": version: 2.3.1 resolution: "punycode@npm:2.3.1" @@ -4285,6 +4159,20 @@ __metadata: languageName: node linkType: hard +"regenerator-runtime@npm:^0.14.0": + version: 0.14.1 + resolution: "regenerator-runtime@npm:0.14.1" + checksum: 9f57c93277b5585d3c83b0cf76be47b473ae8c6d9142a46ce8b0291a04bb2cf902059f0f8445dcabb3fb7378e5fe4bb4ea1e008876343d42e46d3b484534ce38 + languageName: node + linkType: hard + +"regexp-to-ast@npm:0.5.0": + version: 0.5.0 + resolution: "regexp-to-ast@npm:0.5.0" + checksum: 72e32f2a1217bb22398ac30867ddd43e16943b6b569dd4eb472de47494c7a39e34f47ee3e92ad4cbf92308f98997da366fe094a0e58eb6b93eab0ee956fff86d + languageName: node + linkType: hard + "require-directory@npm:^2.1.1": version: 2.1.1 resolution: "require-directory@npm:2.1.1" @@ -4389,24 +4277,6 @@ __metadata: languageName: node linkType: hard -"rust.aws-cdk-lambda@npm:^1.2.1": - version: 1.2.1 - resolution: "rust.aws-cdk-lambda@npm:1.2.1" - dependencies: - "@ziglang/cli": ^0.0.11 - aws-cdk-lib: ^2.* - constructs: ^10.* - toml: ~3.0.0 - peerDependencies: - aws-cdk-lib: ^2.* - constructs: ^10.* - dependenciesMeta: - "@ziglang/cli": - optional: true - checksum: d177d1fb5b05f51bd6103e21d213919470afceeb841eac5279e203b1ef927e73a9f59f3f694e67681f2115044cc1ce829909c920f11e7065c878bc7c53b8ad53 - languageName: node - linkType: hard - "safer-buffer@npm:>= 2.1.2 < 3.0.0": version: 2.1.2 resolution: "safer-buffer@npm:2.1.2" @@ -4469,13 +4339,6 @@ __metadata: languageName: node linkType: hard -"slash@npm:^4.0.0": - version: 4.0.0 - resolution: "slash@npm:4.0.0" - checksum: da8e4af73712253acd21b7853b7e0dbba776b786e82b010a5bfc8b5051a1db38ed8aba8e1e8f400dd2c9f373be91eb1c42b66e91abb407ff42b10feece5e1d2d - languageName: node - linkType: hard - "slice-ansi@npm:^4.0.0": version: 4.0.0 resolution: "slice-ansi@npm:4.0.0" @@ -4542,15 +4405,6 @@ __metadata: languageName: node linkType: hard -"split@npm:0.3": - version: 0.3.3 - resolution: "split@npm:0.3.3" - dependencies: - through: 2 - checksum: 2e076634c9637cfdc54ab4387b6a243b8c33b360874a25adf6f327a5647f07cb3bf1c755d515248eb3afee4e382278d01f62c62d87263c118f28065b86f74f02 - languageName: node - linkType: hard - "sprintf-js@npm:~1.0.2": version: 1.0.3 resolution: "sprintf-js@npm:1.0.3" @@ -4576,15 +4430,6 @@ __metadata: languageName: node linkType: hard -"stream-combiner@npm:~0.0.4": - version: 0.0.4 - resolution: "stream-combiner@npm:0.0.4" - dependencies: - duplexer: ~0.1.1 - checksum: 844b622cfe8b9de45a6007404f613b60aaf85200ab9862299066204242f89a7c8033b1c356c998aa6cfc630f6cd9eba119ec1c6dc1f93e245982be4a847aee7d - languageName: node - linkType: hard - "string-length@npm:^4.0.1": version: 4.0.2 resolution: "string-length@npm:4.0.2" @@ -4745,13 +4590,6 @@ __metadata: languageName: node linkType: hard -"through@npm:2, through@npm:~2.3, through@npm:~2.3.1": - version: 2.3.8 - resolution: "through@npm:2.3.8" - checksum: a38c3e059853c494af95d50c072b83f8b676a9ba2818dcc5b108ef252230735c54e0185437618596c790bbba8fcdaef5b290405981ffa09dce67b1f1bf190cbd - languageName: node - linkType: hard - "tmpl@npm:1.0.5": version: 1.0.5 resolution: "tmpl@npm:1.0.5" @@ -4775,13 +4613,6 @@ __metadata: languageName: node linkType: hard -"toml@npm:~3.0.0": - version: 3.0.0 - resolution: "toml@npm:3.0.0" - checksum: 5d7f1d8413ad7780e9bdecce8ea4c3f5130dd53b0a4f2e90b93340979a137739879d7b9ce2ce05c938b8cc828897fe9e95085197342a1377dd8850bf5125f15f - languageName: node - linkType: hard - "ts-api-utils@npm:^1.0.1": version: 1.0.3 resolution: "ts-api-utils@npm:1.0.3" @@ -5010,14 +4841,7 @@ __metadata: languageName: node linkType: hard -"web-streams-polyfill@npm:^3.0.3": - version: 3.2.1 - resolution: "web-streams-polyfill@npm:3.2.1" - checksum: b119c78574b6d65935e35098c2afdcd752b84268e18746606af149e3c424e15621b6f1ff0b42b2676dc012fc4f0d313f964b41a4b5031e525faa03997457da02 - languageName: node - linkType: hard - -"which@npm:^2.0.1, which@npm:^2.0.2": +"which@npm:^2.0.1": version: 2.0.2 resolution: "which@npm:2.0.2" dependencies: @@ -5085,6 +4909,15 @@ __metadata: languageName: node linkType: hard +"xregexp@npm:^5.1.1": + version: 5.1.1 + resolution: "xregexp@npm:5.1.1" + dependencies: + "@babel/runtime-corejs3": ^7.16.5 + checksum: b7fee45db0daacc68d8f747c9d3865af6b1135866c6dbd72980fc7d61138310018c397effdc0ec1f50ced71b9437ccb70c27818f4470e28e48dc59ea6e3900a0 + languageName: node + linkType: hard + "y18n@npm:^5.0.5": version: 5.0.8 resolution: "y18n@npm:5.0.8" @@ -5113,13 +4946,6 @@ __metadata: languageName: node linkType: hard -"yaml@npm:^2.1.1": - version: 2.3.4 - resolution: "yaml@npm:2.3.4" - checksum: e6d1dae1c6383bcc8ba11796eef3b8c02d5082911c6723efeeb5ba50fc8e881df18d645e64de68e421b577296000bea9c75d6d9097c2f6699da3ae0406c030d8 - languageName: node - linkType: hard - "yargs-parser@npm:^21.0.1, yargs-parser@npm:^21.1.1": version: 21.1.1 resolution: "yargs-parser@npm:21.1.1" @@ -5155,27 +4981,3 @@ __metadata: checksum: f77b3d8d00310def622123df93d4ee654fc6a0096182af8bd60679ddcdfb3474c56c6c7190817c84a2785648cdee9d721c0154eb45698c62176c322fb46fc700 languageName: node linkType: hard - -"zx@npm:^6.0.7": - version: 6.2.5 - resolution: "zx@npm:6.2.5" - dependencies: - "@types/fs-extra": ^9.0.13 - "@types/minimist": ^1.2.2 - "@types/node": ^17.0 - "@types/ps-tree": ^1.1.2 - "@types/which": ^2.0.1 - chalk: ^5.0.1 - fs-extra: ^10.1.0 - globby: ^13.1.1 - ignore: ^5.2.0 - minimist: ^1.2.6 - node-fetch: ^3.2.5 - ps-tree: ^1.2.0 - which: ^2.0.2 - yaml: ^2.1.1 - bin: - zx: build/cli.js - checksum: 03374884ca4fb68fd80a1a692063e9d08d81208b4a3a8974106bd8090058c5800747fbd22ae96be58c8a98f40ef0642a3925de8d696b93b0fb5b8ece7d6f93e6 - languageName: node - linkType: hard From d569b47c216af62521bd6f7bf3628a3f86129d95 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 11 Mar 2024 16:54:52 +1100 Subject: [PATCH 2/3] refactor: remove query files, update cdk to use database url and separate compile directory --- lib/workload/stateless/filemanager/.gitignore | 1 + ...edb7a794c72051d12c8c3b57d853b37a8392e.json | 20 --- ...92fe9ea92767a8274859de371f4e954e554b8.json | 116 -------------- ...d933fe91e5f9a12f3a2d1807bd3496d7ee134.json | 147 ------------------ ...f43b6f0d78fd3a31ae2517ba889ed50496dff.json | 64 -------- ...df247af518d4fafe42761f1488d1927d616c6.json | 65 -------- ...91338f4623a239c677301c9d13bcbd5e65e78.json | 14 -- ...2997148a6768d5d19f5176257c1ebb5c09fc2.json | 117 -------------- lib/workload/stateless/filemanager/Makefile | 4 - lib/workload/stateless/filemanager/README.md | 38 ----- .../deploy/constructs/functions/function.ts | 20 ++- yarn.lock | 6 +- 12 files changed, 22 insertions(+), 590 deletions(-) delete mode 100644 lib/workload/stateless/filemanager/.sqlx/query-3fe88592da137c9706a5b541ef8edb7a794c72051d12c8c3b57d853b37a8392e.json delete mode 100644 lib/workload/stateless/filemanager/.sqlx/query-63da484d18ed2571639cfddadf992fe9ea92767a8274859de371f4e954e554b8.json delete mode 100644 lib/workload/stateless/filemanager/.sqlx/query-9fd0e02337484c56ffc4c72f7cbd933fe91e5f9a12f3a2d1807bd3496d7ee134.json delete mode 100644 lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json delete mode 100644 lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json delete mode 100644 lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json delete mode 100644 lib/workload/stateless/filemanager/.sqlx/query-df0e4c422260213654668cd5dc62997148a6768d5d19f5176257c1ebb5c09fc2.json diff --git a/lib/workload/stateless/filemanager/.gitignore b/lib/workload/stateless/filemanager/.gitignore index 98331b258..56d0a8cc8 100644 --- a/lib/workload/stateless/filemanager/.gitignore +++ b/lib/workload/stateless/filemanager/.gitignore @@ -2,3 +2,4 @@ target/ *.swp /volume/ .build +target-cdk-docker-bundling/ \ No newline at end of file diff --git a/lib/workload/stateless/filemanager/.sqlx/query-3fe88592da137c9706a5b541ef8edb7a794c72051d12c8c3b57d853b37a8392e.json b/lib/workload/stateless/filemanager/.sqlx/query-3fe88592da137c9706a5b541ef8edb7a794c72051d12c8c3b57d853b37a8392e.json deleted file mode 100644 index b0d40cb2a..000000000 --- a/lib/workload/stateless/filemanager/.sqlx/query-3fe88592da137c9706a5b541ef8edb7a794c72051d12c8c3b57d853b37a8392e.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "select exists (select from information_schema.tables where table_name = 'object')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "3fe88592da137c9706a5b541ef8edb7a794c72051d12c8c3b57d853b37a8392e" -} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-63da484d18ed2571639cfddadf992fe9ea92767a8274859de371f4e954e554b8.json b/lib/workload/stateless/filemanager/.sqlx/query-63da484d18ed2571639cfddadf992fe9ea92767a8274859de371f4e954e554b8.json deleted file mode 100644 index 6ca23412b..000000000 --- a/lib/workload/stateless/filemanager/.sqlx/query-63da484d18ed2571639cfddadf992fe9ea92767a8274859de371f4e954e554b8.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "select s3_object_id as \"s3_object_id!\",\n object_id as \"object_id!\",\n bucket,\n key,\n created_date,\n deleted_date,\n last_modified_date,\n e_tag,\n storage_class as \"storage_class: StorageClass\",\n version_id,\n created_sequencer,\n deleted_sequencer,\n number_reordered,\n number_duplicate_events from s3_object", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "s3_object_id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "object_id!", - "type_info": "Uuid" - }, - { - "ordinal": 2, - "name": "bucket", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "key", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "created_date", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "deleted_date", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "last_modified_date", - "type_info": "Timestamptz" - }, - { - "ordinal": 7, - "name": "e_tag", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "storage_class: StorageClass", - "type_info": { - "Custom": { - "name": "storage_class", - "kind": { - "Enum": [ - "DeepArchive", - "Glacier", - "GlacierIr", - "IntelligentTiering", - "OnezoneIa", - "Outposts", - "ReducedRedundancy", - "Snow", - "Standard", - "StandardIa" - ] - } - } - } - }, - { - "ordinal": 9, - "name": "version_id", - "type_info": "Text" - }, - { - "ordinal": 10, - "name": "created_sequencer", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "deleted_sequencer", - "type_info": "Text" - }, - { - "ordinal": 12, - "name": "number_reordered", - "type_info": "Int4" - }, - { - "ordinal": 13, - "name": "number_duplicate_events", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false - ] - }, - "hash": "63da484d18ed2571639cfddadf992fe9ea92767a8274859de371f4e954e554b8" -} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-9fd0e02337484c56ffc4c72f7cbd933fe91e5f9a12f3a2d1807bd3496d7ee134.json b/lib/workload/stateless/filemanager/.sqlx/query-9fd0e02337484c56ffc4c72f7cbd933fe91e5f9a12f3a2d1807bd3496d7ee134.json deleted file mode 100644 index a837b73cf..000000000 --- a/lib/workload/stateless/filemanager/.sqlx/query-9fd0e02337484c56ffc4c72f7cbd933fe91e5f9a12f3a2d1807bd3496d7ee134.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "-- Update the matching s3_objects which should be re-ordered based on the created event. Returns the\n-- data associated with the event before the update, if an update occurred.\n\n-- First, unnest the input parameters into a query.\nwith input as (\n select\n *\n from unnest(\n $1::uuid[],\n $2::text[],\n $3::text[],\n $4::timestamptz[],\n $5::integer[],\n $6::text[],\n $7::timestamptz[],\n $8::text[],\n $9::storage_class[],\n $10::text[],\n $11::text[]\n ) as input (\n s3_object_id,\n bucket,\n key,\n created_date,\n size,\n checksum,\n last_modified_date,\n e_tag,\n storage_class,\n version_id,\n created_sequencer\n )\n),\n-- Then, select the objects that need to be updated.\ncurrent_objects as (\n select\n s3_object.*,\n input.s3_object_id as input_id,\n input.bucket as input_bucket,\n input.key as input_key,\n input.version_id as input_version_id,\n input.created_sequencer as input_created_sequencer,\n input.created_date as input_created_date,\n input.size as input_size,\n input.checksum as input_checksum,\n input.last_modified_date as input_last_modified_date,\n input.e_tag as input_e_tag,\n input.storage_class as input_storage_class\n from s3_object\n -- Grab the relevant values to update with.\n join input on\n input.bucket = s3_object.bucket and\n input.key = s3_object.key and\n input.version_id is not distinct from s3_object.version_id\n -- Lock this pre-emptively for the update.\n for update\n),\n-- And filter them to the objects that need to be updated.\nobjects_to_update as (\n select\n *\n from current_objects\n where\n -- Check the sequencer condition. We only update if there is a created\n -- sequencer that is closer to the deleted sequencer.\n current_objects.deleted_sequencer > current_objects.input_created_sequencer and\n (\n -- Updating a null sequencer doesn't cause the event to be reprocessed.\n current_objects.created_sequencer is null or\n -- If a sequencer already exists this event should be reprocessed because this\n -- sequencer could belong to another object.\n current_objects.created_sequencer < current_objects.input_created_sequencer\n )\n -- And there should not be any objects with a created sequencer that is the same as the input created\n -- sequencer because this is a duplicate event that would cause a constraint error in the update.\n and current_objects.input_created_sequencer not in (\n select created_sequencer from current_objects where created_sequencer is not null\n )\n),\n-- Finally, update the required objects.\nupdate as (\n update s3_object\n set created_sequencer = objects_to_update.input_created_sequencer,\n created_date = objects_to_update.input_created_date,\n size = coalesce(objects_to_update.input_size, objects_to_update.size),\n checksum = coalesce(objects_to_update.input_checksum, objects_to_update.checksum),\n last_modified_date = coalesce(objects_to_update.input_last_modified_date, objects_to_update.last_modified_date),\n e_tag = coalesce(objects_to_update.e_tag, objects_to_update.e_tag),\n storage_class = objects_to_update.storage_class,\n number_reordered = s3_object.number_reordered +\n -- Note the asymmetry between this and the reorder for deleted query.\n case when objects_to_update.deleted_sequencer is not null or objects_to_update.created_sequencer is not null then\n 1\n else\n 0\n end\n from objects_to_update\n where s3_object.s3_object_id = objects_to_update.s3_object_id\n)\n-- Return the old values because these need to be reprocessed.\nselect\n -- Note, this is the passed through value from the input in order to identify this event later.\n input_id as \"s3_object_id!\",\n bucket,\n key,\n created_date as event_time,\n last_modified_date,\n e_tag,\n storage_class as \"storage_class?: StorageClass\",\n version_id,\n created_sequencer as sequencer,\n number_reordered,\n number_duplicate_events,\n size,\n -- This is used to simplify re-constructing the FlatS3EventMessages in the Lambda. I.e. this update detected an\n -- out of order created event, so return a created event back.\n 'Created' as \"event_type!: EventType\"\nfrom objects_to_update;\n", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "s3_object_id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "bucket", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "key", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "event_time", - "type_info": "Timestamptz" - }, - { - "ordinal": 4, - "name": "last_modified_date", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "e_tag", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "storage_class?: StorageClass", - "type_info": { - "Custom": { - "name": "storage_class", - "kind": { - "Enum": [ - "DeepArchive", - "Glacier", - "GlacierIr", - "IntelligentTiering", - "OnezoneIa", - "Outposts", - "ReducedRedundancy", - "Snow", - "Standard", - "StandardIa" - ] - } - } - } - }, - { - "ordinal": 7, - "name": "version_id", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "sequencer", - "type_info": "Text" - }, - { - "ordinal": 9, - "name": "number_reordered", - "type_info": "Int4" - }, - { - "ordinal": 10, - "name": "number_duplicate_events", - "type_info": "Int4" - }, - { - "ordinal": 11, - "name": "size", - "type_info": "Int4" - }, - { - "ordinal": 12, - "name": "event_type!: EventType", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "UuidArray", - "TextArray", - "TextArray", - "TimestamptzArray", - "Int4Array", - "TextArray", - "TimestamptzArray", - "TextArray", - { - "Custom": { - "name": "_storage_class", - "kind": { - "Array": { - "Custom": { - "name": "storage_class", - "kind": { - "Enum": [ - "DeepArchive", - "Glacier", - "GlacierIr", - "IntelligentTiering", - "OnezoneIa", - "Outposts", - "ReducedRedundancy", - "Snow", - "Standard", - "StandardIa" - ] - } - } - } - } - } - }, - "TextArray", - "TextArray" - ] - }, - "nullable": [ - null, - false, - false, - true, - true, - true, - true, - true, - true, - false, - false, - true, - null - ] - }, - "hash": "9fd0e02337484c56ffc4c72f7cbd933fe91e5f9a12f3a2d1807bd3496d7ee134" -} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json b/lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json deleted file mode 100644 index 81955501e..000000000 --- a/lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "-- Bulk insert of s3 objects.\ninsert into s3_object (\n s3_object_id,\n object_id,\n bucket,\n key,\n created_date,\n size,\n checksum,\n last_modified_date,\n e_tag,\n storage_class,\n version_id,\n created_sequencer\n)\nvalues (\n unnest($1::uuid[]),\n unnest($2::uuid[]),\n unnest($3::text[]),\n unnest($4::text[]),\n unnest($5::timestamptz[]),\n unnest($6::integer[]),\n unnest($7::text[]),\n unnest($8::timestamptz[]),\n unnest($9::text[]),\n unnest($10::storage_class[]),\n unnest($11::text[]),\n unnest($12::text[])\n) on conflict on constraint created_sequencer_unique do update\n set number_duplicate_events = s3_object.number_duplicate_events + 1\n returning object_id, number_duplicate_events;\n", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "object_id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "number_duplicate_events", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "UuidArray", - "UuidArray", - "TextArray", - "TextArray", - "TimestamptzArray", - "Int4Array", - "TextArray", - "TimestamptzArray", - "TextArray", - { - "Custom": { - "name": "_storage_class", - "kind": { - "Array": { - "Custom": { - "name": "storage_class", - "kind": { - "Enum": [ - "DeepArchive", - "Glacier", - "GlacierIr", - "IntelligentTiering", - "OnezoneIa", - "Outposts", - "ReducedRedundancy", - "Snow", - "Standard", - "StandardIa" - ] - } - } - } - } - } - }, - "TextArray", - "TextArray" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff" -} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json b/lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json deleted file mode 100644 index ee9c54553..000000000 --- a/lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "-- Bulk insert of s3 objects.\ninsert into s3_object (\n s3_object_id,\n object_id,\n bucket,\n key,\n deleted_date,\n size,\n checksum,\n last_modified_date,\n e_tag,\n storage_class,\n version_id,\n deleted_sequencer,\n number_reordered\n)\nvalues (\n unnest($1::uuid[]),\n unnest($2::uuid[]),\n unnest($3::text[]),\n unnest($4::text[]),\n unnest($5::timestamptz[]),\n unnest($6::integer[]),\n unnest($7::text[]),\n unnest($8::timestamptz[]),\n unnest($9::text[]),\n unnest($10::storage_class[]),\n unnest($11::text[]),\n unnest($12::text[]),\n unnest($13::integer[])\n) on conflict on constraint deleted_sequencer_unique do update\n set number_duplicate_events = s3_object.number_duplicate_events + 1\n returning object_id, number_duplicate_events;\n", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "object_id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "number_duplicate_events", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "UuidArray", - "UuidArray", - "TextArray", - "TextArray", - "TimestamptzArray", - "Int4Array", - "TextArray", - "TimestamptzArray", - "TextArray", - { - "Custom": { - "name": "_storage_class", - "kind": { - "Array": { - "Custom": { - "name": "storage_class", - "kind": { - "Enum": [ - "DeepArchive", - "Glacier", - "GlacierIr", - "IntelligentTiering", - "OnezoneIa", - "Outposts", - "ReducedRedundancy", - "Snow", - "Standard", - "StandardIa" - ] - } - } - } - } - } - }, - "TextArray", - "TextArray", - "Int4Array" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6" -} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json b/lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json deleted file mode 100644 index 6a44fe521..000000000 --- a/lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "-- Bulk insert of objects\ninsert into object (object_id)\nvalues (\n unnest($1::uuid[])\n);\n", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78" -} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-df0e4c422260213654668cd5dc62997148a6768d5d19f5176257c1ebb5c09fc2.json b/lib/workload/stateless/filemanager/.sqlx/query-df0e4c422260213654668cd5dc62997148a6768d5d19f5176257c1ebb5c09fc2.json deleted file mode 100644 index d9b45ac03..000000000 --- a/lib/workload/stateless/filemanager/.sqlx/query-df0e4c422260213654668cd5dc62997148a6768d5d19f5176257c1ebb5c09fc2.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "-- Update the matching s3_objects which should be re-ordered based on the deleted event. Returns the\n-- data associated with the event before the update, if an update occurred.\n\n-- First, unnest the input parameters into a query.\nwith input as (\n select\n *\n from unnest(\n $1::uuid[],\n $2::text[],\n $3::text[],\n $4::timestamptz[],\n $5::text[],\n $6::text[]\n ) as input (\n s3_object_id,\n bucket,\n key,\n deleted_date,\n version_id,\n deleted_sequencer\n )\n),\n-- Then, select the objects that match the bucket, key and version_id\ncurrent_objects as (\n select\n s3_object.*,\n input.s3_object_id as input_id,\n input.bucket as input_bucket,\n input.key as input_key,\n input.version_id as input_version_id,\n input.deleted_sequencer as input_deleted_sequencer,\n input.deleted_date as input_deleted_date\n from s3_object\n -- Grab the relevant values to update with.\n join input on\n input.bucket = s3_object.bucket and\n input.key = s3_object.key and\n input.version_id is not distinct from s3_object.version_id\n -- Lock this pre-emptively for the update.\n for update\n),\n-- And filter them to the objects that need to be updated.\nobjects_to_update as (\n select\n *\n from current_objects\n where\n -- Check the sequencer condition. We only update if there is a deleted\n -- sequencer that is closer to the created sequencer.\n current_objects.created_sequencer < current_objects.input_deleted_sequencer and\n (\n -- Updating a null sequencer doesn't cause the event to be reprocessed.\n current_objects.deleted_sequencer is null or\n -- If a sequencer already exists this event should be reprocessed because this\n -- sequencer would belong to another object.\n current_objects.deleted_sequencer > current_objects.input_deleted_sequencer\n )\n -- And there should not be any objects with a deleted sequencer that is the same as the input deleted\n -- sequencer because this is a duplicate event that would cause a constraint error in the update.\n and current_objects.input_deleted_sequencer not in (\n select deleted_sequencer from current_objects where deleted_sequencer is not null\n )\n),\n-- Finally, update the required objects.\nupdate as (\n update s3_object\n set deleted_sequencer = objects_to_update.input_deleted_sequencer,\n deleted_date = objects_to_update.input_deleted_date,\n number_reordered = s3_object.number_reordered +\n case when objects_to_update.deleted_sequencer is null then 0 else 1 end\n from objects_to_update\n where s3_object.s3_object_id = objects_to_update.s3_object_id\n)\n-- Return the old values because these need to be reprocessed.\nselect\n -- Note, this is the passed through value from the input in order to identify this event later.\n input_id as \"s3_object_id!\",\n bucket,\n key,\n deleted_date as event_time,\n last_modified_date,\n e_tag,\n storage_class as \"storage_class?: StorageClass\",\n version_id,\n deleted_sequencer as sequencer,\n number_reordered,\n number_duplicate_events,\n size,\n -- This is used to simplify re-constructing the FlatS3EventMessages in the Lambda. I.e. this update detected an\n -- out of order deleted event, so return a deleted event back.\n 'Deleted' as \"event_type!: EventType\"\nfrom objects_to_update;\n", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "s3_object_id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "bucket", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "key", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "event_time", - "type_info": "Timestamptz" - }, - { - "ordinal": 4, - "name": "last_modified_date", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "e_tag", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "storage_class?: StorageClass", - "type_info": { - "Custom": { - "name": "storage_class", - "kind": { - "Enum": [ - "DeepArchive", - "Glacier", - "GlacierIr", - "IntelligentTiering", - "OnezoneIa", - "Outposts", - "ReducedRedundancy", - "Snow", - "Standard", - "StandardIa" - ] - } - } - } - }, - { - "ordinal": 7, - "name": "version_id", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "sequencer", - "type_info": "Text" - }, - { - "ordinal": 9, - "name": "number_reordered", - "type_info": "Int4" - }, - { - "ordinal": 10, - "name": "number_duplicate_events", - "type_info": "Int4" - }, - { - "ordinal": 11, - "name": "size", - "type_info": "Int4" - }, - { - "ordinal": 12, - "name": "event_type!: EventType", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "UuidArray", - "TextArray", - "TextArray", - "TimestamptzArray", - "TextArray", - "TextArray" - ] - }, - "nullable": [ - null, - false, - false, - true, - true, - true, - true, - true, - true, - false, - false, - true, - null - ] - }, - "hash": "df0e4c422260213654668cd5dc62997148a6768d5d19f5176257c1ebb5c09fc2" -} diff --git a/lib/workload/stateless/filemanager/Makefile b/lib/workload/stateless/filemanager/Makefile index b1cecf59b..d64747120 100644 --- a/lib/workload/stateless/filemanager/Makefile +++ b/lib/workload/stateless/filemanager/Makefile @@ -43,10 +43,6 @@ clean: docker-clean ## Database related targets psql: @docker exec -it filemanager_db psql filemanager -U filemanager -prepare: - cargo sqlx prepare --workspace -- --all-features --all-targets -prepare-check: - cargo sqlx prepare --workspace --check -- --all-features --all-targets ## Help text help: diff --git a/lib/workload/stateless/filemanager/README.md b/lib/workload/stateless/filemanager/README.md index 656f8b7b6..d3a8f051d 100644 --- a/lib/workload/stateless/filemanager/README.md +++ b/lib/workload/stateless/filemanager/README.md @@ -42,12 +42,6 @@ or cargo install sccache && export RUSTC_WRAPPER=`which sccache` ``` -The sqlx-cli is also required for compiling code without a database running: - -```sh -cargo install sqlx-cli -``` - cargo-watch can be used to recompile files as they change: ```sh @@ -88,38 +82,6 @@ Which runs `cargo test -- --ignored`. ## Database -### Offline compilation - -In order to compile code without a local database running, queries should be prepared using `sqlx-cli`: - -```sh -make prepare -``` - -which runs: - -```sh -cargo sqlx prepare --workspace --all-features --all-targets -``` - -This creates a `.sqlx` file in the workspace directory which contains the pre-compiled query information. -This is useful for deploying the orcabus code without a dependency on the filemanager's Docker compose, and it also -ensures that cdk tests which synthesize the filemanager stack without additional modifications. - -In order to check if this file is out of date, run: - -```sh -make prepare-check -``` - -which runs: - -```sh -cargo sqlx prepare --workspace --check --all-features --all-targets -``` - -### Database connection - To connect to the local postgres database, run: ```bash diff --git a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts index 8e6bcbb54..61c59abb3 100644 --- a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts @@ -6,6 +6,7 @@ import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk- import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; import { RustFunction } from 'cargo-lambda-cdk'; import path from 'path'; +import { exec } from 'cargo-lambda-cdk/lib/util'; /** * Properties for the database. @@ -92,11 +93,26 @@ export class Function extends Construct { `/` + `${props.databaseSecret.secretValueFromJson('dbname').unsafeUnwrap()}`; + const manifestPath = path.join(__dirname, '..', '..', '..'); + // This starts the container running postgres in order to compile queries using sqlx. + // It needs to be executed outside `beforeBundling`, because `beforeBundling` runs inside + // the container context, and docker compose needs to run outside of this context. + exec('make', ['docker-postgres'], { cwd: manifestPath, shell: true }); + this._function = new RustFunction(this, 'RustFunction', { - manifestPath: path.join(__dirname, '..', '..', '..'), + manifestPath, binaryName: props.package, bundling: { - environment: props.buildEnvironment + environment: { + ...props.buildEnvironment, + // Avoid permission issues by creating another target directory. + CARGO_TARGET_DIR: "target-cdk-docker-bundling", + // The bundling container needs to be able to connect to the container running postgres. + DATABASE_URL: "postgresql://filemanager:filemanager@host.docker.internal:4321/filemanager", // pragma: allowlist secret + }, + // This uses docker only to compile the binary. I.e. the Lambda function still runs natively and + // is not dockerized. + forcedDockerBundling: true, }, memorySize: 128, timeout: Duration.seconds(28), diff --git a/yarn.lock b/yarn.lock index 2e3863022..53870ff35 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1500,8 +1500,8 @@ __metadata: linkType: hard "aws-cdk@npm:^2.126.0": - version: 2.126.0 - resolution: "aws-cdk@npm:2.126.0" + version: 2.132.0 + resolution: "aws-cdk@npm:2.132.0" dependencies: fsevents: 2.3.2 dependenciesMeta: @@ -1509,7 +1509,7 @@ __metadata: optional: true bin: cdk: bin/cdk - checksum: 93c82ccb949059ef07e88a2952df7fe025dc60a8d3e687c26497660de5ac2433bf719f0dbff415fa41eef2c1a64caf37a77110d3d2a4c6de9f122dcd1f293506 + checksum: 17d653260f02825dc4e74d7af3b515928438e07a7488344adbbe98133fc9c58fdbd5c8d6ad20c900cf7f11e9a8e2240558d75444c706900cdb095f41eec77918 languageName: node linkType: hard From eb3654e6992858c9cfc338b8b6b17427426189ab Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 14 Mar 2024 12:52:07 +1100 Subject: [PATCH 3/3] docs(filemanager): add note about permission issues --- lib/workload/stateless/filemanager/.gitignore | 2 +- lib/workload/stateless/filemanager/deploy/README.md | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/workload/stateless/filemanager/.gitignore b/lib/workload/stateless/filemanager/.gitignore index 56d0a8cc8..7709b0d30 100644 --- a/lib/workload/stateless/filemanager/.gitignore +++ b/lib/workload/stateless/filemanager/.gitignore @@ -2,4 +2,4 @@ target/ *.swp /volume/ .build -target-cdk-docker-bundling/ \ No newline at end of file +target-cdk-docker-bundling/ diff --git a/lib/workload/stateless/filemanager/deploy/README.md b/lib/workload/stateless/filemanager/deploy/README.md index caea31a55..cab0e3fa1 100644 --- a/lib/workload/stateless/filemanager/deploy/README.md +++ b/lib/workload/stateless/filemanager/deploy/README.md @@ -21,3 +21,11 @@ The filemanager expects a dedicated database within the shared database cluster, Initially, the filemanager-migrate-lambda function is deployed to perform database table migrations using the cdk_resource_invoke.ts construct. Then, the other Lambda functions are deployed normally within the filemanager construct. + +### Building + +Note, the `RustFunction` compiles code using `cargo-lambda` running inside a Docker container, and produces a Lambda +function which runs natively on AWS (i.e. not a dockerized Lambda function). This makes it simpler for consumers of +the Filemanager CDK to build the project. There is a small downside in that the compiled code is stored under a +`target-cdk-docker-bundling` directory which is owned by Docker, so there may be some permission issues if trying to +clean this directory. \ No newline at end of file