From 1d19dd108dee920afbfcc74840004fa94d6ed884 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 14 Dec 2023 15:59:01 +1100 Subject: [PATCH] Filemanager: fix ingestion/deployment (#74) * Make deployment idempotent and callable through cargo watch to avoid any manual intervention while editing code * Give up on LocalStack, excise it from all the FM codebase * Remove pathstyle * Remove CfnBucket hack meant to appease localstack, not needed anumore * Successful deployment, Aurora Serverless V2 versions lag behind regular PostgreSQL ones, see: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Concepts.Aurora_Fea_Regions_DB-eng.Feature.ServerlessV2.html#Concepts.Aurora_Fea_Regions_DB-eng.Feature.ServerlessV2.apg --- docs/developer/FM.md | 11 +- lib/workload/stateful/filemanager/Cargo.lock | 682 +++++------------- lib/workload/stateful/filemanager/README.md | 42 +- .../stateful/filemanager/deploy/README.md | 6 +- .../stateful/filemanager/deploy/cdk.json | 2 +- .../filemanager/deploy/package-lock.json | 137 ++-- .../stateful/filemanager/deploy/package.json | 13 +- .../deploy/stack/filemanager_stack.ts | 75 +- .../stack.ts} | 9 +- .../stateful/filemanager/docker-compose.yml | 24 +- .../filemanager-http-lambda/src/main.rs | 4 +- .../filemanager-ingest-lambda/Cargo.toml | 10 +- .../filemanager/filemanager/Cargo.toml | 6 +- .../filemanager/src/events/s3/collect.rs | 6 +- .../filemanager/src/events/s3/mod.rs | 2 +- .../filemanager/src/events/s3/s3_client.rs | 29 +- .../stateful/filemanager/scripts/deploy.sh | 10 +- .../stateful/filemanager/scripts/logs.sh | 3 +- .../stateful/filemanager/scripts/watch.sh | 3 + 19 files changed, 391 insertions(+), 683 deletions(-) rename lib/workload/stateful/filemanager/deploy/{localstack/filemanager_localstack.ts => stack/stack.ts} (83%) create mode 100755 lib/workload/stateful/filemanager/scripts/watch.sh diff --git a/docs/developer/FM.md b/docs/developer/FM.md index 3bdfcd05c..0be856401 100644 --- a/docs/developer/FM.md +++ b/docs/developer/FM.md @@ -4,13 +4,6 @@ Namespace: orcabus.fm ``` -## How to run FM locally +### Quickstart -### Ready Check - -``` -conda activate orcabus -make up -./lib/workload/stateless/filemanager/scripts/localstack-s3-events-to-sqs.sh # sets up s3 events simulation -cd ./lib/workload/stateless/filemanager && cargo run -``` +Refer to README.md under `lib/stateful/filemanager/deploy/README.md` \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/Cargo.lock b/lib/workload/stateful/filemanager/Cargo.lock index 801bcc156..f60c8b519 100644 --- a/lib/workload/stateful/filemanager/Cargo.lock +++ b/lib/workload/stateful/filemanager/Cargo.lock @@ -109,53 +109,23 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6b3804dca60326e07205179847f17a4fce45af3a1106939177ad41ac08a6de" -dependencies = [ - "aws-credential-types 0.56.1", - "aws-http 0.56.1", - "aws-sdk-sso 0.30.0", - "aws-sdk-sts 0.30.0", - "aws-smithy-async 0.56.1", - "aws-smithy-client", - "aws-smithy-http 0.56.1", - "aws-smithy-http-tower", - "aws-smithy-json 0.56.1", - "aws-smithy-types 0.56.1", - "aws-types 0.56.1", - "bytes", - "fastrand", - "hex", - "http 0.2.9", - "hyper 0.14.27", - "ring 0.16.20", - "time", - "tokio", - "tower", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-config" -version = "0.100.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14044e9a7e7ae811d99e71d35e3698113ec27eb618445863b611c60811574cc6" +checksum = "80c950a809d39bc9480207cb1cfc879ace88ea7e3a4392a8e9999e45d6e5692e" dependencies = [ - "aws-credential-types 0.58.0", - "aws-http 0.58.0", - "aws-runtime 0.58.0", - "aws-sdk-sso 0.37.0", + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-sdk-sso", "aws-sdk-ssooidc", - "aws-sdk-sts 0.37.0", - "aws-smithy-async 0.100.0", - "aws-smithy-http 0.58.0", - "aws-smithy-json 0.58.0", - "aws-smithy-runtime 0.58.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", - "aws-types 0.100.0", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", "bytes", "fastrand", "hex", @@ -170,58 +140,25 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a66ac8ef5fa9cf01c2d999f39d16812e90ec1467bd382cbbb74ba23ea86201" -dependencies = [ - "aws-smithy-async 0.56.1", - "aws-smithy-types 0.56.1", - "fastrand", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "0.58.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1eca93b9b017fab7c53a2d1518efc7f03de631e32f420298440acbfef146d21" +checksum = "8c1317e1a3514b103cf7d5828bbab3b4d30f56bd22d684f8568bc51b6cfbbb1c" dependencies = [ - "aws-smithy-async 0.100.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", "zeroize", ] [[package]] name = "aws-http" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e626370f9ba806ae4c439e49675fd871f5767b093075cdf4fef16cac42ba900" -dependencies = [ - "aws-credential-types 0.56.1", - "aws-smithy-http 0.56.1", - "aws-smithy-types 0.56.1", - "aws-types 0.56.1", - "bytes", - "http 0.2.9", - "http-body 0.4.5", - "lazy_static", - "percent-encoding", - "pin-project-lite", - "tracing", -] - -[[package]] -name = "aws-http" -version = "0.58.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31eca8162dedcccb0ff256a878689510763b9ffd5a2c0d51ee0199dc2ec593a" +checksum = "361c4310fdce94328cc2d1ca0c8a48c13f43009c61d3367585685a50ca8c66b6" dependencies = [ - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", - "aws-types 0.100.0", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", "bytes", "http 0.2.9", "http-body 0.4.5", @@ -231,40 +168,19 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ac5cf0ff19c1bca0cea7932e11b239d1025a45696a4f44f72ea86e2b8bdd07" -dependencies = [ - "aws-credential-types 0.56.1", - "aws-http 0.56.1", - "aws-sigv4 0.56.1", - "aws-smithy-async 0.56.1", - "aws-smithy-http 0.56.1", - "aws-smithy-runtime-api 0.56.1", - "aws-smithy-types 0.56.1", - "aws-types 0.56.1", - "fastrand", - "http 0.2.9", - "percent-encoding", - "tracing", - "uuid", -] - -[[package]] -name = "aws-runtime" -version = "0.58.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb1cd6bf27e01014e52b5b181d1e700841abd8adccee55055b1a86757f9afbb" +checksum = "1ed7ef604a15fd0d4d9e43701295161ea6b504b63c44990ead352afea2bc15e9" dependencies = [ - "aws-credential-types 0.58.0", - "aws-http 0.58.0", - "aws-sigv4 0.58.0", - "aws-smithy-async 0.100.0", + "aws-credential-types", + "aws-http", + "aws-sigv4", + "aws-smithy-async", "aws-smithy-eventstream", - "aws-smithy-http 0.58.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", - "aws-types 0.100.0", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", "fastrand", "http 0.2.9", "percent-encoding", @@ -274,24 +190,24 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "0.37.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a199d3c811f6dbe2aabc0f0e1c60bf06b50a2e0106be896f2d1d223147940cc9" +checksum = "9dcafc2fe52cc30b2d56685e2fa6a879ba50d79704594852112337a472ddbd24" dependencies = [ - "aws-credential-types 0.58.0", - "aws-http 0.58.0", - "aws-runtime 0.58.0", - "aws-sigv4 0.58.0", - "aws-smithy-async 0.100.0", + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", "aws-smithy-checksums", "aws-smithy-eventstream", - "aws-smithy-http 0.58.0", - "aws-smithy-json 0.58.0", - "aws-smithy-runtime 0.58.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", - "aws-smithy-xml 0.58.0", - "aws-types 0.100.0", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", "bytes", "http 0.2.9", "http-body 0.4.5", @@ -304,20 +220,20 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1b2110f7db7ae8c942324c479846b6a4a84bfcbe2548a2fc4081df467a704f" -dependencies = [ - "aws-credential-types 0.58.0", - "aws-http 0.58.0", - "aws-runtime 0.58.0", - "aws-smithy-async 0.100.0", - "aws-smithy-http 0.58.0", - "aws-smithy-json 0.58.0", - "aws-smithy-runtime 0.58.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", - "aws-types 0.100.0", +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5736d9255f65f36df4f0812665c33fa36042b96192e6bba843ef5fcc75187cd8" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", "bytes", "http 0.2.9", "regex", @@ -326,44 +242,20 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "903f888ff190e64f6f5c83fb0f8d54f9c20481f1dc26359bb8896f5d99908949" -dependencies = [ - "aws-credential-types 0.56.1", - "aws-http 0.56.1", - "aws-runtime 0.56.1", - "aws-smithy-async 0.56.1", - "aws-smithy-client", - "aws-smithy-http 0.56.1", - "aws-smithy-json 0.56.1", - "aws-smithy-runtime 0.56.1", - "aws-smithy-runtime-api 0.56.1", - "aws-smithy-types 0.56.1", - "aws-types 0.56.1", - "bytes", - "http 0.2.9", - "regex", - "tokio-stream", - "tracing", -] - -[[package]] -name = "aws-sdk-sso" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73693581f8ad829c05e52738bdc44a650881e6bb2601d06a30bd67201228536b" -dependencies = [ - "aws-credential-types 0.58.0", - "aws-http 0.58.0", - "aws-runtime 0.58.0", - "aws-smithy-async 0.100.0", - "aws-smithy-http 0.58.0", - "aws-smithy-json 0.58.0", - "aws-smithy-runtime 0.58.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", - "aws-types 0.100.0", +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0619ab97a5ca8982e7de073cdc66f93e5f6a1b05afc09e696bec1cb3607cd4df" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", "bytes", "http 0.2.9", "regex", @@ -372,20 +264,20 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "607c4f6ca50ff57ead33b8b5f89e01aefcd82a408b5b5d2a7c14385b9f93c88f" -dependencies = [ - "aws-credential-types 0.58.0", - "aws-http 0.58.0", - "aws-runtime 0.58.0", - "aws-smithy-async 0.100.0", - "aws-smithy-http 0.58.0", - "aws-smithy-json 0.58.0", - "aws-smithy-runtime 0.58.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", - "aws-types 0.100.0", +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04b9f5474cc0f35d829510b2ec8c21e352309b46bf9633c5a81fb9321e9b1c7" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", "bytes", "http 0.2.9", "regex", @@ -394,46 +286,22 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47ad6bf01afc00423d781d464220bf69fb6a674ad6629cbbcb06d88cdc2be82" -dependencies = [ - "aws-credential-types 0.56.1", - "aws-http 0.56.1", - "aws-runtime 0.56.1", - "aws-smithy-async 0.56.1", - "aws-smithy-client", - "aws-smithy-http 0.56.1", - "aws-smithy-json 0.56.1", - "aws-smithy-query 0.56.1", - "aws-smithy-runtime 0.56.1", - "aws-smithy-runtime-api 0.56.1", - "aws-smithy-types 0.56.1", - "aws-smithy-xml 0.56.1", - "aws-types 0.56.1", - "http 0.2.9", - "regex", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89064d7755ed95727f0ddb3e3aaab8191e37dd9c89fc38343a1604b9462fa583" -dependencies = [ - "aws-credential-types 0.58.0", - "aws-http 0.58.0", - "aws-runtime 0.58.0", - "aws-smithy-async 0.100.0", - "aws-smithy-http 0.58.0", - "aws-smithy-json 0.58.0", - "aws-smithy-query 0.58.0", - "aws-smithy-runtime 0.58.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", - "aws-smithy-xml 0.58.0", - "aws-types 0.100.0", +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "798c8d82203af9e15a8b406574e0b36da91dd6db533028b74676489a1bc8bc7d" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", "http 0.2.9", "regex", "tracing", @@ -441,46 +309,28 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b28f4910bb956b7ab320b62e98096402354eca976c587d1eeccd523d9bac03" -dependencies = [ - "aws-smithy-http 0.56.1", - "form_urlencoded", - "hex", - "hmac", - "http 0.2.9", - "once_cell", - "percent-encoding", - "regex", - "sha2", - "time", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "0.58.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2b2ae7efc433aac9c367e6038484e604df711b9ae7ad1224a1db5e3caaab0f1" +checksum = "380adcc8134ad8bbdfeb2ace7626a869914ee266322965276cbc54066186d236" dependencies = [ - "aws-credential-types 0.58.0", + "aws-credential-types", "aws-smithy-eventstream", - "aws-smithy-http 0.58.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", + "crypto-bigint 0.5.5", "form_urlencoded", "hex", "hmac", "http 0.2.9", - "num-bigint", "once_cell", "p256", "percent-encoding", "regex", "ring 0.17.5", "sha2", + "subtle", "time", "tracing", "zeroize", @@ -488,21 +338,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cdb73f85528b9d19c23a496034ac53703955a59323d581c06aa27b4e4e247af" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", - "tokio-stream", -] - -[[package]] -name = "aws-smithy-async" -version = "0.100.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324538ebc89f2414c9c92d878fe7b0538d1dea8ebb72c45f8add0aa1002a3666" +checksum = "3e37ca17d25fe1e210b6d4bdf59b81caebfe99f986201a1228cb5061233b4b13" dependencies = [ "futures-util", "pin-project-lite", @@ -511,12 +349,12 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.58.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e6859852e8dc253cd61cd9333f0eda852b2559b9e5b3816a3bc6389e030435" +checksum = "c5a373ec01aede3dd066ec018c1bc4e8f5dd11b2c11c59c8eef1a5c68101f397" dependencies = [ - "aws-smithy-http 0.58.0", - "aws-smithy-types 0.100.0", + "aws-smithy-http", + "aws-smithy-types", "bytes", "crc32c", "crc32fast", @@ -530,72 +368,26 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-client" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27b2756264c82f830a91cb4d2d485b2d19ad5bea476d9a966e03d27f27ba59a" -dependencies = [ - "aws-smithy-async 0.56.1", - "aws-smithy-http 0.56.1", - "aws-smithy-http-tower", - "aws-smithy-types 0.56.1", - "bytes", - "fastrand", - "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.27", - "hyper-rustls", - "lazy_static", - "pin-project-lite", - "rustls", - "tokio", - "tower", - "tracing", -] - [[package]] name = "aws-smithy-eventstream" -version = "0.58.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b06bdbc42a78b92e81c1582ed5b4f149b7d98e05536126ff329462f4828100" +checksum = "1c669e1e5fc0d79561bf7a122b118bd50c898758354fe2c53eb8f2d31507cbc3" dependencies = [ - "aws-smithy-types 0.100.0", + "aws-smithy-types", "bytes", "crc32fast", ] [[package]] name = "aws-smithy-http" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54cdcf365d8eee60686885f750a34c190e513677db58bbc466c44c588abf4199" -dependencies = [ - "aws-smithy-types 0.56.1", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.27", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "aws-smithy-http" -version = "0.58.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03f240d3eafae9359943d0ccdabf96f507e2ce00d865578bb10fc21c6a08a72" +checksum = "5b1de8aee22f67de467b2e3d0dd0fb30859dc53f579a63bd5381766b987db644" dependencies = [ "aws-smithy-eventstream", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", "bytes-utils", "futures-core", @@ -608,94 +400,38 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-http-tower" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822de399d0ce62829a69dfa8c5cd08efdbe61a7426b953e2268f8b8b52a607bd" -dependencies = [ - "aws-smithy-http 0.56.1", - "aws-smithy-types 0.56.1", - "bytes", - "http 0.2.9", - "http-body 0.4.5", - "pin-project-lite", - "tower", - "tracing", -] - [[package]] name = "aws-smithy-json" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1e7ab8fa7ad10c193af7ae56d2420989e9f4758bf03601a342573333ea34f" -dependencies = [ - "aws-smithy-types 0.56.1", -] - -[[package]] -name = "aws-smithy-json" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5593def1d50cab58bffa61483a66d369520ae8d95902562314bac9bad6aaa64d" -dependencies = [ - "aws-smithy-types 0.100.0", -] - -[[package]] -name = "aws-smithy-query" -version = "0.56.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28556a3902091c1f768a34f6c998028921bdab8d47d92586f363f14a4a32d047" +checksum = "6a46dd338dc9576d6a6a5b5a19bd678dcad018ececee11cf28ecd7588bd1a55c" dependencies = [ - "aws-smithy-types 0.56.1", - "urlencoding", + "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.58.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71452a6f146914adf3c6b1d4e2fc8a9df6bf10a2d05bd5b982e152d62280e18c" +checksum = "feb5b8c7a86d4b6399169670723b7e6f21a39fc833a30f5c5a2f997608178129" dependencies = [ - "aws-smithy-types 0.100.0", + "aws-smithy-types", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745e096b3553e7e0f40622aa04971ce52765af82bebdeeac53aa6fc82fe801e6" -dependencies = [ - "aws-smithy-async 0.56.1", - "aws-smithy-client", - "aws-smithy-http 0.56.1", - "aws-smithy-runtime-api 0.56.1", - "aws-smithy-types 0.56.1", - "bytes", - "fastrand", - "http 0.2.9", - "http-body 0.4.5", - "once_cell", - "pin-project-lite", - "pin-utils", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime" -version = "0.58.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1ad73bc100c48acc09794914f06ae717f8d07132c2512a667e516f0c0a5fd4d" +checksum = "273479291efc55e7b0bce985b139d86b6031adb8e50f65c1f712f20ba38f6388" dependencies = [ - "aws-smithy-async 0.100.0", - "aws-smithy-http 0.58.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", "fastrand", + "h2 0.3.21", "http 0.2.9", "http-body 0.4.5", "hyper 0.14.27", @@ -710,27 +446,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d0ae0c9cfd57944e9711ea610b48a963fb174a53aabacc08c5794a594b1d02" -dependencies = [ - "aws-smithy-async 0.56.1", - "aws-smithy-http 0.56.1", - "aws-smithy-types 0.56.1", - "bytes", - "http 0.2.9", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "0.100.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354f37c2df3661338507bdb78d4ace08d3ff687c3ece9b0dca12f6fd5d09184" +checksum = "c6cebff0d977b6b6feed2fd07db52aac58ba3ccaf26cdd49f1af4add5061bef9" dependencies = [ - "aws-smithy-async 0.100.0", - "aws-smithy-types 0.100.0", + "aws-smithy-async", + "aws-smithy-types", "bytes", "http 0.2.9", "pin-project-lite", @@ -741,23 +462,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d90dbc8da2f6be461fa3c1906b20af8f79d14968fe47f2b7d29d086f62a51728" -dependencies = [ - "base64-simd", - "itoa", - "num-integer", - "ryu", - "serde", - "time", -] - -[[package]] -name = "aws-smithy-types" -version = "0.100.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d66124208cd17da157b8603fee220187239caa3531a0c71260704a076281d8" +checksum = "d7f48b3f27ddb40ab19892a5abda331f403e3cb877965e4e51171447807104af" dependencies = [ "base64-simd", "bytes", @@ -778,58 +485,49 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.56.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01d2dedcdd8023043716cfeeb3c6c59f2d447fce365d8e194838891794b23b6" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfe149c4afea01e25105c712d16295c0b604631e97914968e3f77af27d646f2" +checksum = "0ec40d74a67fd395bc3f6b4ccbdf1543672622d905ef3f979689aea5b730cb95" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.56.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85aa0451bf8af1bf22a4f028d5d28054507a14be43cb8ac0597a8471fba9edfe" +checksum = "8403fc56b1f3761e8efe45771ddc1165e47ec3417c68e68a4519b5cb030159ca" dependencies = [ - "aws-credential-types 0.56.1", - "aws-smithy-async 0.56.1", - "aws-smithy-client", - "aws-smithy-http 0.56.1", - "aws-smithy-types 0.56.1", + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", "http 0.2.9", "rustc_version", "tracing", ] [[package]] -name = "aws-types" -version = "0.100.0" +name = "aws_lambda_events" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bee3e0dfe1b104957a49f17b5a77b1e783deac4dfd829f2718ca0437ac7b814" +checksum = "65991dbc3bfb586939ba1527eefdc99bc21157b6ec891f180fb1e16e2dddc7a9" dependencies = [ - "aws-credential-types 0.58.0", - "aws-smithy-async 0.100.0", - "aws-smithy-runtime-api 0.100.0", - "aws-smithy-types 0.100.0", + "base64", + "bytes", "http 0.2.9", - "rustc_version", - "tracing", + "http-body 0.4.5", + "http-serde", + "query_map 0.6.0", + "serde", + "serde_json", ] [[package]] name = "aws_lambda_events" -version = "0.10.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65991dbc3bfb586939ba1527eefdc99bc21157b6ec891f180fb1e16e2dddc7a9" +checksum = "03611508dd1e514e311caec235b581c99a4cb66fa1771bd502819eed69894f12" dependencies = [ "base64", "bytes", @@ -838,7 +536,7 @@ dependencies = [ "http 0.2.9", "http-body 0.4.5", "http-serde", - "query_map", + "query_map 0.7.0", "serde", "serde_dynamo", "serde_json", @@ -1117,6 +815,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1258,7 +966,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" dependencies = [ "base16ct", - "crypto-bigint", + "crypto-bigint 0.4.9", "der 0.6.1", "digest", "ff", @@ -1345,7 +1053,7 @@ name = "filemanager" version = "0.1.0" dependencies = [ "async-trait", - "aws-config 0.100.0", + "aws-config", "aws-sdk-s3", "aws-sdk-sqs", "axum", @@ -1385,9 +1093,9 @@ dependencies = [ name = "filemanager-ingest-lambda" version = "0.1.0" dependencies = [ - "aws-config 0.56.1", - "aws-sdk-sts 0.30.0", - "aws_lambda_events", + "aws-config", + "aws-sdk-sts", + "aws_lambda_events 0.12.1", "filemanager", "lambda_runtime", "serde", @@ -1910,7 +1618,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b15e8ab48a5d8eab3110567008caad4d191924d1470b74c5c8b802b904b5a34" dependencies = [ - "aws_lambda_events", + "aws_lambda_events 0.10.0", "base64", "bytes", "encoding_rs", @@ -2107,17 +1815,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "num-bigint" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -2383,6 +2080,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "query_map" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eab6b8b1074ef3359a863758dae650c7c0c6027927a085b7af911c8e0bf3a15" +dependencies = [ + "form_urlencoded", + "serde", + "serde_derive", +] + [[package]] name = "quote" version = "1.0.33" @@ -2501,7 +2209,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" dependencies = [ - "crypto-bigint", + "crypto-bigint 0.4.9", "hmac", "zeroize", ] diff --git a/lib/workload/stateful/filemanager/README.md b/lib/workload/stateful/filemanager/README.md index 9a60d410e..7d6eefb64 100644 --- a/lib/workload/stateful/filemanager/README.md +++ b/lib/workload/stateful/filemanager/README.md @@ -2,14 +2,13 @@ The filemanager ingests events from cloud storage like S3 and maintains a queryable table of objects. -This project is split up into multiple crates in a workspace. For development, docker is used, which enables localstack and a postgres database. +This project is split up into multiple crates in a workspace. For development, docker is used, which enables a postgres database, the rest of the stack is developed against an AWS account. ## Prerequisites - [Docker](https://docs.docker.com/get-docker/) - [Rust](https://www.rust-lang.org/tools/install) - [npm](https://www.npmjs.com/get-npm) -- [awslocal](https://github.com/localstack/awscli-local) ## Rust code development @@ -27,20 +26,16 @@ Filemanager uses docker to run a postgres database to track objects. This means running inside the docker compose container. If there are additional postgres installations locally (outside of docker), this might interfere and complain about non-existing roles and users. +### Tooling prerequisites, testing and building the code + For development of the rust workspace, install a build cache (sccache) and build manually: ```sh brew install sccache && export RUSTC_WRAPPER=`which sccache` +cargo install cargo-watch sqlx-cli cargo build --all-targets --all-features ``` -Or watch and automatically recompile changes: - -```sh -cargo install cargo-watch # if not installed previously -cargo watch -c -``` - Test with: ```sh @@ -54,32 +49,19 @@ cargo clippy --all-targets cargo fmt ``` -## Localstack development - -Localstack enables deploying and testing AWS services locally. See the [deploy][deploy] directory -for the cdk infrastructure code. - -## Setup and deployment +## Local development -For localstack testing and development: +See the [deploy][deploy] directory for the cdk infrastructure code. -```sh -docker compose up -``` - -Then deploy the cdk to localstack: +In a nutshell, a filemanager developer only needs to run the following: ```sh -./scripts deploy.sh +./scripts/watch.sh ``` -It's possible that a profile called "default" in `~/.aws/config` could interfere with awslocal. A recommended `~/.aws/credentials` that works with localstack's dummy `0000000000` AWS account would look like this: +To automatically recompile changes and re-deploy the changes. -``` -[default] -aws_access_key_id = access_key -aws_secret_access_key = secret_key -``` +Please don't use `scripts/deploy.sh` on production deployments, it is only meant for development. ## Database @@ -89,5 +71,7 @@ A shortcut for connecting to the docker database and inspecting its contents: docker exec -it filemanager_db psql filemanager -U filemanager ``` +Alternatively, just `brew install dbeaver-community` to easily browse the database contents (or any other DB viewer you prefer). + [deploy]: ./deploy -[env-example]: .env.example +[env-example]: .env.example \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/deploy/README.md b/lib/workload/stateful/filemanager/deploy/README.md index e55730daa..33425be40 100644 --- a/lib/workload/stateful/filemanager/deploy/README.md +++ b/lib/workload/stateful/filemanager/deploy/README.md @@ -1,10 +1,10 @@ # Deployment of filemanager -This folder contains CDK deployment code for filemanager. The CDK code can be deployed locally using `cdklocal` and `localstack`, or on AWS using `cdk`: +This folder contains CDK deployment code for filemanager. The CDK code can be deployed using `cdk`: ```sh npm install -npx cdklocal bootstrap -npx cdklocal deploy +cdk bootstrap +cdk deploy ``` diff --git a/lib/workload/stateful/filemanager/deploy/cdk.json b/lib/workload/stateful/filemanager/deploy/cdk.json index 0861d38e0..7447948b9 100644 --- a/lib/workload/stateful/filemanager/deploy/cdk.json +++ b/lib/workload/stateful/filemanager/deploy/cdk.json @@ -1,5 +1,5 @@ { - "app": "npx ts-node --prefer-ts-exts localstack/filemanager_localstack.ts", + "app": "npx ts-node --prefer-ts-exts stack/stack.ts", "watch": { "include": ["**"], "exclude": [ diff --git a/lib/workload/stateful/filemanager/deploy/package-lock.json b/lib/workload/stateful/filemanager/deploy/package-lock.json index db8069fba..29ffd9696 100644 --- a/lib/workload/stateful/filemanager/deploy/package-lock.json +++ b/lib/workload/stateful/filemanager/deploy/package-lock.json @@ -8,22 +8,21 @@ "name": "filemanager", "version": "0.1", "dependencies": { - "@aws-cdk/aws-apigatewayv2-alpha": "^2.110.0-alpha.0", - "@aws-cdk/aws-apigatewayv2-authorizers-alpha": "^2.110.0-alpha.0", - "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.110.0-alpha.0", - "aws-cdk-lib": "^2.110.0", + "@aws-cdk/aws-apigatewayv2-alpha": "^2.114.1-alpha.0", + "@aws-cdk/aws-apigatewayv2-authorizers-alpha": "^2.114.1-alpha.0", + "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.114.1-alpha.0", + "aws-cdk-lib": "^2.114.1", "constructs": "^10.3.0", "dotenv": "^16.3.1", "rust.aws-cdk-lambda": "^1.2.1", "source-map-support": "^0.5.21" }, "bin": { - "filemanager-localstack": "localstack/filemanager_localstack.js" + "filemanager-stack": "stack/filemanager_stack.js" }, "devDependencies": { "@types/node": "^20.5.9", - "aws-cdk": "^2.109.0", - "aws-cdk-local": "^2.18.0", + "aws-cdk": "^2.114.1", "prettier": "3.0.3", "typescript": "^5.2.2" } @@ -44,40 +43,40 @@ "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==" }, "node_modules/@aws-cdk/aws-apigatewayv2-alpha": { - "version": "2.110.0-alpha.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-alpha/-/aws-apigatewayv2-alpha-2.110.0-alpha.0.tgz", - "integrity": "sha512-9JhPK6azSrzL6kCewbvrN4nGJLqR6iEy0KY9TqQ2VL3EqW+W8YLfH2eovcmgCU7ggE0J0zlEtCt8Ixr3Wy1r7g==", + "version": "2.114.1-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-alpha/-/aws-apigatewayv2-alpha-2.114.1-alpha.0.tgz", + "integrity": "sha512-+urpw7rGrtdGvnHQlDXVfpI3TmQJpjuT9jTOeuuG5dNDczLJrUokBvQdj6H6KsngdmBC07WfWU+yL2MBp71ozA==", "engines": { "node": ">= 14.15.0" }, "peerDependencies": { - "aws-cdk-lib": "^2.110.0", + "aws-cdk-lib": "^2.114.1", "constructs": "^10.0.0" } }, "node_modules/@aws-cdk/aws-apigatewayv2-authorizers-alpha": { - "version": "2.110.0-alpha.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-authorizers-alpha/-/aws-apigatewayv2-authorizers-alpha-2.110.0-alpha.0.tgz", - "integrity": "sha512-bYKCYMSLvRUHk1rpEw5we0PLQb3w1ns7uDu0fdeYFdYlZJsf0YrvWsXC2uUwnrWe7S5fQOogkA4m2VZ+RyGNNQ==", + "version": "2.114.1-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-authorizers-alpha/-/aws-apigatewayv2-authorizers-alpha-2.114.1-alpha.0.tgz", + "integrity": "sha512-wvQ7l+VsDf8SgcOylLnp/6SirDcE5D3WNyQMl5ynxtmzuK5OKSnlC07bruKtKpYVT4A/6tBq/VVILfCBphyMJg==", "engines": { "node": ">= 14.15.0" }, "peerDependencies": { - "@aws-cdk/aws-apigatewayv2-alpha": "2.110.0-alpha.0", - "aws-cdk-lib": "^2.110.0", + "@aws-cdk/aws-apigatewayv2-alpha": "2.114.1-alpha.0", + "aws-cdk-lib": "^2.114.1", "constructs": "^10.0.0" } }, "node_modules/@aws-cdk/aws-apigatewayv2-integrations-alpha": { - "version": "2.110.0-alpha.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-integrations-alpha/-/aws-apigatewayv2-integrations-alpha-2.110.0-alpha.0.tgz", - "integrity": "sha512-OfdIfRt5I2poc9EOyWTUSHhuwYDI4G1WEmVZPFvdml2eHmf6RxaI+oIPMXAUsJsjGa7XcKHP4wM2upSuZwiYEQ==", + "version": "2.114.1-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-integrations-alpha/-/aws-apigatewayv2-integrations-alpha-2.114.1-alpha.0.tgz", + "integrity": "sha512-iB7vHoTguDKLeatJS4p/8OBqH4oLCG2xBn1toUFwMD9iWoRGahGiK0pYuq24baab3SuNS1LFThJw5Zu0R+cZGA==", "engines": { "node": ">= 14.15.0" }, "peerDependencies": { - "@aws-cdk/aws-apigatewayv2-alpha": "2.110.0-alpha.0", - "aws-cdk-lib": "^2.110.0", + "@aws-cdk/aws-apigatewayv2-alpha": "2.114.1-alpha.0", + "aws-cdk-lib": "^2.114.1", "constructs": "^10.0.0" } }, @@ -126,21 +125,24 @@ } }, "node_modules/@types/minimist": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", - "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", "optional": true }, "node_modules/@types/node": { - "version": "20.6.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.6.2.tgz", - "integrity": "sha512-Y+/1vGBHV/cYk6OI1Na/LHzwnlNCAfU3ZNGrc1LdRe/LAIbdDPTTv/HU3M7yXN448aTVDq3eKRm2cg7iKLb8gw==", - "devOptional": true + "version": "20.10.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.4.tgz", + "integrity": "sha512-D08YG6rr8X90YB56tSIuBaddy/UXAA9RKJoFvrsnogAum/0pmjkgi4+2nx96A330FmioegBWmEYQ+syqCFaveg==", + "devOptional": true, + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/ps-tree": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@types/ps-tree/-/ps-tree-1.1.2.tgz", - "integrity": "sha512-ZREFYlpUmPQJ0esjxoG1fMvB2HNaD3z+mjqdSosZvd3RalncI9NEur73P8ZJz4YQdL64CmV1w0RuqoRUlhQRBw==", + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@types/ps-tree/-/ps-tree-1.1.6.tgz", + "integrity": "sha512-PtrlVaOaI44/3pl3cvnlK+GxOM3re2526TJvPvh7W+keHIXdV4TE0ylpPBAcvFQCbGitaTXwL9u+RF7qtVeazQ==", "optional": true }, "node_modules/@types/which": { @@ -168,9 +170,9 @@ } }, "node_modules/aws-cdk": { - "version": "2.109.0", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.109.0.tgz", - "integrity": "sha512-e06YlA4HsKZCOdh3ApynZauJ3/224o/q5vOso3Fs5ksLkYhfXREl+O7UmAOZ1Nenq5ADNgccvNwuChQWbNSvSg==", + "version": "2.114.1", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.114.1.tgz", + "integrity": "sha512-iLOCPb3WAJOgVYQ4GvAnrjtScJfPwcczlB4995h3nUYQdHbus0jNffFv13zBShdWct3cuX+bqLuZ4JyEmJ9+rg==", "dev": true, "bin": { "cdk": "bin/cdk" @@ -183,9 +185,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.110.1", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.110.1.tgz", - "integrity": "sha512-Z+42Jc/KSKFdBOpEv4LK9tz6kQUdVvUBquIYhLajam3aGblkonM0/FgexvAqy8iGwUaVEIpVyBTZUP2/VUMalg==", + "version": "2.114.1", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.114.1.tgz", + "integrity": "sha512-pJy+Sa3+s6K9I0CXYGU8J5jumw9uQEbl8zPK8EMA+A6hP9qb1JN+a8ohyw6a1O1cb4D5S6gwH+hE7Fq7hGPY3A==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -205,7 +207,7 @@ "@balena/dockerignore": "^1.0.2", "case": "1.6.3", "fs-extra": "^11.1.1", - "ignore": "^5.2.4", + "ignore": "^5.3.0", "jsonschema": "^1.4.1", "minimatch": "^3.1.2", "punycode": "^2.3.1", @@ -342,7 +344,7 @@ "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/ignore": { - "version": "5.2.4", + "version": "5.3.0", "inBundle": true, "license": "MIT", "engines": { @@ -522,18 +524,6 @@ "node": ">= 6" } }, - "node_modules/aws-cdk-local": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/aws-cdk-local/-/aws-cdk-local-2.18.0.tgz", - "integrity": "sha512-ZNQxs4GKHG2MC5y19MJZE/Xf/c0tAdjH6LWbZdouyG4k+1DeuFtBZ+UoYjoUfGPI8dycGpwUiewsLBY1lbtTWg==", - "dev": true, - "dependencies": { - "diff": "^5.0.0" - }, - "bin": { - "cdklocal": "bin/cdklocal" - } - }, "node_modules/braces": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", @@ -580,15 +570,6 @@ "node": ">= 12" } }, - "node_modules/diff": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", - "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", - "dev": true, - "engines": { - "node": ">=0.3.1" - } - }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -634,9 +615,9 @@ } }, "node_modules/fast-glob": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz", - "integrity": "sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", + "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", "optional": true, "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -777,9 +758,9 @@ "optional": true }, "node_modules/ignore": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", - "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz", + "integrity": "sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==", "optional": true, "engines": { "node": ">= 4" @@ -1114,9 +1095,9 @@ "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" }, "node_modules/typescript": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", - "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -1126,10 +1107,16 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "devOptional": true + }, "node_modules/universalify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", - "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", "optional": true, "engines": { "node": ">= 10.0.0" @@ -1160,9 +1147,9 @@ } }, "node_modules/yaml": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.2.tgz", - "integrity": "sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==", + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.4.tgz", + "integrity": "sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==", "optional": true, "engines": { "node": ">= 14" diff --git a/lib/workload/stateful/filemanager/deploy/package.json b/lib/workload/stateful/filemanager/deploy/package.json index 7a14e6115..6a596b27a 100644 --- a/lib/workload/stateful/filemanager/deploy/package.json +++ b/lib/workload/stateful/filemanager/deploy/package.json @@ -2,7 +2,7 @@ "name": "filemanager", "version": "0.1", "bin": { - "filemanager-localstack": "localstack/filemanager_localstack.js" + "filemanager-stack": "stack/filemanager_stack.js" }, "scripts": { "build": "tsc", @@ -11,16 +11,15 @@ }, "devDependencies": { "@types/node": "^20.5.9", - "aws-cdk": "^2.109.0", - "aws-cdk-local": "^2.18.0", + "aws-cdk": "^2.114.1", "prettier": "3.0.3", "typescript": "^5.2.2" }, "dependencies": { - "@aws-cdk/aws-apigatewayv2-alpha": "^2.110.0-alpha.0", - "@aws-cdk/aws-apigatewayv2-authorizers-alpha": "^2.110.0-alpha.0", - "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.110.0-alpha.0", - "aws-cdk-lib": "^2.110.0", + "@aws-cdk/aws-apigatewayv2-alpha": "^2.114.1-alpha.0", + "@aws-cdk/aws-apigatewayv2-authorizers-alpha": "^2.114.1-alpha.0", + "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.114.1-alpha.0", + "aws-cdk-lib": "^2.114.1", "constructs": "^10.3.0", "dotenv": "^16.3.1", "rust.aws-cdk-lambda": "^1.2.1", diff --git a/lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts index 538b9d194..d62721802 100644 --- a/lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts +++ b/lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts @@ -4,18 +4,20 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import { RustFunction, Settings as CargoSettings } from 'rust.aws-cdk-lambda'; import { Architecture } from 'aws-cdk-lib/aws-lambda'; import * as s3 from 'aws-cdk-lib/aws-s3'; -import { CfnBucket } from 'aws-cdk-lib/aws-s3'; +import * as rds from 'aws-cdk-lib/aws-rds'; import * as sqs from 'aws-cdk-lib/aws-sqs'; import * as lambdaDestinations from 'aws-cdk-lib/aws-lambda-destinations'; import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; +import * as s3n from 'aws-cdk-lib/aws-s3-notifications'; /** * Common settings for the filemanager stack. */ interface Settings { database_url: string; - endpoint_url: string; - force_path_style: boolean; + endpoint_url?: string; stack_name: string; buildEnvironment?: NodeJS.ProcessEnv; } @@ -41,26 +43,11 @@ export class FilemanagerStack extends Stack { blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, encryption: s3.BucketEncryption.S3_MANAGED, enforceSSL: true, - removalPolicy: RemovalPolicy.RETAIN, + removalPolicy: RemovalPolicy.DESTROY, }); - // Workaround for localstack, see https://github.com/localstack/localstack/issues/3468. - const cfnBucket = testBucket.node.defaultChild as CfnBucket; - cfnBucket.notificationConfiguration = { - queueConfigurations: [ - { - event: 's3:ObjectCreated:*', - queue: queue.queueArn, - }, - { - event: 's3:ObjectRemoved:*', - queue: queue.queueArn, - }, - ], - }; - - // testBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.SqsDestination(queue)); - // testBucket.addEventNotification(s3.EventType.OBJECT_REMOVED, new s3n.SqsDestination(queue)); + testBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.SqsDestination(queue)); + testBucket.addEventNotification(s3.EventType.OBJECT_REMOVED, new s3n.SqsDestination(queue)); lambdaRole.addManagedPolicy( iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaSQSQueueExecutionRole') @@ -86,8 +73,7 @@ export class FilemanagerStack extends Stack { timeout: Duration.seconds(28), environment: { DATABASE_URL: settings.database_url, - ENDPOINT_URL: settings.endpoint_url, - FORCE_PATH_STYLE: settings.force_path_style.toString(), + ...(settings.endpoint_url && { ENDPOINT_URL: settings.endpoint_url }), SQS_QUEUE_URL: queue.queueUrl, RUST_LOG: 'info,filemanager_ingest_lambda=trace,filemanager=trace', }, @@ -100,6 +86,47 @@ export class FilemanagerStack extends Stack { const eventSource = new lambdaEventSources.SqsEventSource(queue); filemanagerLambda.addEventSource(eventSource); - // todo RDS instance. + // VPC + //const vpc = ec2.Vpc.fromLookup(this, 'main-vpc', { isDefault: false }); + const vpc = new ec2.Vpc(this, 'vpc', { + ipAddresses: ec2.IpAddresses.cidr('10.0.0.0/24'), + maxAzs: 99, // As many as there are available in the region + natGateways: 1, + subnetConfiguration: [ + { + name: 'ingress', + subnetType: ec2.SubnetType.PUBLIC, + }, + { + name: 'application', + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + }, + { + name: 'database', + subnetType: ec2.SubnetType.PRIVATE_ISOLATED, + }, + ], + }); + + // Secret + new secretsmanager.Secret(this, 'filemanager_db_secret', { + secretName: 'filemanager_db_secret', // pragma: allowlist secret + generateSecretString: { + secretStringTemplate: JSON.stringify({ username: 'filemanager' }), + excludePunctuation: true, + generateStringKey: 'password', + }, + }); + + // RDS + new rds.ServerlessCluster(this, 'Database', { + engine: rds.DatabaseClusterEngine.auroraPostgres({ + version: rds.AuroraPostgresEngineVersion.VER_13_12, + }), + defaultDatabaseName: 'filemanager', + credentials: rds.Credentials.fromGeneratedSecret('filemanager_db_secret'), + removalPolicy: RemovalPolicy.DESTROY, + vpc, + }); } } diff --git a/lib/workload/stateful/filemanager/deploy/localstack/filemanager_localstack.ts b/lib/workload/stateful/filemanager/deploy/stack/stack.ts similarity index 83% rename from lib/workload/stateful/filemanager/deploy/localstack/filemanager_localstack.ts rename to lib/workload/stateful/filemanager/deploy/stack/stack.ts index 340407dac..3cc4f175e 100644 --- a/lib/workload/stateful/filemanager/deploy/localstack/filemanager_localstack.ts +++ b/lib/workload/stateful/filemanager/deploy/stack/stack.ts @@ -2,9 +2,9 @@ import 'source-map-support/register'; import * as cdk from 'aws-cdk-lib'; -import { FilemanagerStack } from '../stack/filemanager_stack'; +import { FilemanagerStack } from './filemanager_stack'; -export const STACK_NAME = 'FilemanagerLocalStack'; +export const STACK_NAME = 'FilemanagerStack'; const STACK_DESCRIPTION = 'A stack deploying filemanager to dev.'; const app = new cdk.App(); @@ -13,8 +13,6 @@ new FilemanagerStack( STACK_NAME, { database_url: 'postgresql://filemanager:filemanager@db:5432/filemanager', // pragma: allowlist secret - endpoint_url: 'http://localstack:4566', - force_path_style: true, stack_name: STACK_NAME, buildEnvironment: { // Override release profile to match defaults for dev builds. @@ -37,8 +35,5 @@ new FilemanagerStack( tags: { Stack: STACK_NAME, }, - env: { - account: '000000000000', - }, } ); diff --git a/lib/workload/stateful/filemanager/docker-compose.yml b/lib/workload/stateful/filemanager/docker-compose.yml index 6283a14c3..132744106 100644 --- a/lib/workload/stateful/filemanager/docker-compose.yml +++ b/lib/workload/stateful/filemanager/docker-compose.yml @@ -12,15 +12,15 @@ services: ports: - "5432:5432" - localstack: - container_name: localstack-main - image: localstack/localstack:3.0.1 - ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway - - "127.0.0.1:4510-4559:4510-4559" # external services port range - environment: - - DEBUG=${DEBUG-} - - DOCKER_HOST=unix:///var/run/docker.sock - volumes: - - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" +# localstack: +# container_name: localstack-main +# image: localstack/localstack:3.0.1 +# ports: +# - "127.0.0.1:4566:4566" # LocalStack Gateway +# - "127.0.0.1:4510-4559:4510-4559" # external services port range +# environment: +# - DEBUG=${DEBUG-} +# - DOCKER_HOST=unix:///var/run/docker.sock +# volumes: +# - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" +# - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/lib/workload/stateful/filemanager/filemanager-http-lambda/src/main.rs b/lib/workload/stateful/filemanager/filemanager-http-lambda/src/main.rs index 683bc2405..f70019d8d 100644 --- a/lib/workload/stateful/filemanager/filemanager-http-lambda/src/main.rs +++ b/lib/workload/stateful/filemanager/filemanager-http-lambda/src/main.rs @@ -3,6 +3,7 @@ use lambda_runtime::{run, service_fn, LambdaEvent}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{fmt, EnvFilter}; +use tracing::trace; use filemanager::database::s3::ingester::Ingester; use filemanager::database::Ingest; @@ -15,7 +16,8 @@ use filemanager::events::Collect; async fn event_handler(_: LambdaEvent<()>) -> Result<(), Error> { let sqs = SQS::with_default_client().await?; let events = sqs.receive().await?; - + + trace!("On the filemanager-http-lambda, just before collecter"); let events = Collecter::with_defaults(events).await?.collect().await?; let mut ingester = Ingester::new_with_defaults().await?; diff --git a/lib/workload/stateful/filemanager/filemanager-ingest-lambda/Cargo.toml b/lib/workload/stateful/filemanager/filemanager-ingest-lambda/Cargo.toml index ed0935ce2..2e1bc848b 100644 --- a/lib/workload/stateful/filemanager/filemanager-ingest-lambda/Cargo.toml +++ b/lib/workload/stateful/filemanager/filemanager-ingest-lambda/Cargo.toml @@ -7,15 +7,15 @@ edition.workspace = true authors.workspace = true [dependencies] -aws_lambda_events = "0.10" +aws_lambda_events = "0.12" lambda_runtime = "0.8" -serde = "1.0.136" +serde = "1" tokio = { version = "1", features = ["macros"] } tracing = { version = "0.1", features = ["log"] } tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt"] } filemanager = { path = "../filemanager" } -serde_json = "1.0.105" +serde_json = "1" -aws-config = "0.56" -aws-sdk-sts = "0.30" \ No newline at end of file +aws-config = "1" +aws-sdk-sts = "1" \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/filemanager/Cargo.toml b/lib/workload/stateful/filemanager/filemanager/Cargo.toml index a341716c9..051a56382 100644 --- a/lib/workload/stateful/filemanager/filemanager/Cargo.toml +++ b/lib/workload/stateful/filemanager/filemanager/Cargo.toml @@ -26,9 +26,9 @@ async-trait = "0.1" uuid = { version = "1.4", features = ["v4"] } # AWS -aws-sdk-sqs = "0.37" -aws-config = "0.100" -aws-sdk-s3 = "0.37" +aws-sdk-sqs = "1" +aws-config = "1" +aws-sdk-s3 = "1" futures = "0.3" #lambda_http = "0.8.0" #lambda_runtime = "0.8.0" \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/s3/collect.rs b/lib/workload/stateful/filemanager/filemanager/src/events/s3/collect.rs index 75f03c269..6057c988f 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/s3/collect.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/s3/collect.rs @@ -1,5 +1,7 @@ use async_trait::async_trait; +use tracing::trace; + use crate::error::Result; use crate::events::s3::s3_client::S3; use crate::events::s3::{Events, FlatS3EventMessages}; @@ -20,6 +22,7 @@ impl Collecter { /// Create a new collector with a default S3 client. pub async fn with_defaults(raw_events: FlatS3EventMessages) -> Result { + trace!("Creating collector with defaults"); Ok(Self { s3: S3::with_defaults().await?, raw_events, @@ -30,8 +33,9 @@ impl Collecter { #[async_trait] impl Collect for Collecter { async fn collect(self) -> Result { + trace!("Entering collect"); let events = self.s3.update_events(self.raw_events).await?; - + trace!("Victor #ftw"); Ok(EventType::S3(Events::from(events))) } } diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/s3/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/s3/mod.rs index 216a77530..206ed6309 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/s3/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/s3/mod.rs @@ -285,7 +285,7 @@ impl FlatS3EventMessage { #[derive(Debug, Serialize, Deserialize, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct S3EventMessage { - #[serde(rename = "records")] + #[serde(alias = "Records")] pub records: Vec, } diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/s3/s3_client.rs b/lib/workload/stateful/filemanager/filemanager/src/events/s3/s3_client.rs index 77098552e..7e46469a7 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/s3/s3_client.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/s3/s3_client.rs @@ -88,21 +88,24 @@ impl S3 { /// Process events and add header and datetime fields. pub async fn update_events(&self, events: FlatS3EventMessages) -> Result { Ok(FlatS3EventMessages( - join_all(events.into_inner().into_iter().map(|mut event| async move { + join_all(events.into_inner().into_iter().map(|event| async move { trace!(key = ?event.key, bucket = ?event.bucket, "updating event"); - if let Some(head) = self.head(&event.key, &event.bucket).await? { - let HeadObjectOutput { - storage_class, - last_modified, - .. - } = head; - - event = - event.with_storage_class(storage_class.and_then(StorageClass::from_aws)); - event = event.with_last_modified_date(Self::convert_datetime(last_modified)); - } - + // if let Some(head) = self.head(&event.key, &event.bucket).await? { + // trace!("Before headoject storage/datetime mod"); + // let HeadObjectOutput { + // storage_class, + // last_modified, + // .. + // } = head; + // trace!("In the middle of updating event"); + // event = + // event.with_storage_class(storage_class.and_then(StorageClass::from_aws)); + // event = event.with_last_modified_date(Self::convert_datetime(last_modified)); + // } + + // trace!(key = ?event.key, bucket = ?event.bucket, "event updated"); + Ok(event) })) .await diff --git a/lib/workload/stateful/filemanager/scripts/deploy.sh b/lib/workload/stateful/filemanager/scripts/deploy.sh index 8cadda97a..385fa3c87 100755 --- a/lib/workload/stateful/filemanager/scripts/deploy.sh +++ b/lib/workload/stateful/filemanager/scripts/deploy.sh @@ -1,15 +1,17 @@ #!/bin/sh -x +# TODO: Takes too long for re-deploy, find further shortcuts export AWS_ENDPOINT_URL=http://localhost:4566 export FM_BUCKET=filemanager-test-ingest +docker compose down +docker compose up --wait --wait-timeout 20 -d cd deploy +npm install yes | npx cdklocal destroy yes | npx cdklocal bootstrap -yes | npx cdklocal deploy --require-approval never cd ../database && sqlx migrate run && cd .. -aws s3 mb s3://$FM_BUCKET -aws s3api put-object --bucket $FM_BUCKET --key test +cd deploy && yes | npx cdklocal deploy --require-approval never && cd .. -./scripts/logs.sh +aws s3api put-object --bucket $FM_BUCKET --key test diff --git a/lib/workload/stateful/filemanager/scripts/logs.sh b/lib/workload/stateful/filemanager/scripts/logs.sh index af39aad30..b018b5882 100755 --- a/lib/workload/stateful/filemanager/scripts/logs.sh +++ b/lib/workload/stateful/filemanager/scripts/logs.sh @@ -2,5 +2,6 @@ export AWS_ENDPOINT_URL=http://localhost:4566 +# Don't die when the whole stack re-deploys group_name=$(aws logs describe-log-groups --query 'logGroups[*].logGroupName' --output text) -aws logs tail "$group_name" --follow \ No newline at end of file +aws logs tail "$group_name" --follow diff --git a/lib/workload/stateful/filemanager/scripts/watch.sh b/lib/workload/stateful/filemanager/scripts/watch.sh new file mode 100755 index 000000000..695a0a0c4 --- /dev/null +++ b/lib/workload/stateful/filemanager/scripts/watch.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +cargo watch -w *.toml -w *.rs -- ./scripts/deploy.sh