diff --git a/lib/workload/stateful/filemanager/README.md b/lib/workload/stateful/filemanager/README.md index 6eac40508..9a60d410e 100644 --- a/lib/workload/stateful/filemanager/README.md +++ b/lib/workload/stateful/filemanager/README.md @@ -27,9 +27,10 @@ Filemanager uses docker to run a postgres database to track objects. This means running inside the docker compose container. If there are additional postgres installations locally (outside of docker), this might interfere and complain about non-existing roles and users. -For development of the rust workspace, build manually: +For development of the rust workspace, install a build cache (sccache) and build manually: ```sh +brew install sccache && export RUSTC_WRAPPER=`which sccache` cargo build --all-targets --all-features ``` @@ -69,10 +70,7 @@ docker compose up Then deploy the cdk to localstack: ```sh -cd deploy -npm install -npx cdklocal bootstrap -npx cdklocal deploy +./scripts deploy.sh ``` It's possible that a profile called "default" in `~/.aws/config` could interfere with awslocal. A recommended `~/.aws/credentials` that works with localstack's dummy `0000000000` AWS account would look like this: @@ -83,35 +81,9 @@ aws_access_key_id = access_key aws_secret_access_key = secret_key ``` -Make sure there's no pre-existing deployment with `npx cdklocal destroy`, otherwise your stack might fail to deploy with `CREATE FAILED`. -Also, `cargo install sqlx-cli` tools to easy database migration helpers and perform migrations when needed: - -```shell -cd database && sqlx migrate run -``` - -## Ingestion test - -This allows creating synthetic events that are ingested and stored in filemanager's database. - -Here it is assumed that [`awslocal`](https://github.com/localstack/awscli-local) has been installed beforehand. - -First, push an object (in order to create a log group): - -```sh -awslocal s3 mb s3://filemanager-test-ingest -awslocal s3api put-object --bucket filemanager-test-ingest --key test -``` - -Then in a separate terminal: - -```sh -./deploy/aws-get-filemanager-logs.sh -c awslocal -``` - ## Database -A shortcut for connecting to the docker database: +A shortcut for connecting to the docker database and inspecting its contents: ```bash docker exec -it filemanager_db psql filemanager -U filemanager diff --git a/lib/workload/stateful/filemanager/deploy/aws-get-filemanager-logs.sh b/lib/workload/stateful/filemanager/deploy/aws-get-filemanager-logs.sh deleted file mode 100755 index d2dba69dc..000000000 --- a/lib/workload/stateful/filemanager/deploy/aws-get-filemanager-logs.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -function usage() { - usage="$(basename "$0") [-h] [-c awslocal|aws] -- get cloudwatch logs from the filemanager lambda functions - - commands: - awslocal use awslocal - aws use regular aws cli - - options: - -h show this help text - -c the command for this script to run [default: not set, required]" - - echo "$usage" -} - -function run_command() { - # FIXME: - # Does not warn/fail properly when there are no log groups active - # % awslocal logs describe-log-groups - # { - # "logGroups": [] - # } - group_name=$("$command" logs describe-log-groups --query logGroups[*].logGroupName | jq -r '.[]') - "$command" logs tail "$group_name" --follow -} - -command="" -while getopts ':hc:' option; do - case "$option" in - h) - usage - exit - ;; - c) - command="$OPTARG" - ;; - :) - printf "missing argument for -%s\n" "$OPTARG" >&2 - usage >&2 - exit 1 - ;; - \?) - printf "invalid option: -%s\n" "$OPTARG" >&2 - usage >&2 - exit 1 - ;; - esac -done -shift $((OPTIND - 1)) - -if [[ -z "$command" ]]; then - printf "command option must be set\n" >&2 - usage >&2 - exit 1 -fi - -run_command \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/deploy/redeploy_test_localstack.sh b/lib/workload/stateful/filemanager/deploy/redeploy_test_localstack.sh deleted file mode 100755 index 1fe6b08af..000000000 --- a/lib/workload/stateful/filemanager/deploy/redeploy_test_localstack.sh +++ /dev/null @@ -1,6 +0,0 @@ -npx cdklocal bootstrap -npx cdklocal deploy - -awslocal s3api put-object --bucket filemanager-test-ingest --key test - -/bin/bash aws-get-filemanager-logs.sh -c awslocal > logs.txt \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/docker-compose.yml b/lib/workload/stateful/filemanager/docker-compose.yml index 866126e0d..6283a14c3 100644 --- a/lib/workload/stateful/filemanager/docker-compose.yml +++ b/lib/workload/stateful/filemanager/docker-compose.yml @@ -13,8 +13,8 @@ services: - "5432:5432" localstack: - container_name: localstack - image: localstack/localstack:3.0.0 + container_name: localstack-main + image: localstack/localstack:3.0.1 ports: - "127.0.0.1:4566:4566" # LocalStack Gateway - "127.0.0.1:4510-4559:4510-4559" # external services port range @@ -23,4 +23,4 @@ services: - DOCKER_HOST=unix:///var/run/docker.sock volumes: - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" \ No newline at end of file + - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/s3/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/s3/mod.rs index 09f3b635b..216a77530 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/s3/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/s3/mod.rs @@ -56,6 +56,7 @@ impl StorageClass { } /// AWS S3 events with fields transposed +/// TODO: Document why we need those 'transposed' #[derive(Debug, Eq, PartialEq, Default)] pub struct TransposedS3EventMessages { pub object_ids: Vec, @@ -73,6 +74,7 @@ pub struct TransposedS3EventMessages { impl TransposedS3EventMessages { /// Create a new transposed S3 event messages vector with the given capacity. + /// TODO: There was a S3 messaging spec about how long those fields are supposed to be? pub fn with_capacity(capacity: usize) -> Self { Self { object_ids: Vec::with_capacity(capacity), @@ -283,7 +285,7 @@ impl FlatS3EventMessage { #[derive(Debug, Serialize, Deserialize, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct S3EventMessage { - #[serde(rename = "Records")] + #[serde(rename = "records")] pub records: Vec, }