From 74fdb4ec0539b603f3100bf6c6876b016a8a610a Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Wed, 29 Nov 2023 16:04:35 +1100 Subject: [PATCH] Simplify development and debugging with ./scripts/(re)deploy.sh and ./scripts/logs.sh, making those scripts fully idempotent and easy to use --- lib/workload/stateful/filemanager/README.md | 33 +---------- .../deploy/aws-get-filemanager-logs.sh | 58 ------------------- .../deploy/redeploy_test_localstack.sh | 6 -- .../stateful/filemanager/scripts/deploy.sh | 15 +++++ .../stateful/filemanager/scripts/logs.sh | 6 ++ 5 files changed, 23 insertions(+), 95 deletions(-) delete mode 100755 lib/workload/stateful/filemanager/deploy/aws-get-filemanager-logs.sh delete mode 100755 lib/workload/stateful/filemanager/deploy/redeploy_test_localstack.sh create mode 100755 lib/workload/stateful/filemanager/scripts/deploy.sh create mode 100755 lib/workload/stateful/filemanager/scripts/logs.sh diff --git a/lib/workload/stateful/filemanager/README.md b/lib/workload/stateful/filemanager/README.md index 49f060d52..9a60d410e 100644 --- a/lib/workload/stateful/filemanager/README.md +++ b/lib/workload/stateful/filemanager/README.md @@ -70,10 +70,7 @@ docker compose up Then deploy the cdk to localstack: ```sh -cd deploy -npm install -npx cdklocal bootstrap -npx cdklocal deploy +./scripts deploy.sh ``` It's possible that a profile called "default" in `~/.aws/config` could interfere with awslocal. A recommended `~/.aws/credentials` that works with localstack's dummy `0000000000` AWS account would look like this: @@ -84,35 +81,9 @@ aws_access_key_id = access_key aws_secret_access_key = secret_key ``` -Make sure there's no pre-existing deployment with `npx cdklocal destroy`, otherwise your stack might fail to deploy with `CREATE FAILED`. -Also, `cargo install sqlx-cli` tools to easy database migration helpers and perform migrations when needed: - -```shell -cd database && sqlx migrate run -``` - -## Ingestion test - -This allows creating synthetic events that are ingested and stored in filemanager's database. - -Here it is assumed that [`awslocal`](https://github.com/localstack/awscli-local) has been installed beforehand. - -First, push an object (in order to create a log group): - -```sh -awslocal s3 mb s3://filemanager-test-ingest -awslocal s3api put-object --bucket filemanager-test-ingest --key test -``` - -Then in a separate terminal: - -```sh -./deploy/aws-get-filemanager-logs.sh -c awslocal -``` - ## Database -A shortcut for connecting to the docker database: +A shortcut for connecting to the docker database and inspecting its contents: ```bash docker exec -it filemanager_db psql filemanager -U filemanager diff --git a/lib/workload/stateful/filemanager/deploy/aws-get-filemanager-logs.sh b/lib/workload/stateful/filemanager/deploy/aws-get-filemanager-logs.sh deleted file mode 100755 index d2dba69dc..000000000 --- a/lib/workload/stateful/filemanager/deploy/aws-get-filemanager-logs.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -function usage() { - usage="$(basename "$0") [-h] [-c awslocal|aws] -- get cloudwatch logs from the filemanager lambda functions - - commands: - awslocal use awslocal - aws use regular aws cli - - options: - -h show this help text - -c the command for this script to run [default: not set, required]" - - echo "$usage" -} - -function run_command() { - # FIXME: - # Does not warn/fail properly when there are no log groups active - # % awslocal logs describe-log-groups - # { - # "logGroups": [] - # } - group_name=$("$command" logs describe-log-groups --query logGroups[*].logGroupName | jq -r '.[]') - "$command" logs tail "$group_name" --follow -} - -command="" -while getopts ':hc:' option; do - case "$option" in - h) - usage - exit - ;; - c) - command="$OPTARG" - ;; - :) - printf "missing argument for -%s\n" "$OPTARG" >&2 - usage >&2 - exit 1 - ;; - \?) - printf "invalid option: -%s\n" "$OPTARG" >&2 - usage >&2 - exit 1 - ;; - esac -done -shift $((OPTIND - 1)) - -if [[ -z "$command" ]]; then - printf "command option must be set\n" >&2 - usage >&2 - exit 1 -fi - -run_command \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/deploy/redeploy_test_localstack.sh b/lib/workload/stateful/filemanager/deploy/redeploy_test_localstack.sh deleted file mode 100755 index 1fe6b08af..000000000 --- a/lib/workload/stateful/filemanager/deploy/redeploy_test_localstack.sh +++ /dev/null @@ -1,6 +0,0 @@ -npx cdklocal bootstrap -npx cdklocal deploy - -awslocal s3api put-object --bucket filemanager-test-ingest --key test - -/bin/bash aws-get-filemanager-logs.sh -c awslocal > logs.txt \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/scripts/deploy.sh b/lib/workload/stateful/filemanager/scripts/deploy.sh new file mode 100755 index 000000000..aaec25ece --- /dev/null +++ b/lib/workload/stateful/filemanager/scripts/deploy.sh @@ -0,0 +1,15 @@ +#!/bin/sh -x + +export AWS_ENDPOINT_URL=http://localhost:4566 +export FM_BUCKET=filemanager-test-ingest + +cd deploy +yes | npx cdklocal destroy +yes | npx cdklocal bootstrap +yes | npx cdklocal deploy +cd ../database && sqlx migrate run && cd .. + +aws s3 mb s3://$FM_BUCKET +aws s3api put-object --bucket $FM_BUCKET --key test + +./scripts/logs.sh \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/scripts/logs.sh b/lib/workload/stateful/filemanager/scripts/logs.sh new file mode 100755 index 000000000..8a910a989 --- /dev/null +++ b/lib/workload/stateful/filemanager/scripts/logs.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +export AWS_ENDPOINT_URL=http://localhost:4566 + +group_name=$(aws logs describe-log-groups --query 'logGroups[*].logGroupName' --output text) +aws logs tail "$group_name" --follow \ No newline at end of file