From 9297a92ab51656561b3872fe169112f81f256cc1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 25 Oct 2023 12:35:21 +0200 Subject: [PATCH] [Actions][Integration] Add tests for `mapper` --- .github/integration/sda-s3-integration.yml | 26 +++ .../integration/tests/sda/40_mapper_test.sh | 156 ++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 .github/integration/tests/sda/40_mapper_test.sh diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 0f03b999a..d59e6abe7 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -184,6 +184,30 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared + mapper: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-mapper ] + container_name: mapper + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=mapper + - BROKER_USER=mapper + - BROKER_QUEUE=mappings + - DB_PASSWORD=mapper + - DB_USER=mapper + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + oidc: container_name: oidc command: @@ -222,6 +246,8 @@ services: condition: service_started ingest: condition: service_started + mapper: + condition: service_started s3inbox: condition: service_started verify: diff --git a/.github/integration/tests/sda/40_mapper_test.sh b/.github/integration/tests/sda/40_mapper_test.sh new file mode 100644 index 000000000..7de50977f --- /dev/null +++ b/.github/integration/tests/sda/40_mapper_test.sh @@ -0,0 +1,156 @@ +#!/bin/bash +set -e + +cd shared || true + +## map files to dataset +properties=$( + jq -c -n \ + --argjson delivery_mode 2 \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named' +) + +mappings=$( + jq -c -n \ + '$ARGS.positional' \ + --args "EGAF74900000001" \ + --args "EGAF74900000002" +) + +mapping_payload=$( + jq -r -c -n \ + --arg type mapping \ + --arg dataset_id EGAD74900000101 \ + --argjson accession_ids "$mappings" \ + '$ARGS.named|@base64' +) + +mapping_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "mappings" \ + --arg payload_encoding base64 \ + --arg payload "$mapping_payload" \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$mapping_body" + +# check DB for dataset contents +RETRY_TIMES=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "select count(id) from sda.file_dataset where dataset_id = (select id from sda.datasets where stable_id = 'EGAD74900000101')")" -eq 2 ]; do + echo "waiting for mapper to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for dataset to be mapped" + exit 1 + fi + sleep 2 +done + +## check that files has been removed form the inbox +for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do + result=$(s3cmd -c direct ls s3://inbox/test_dummy.org/"$file") + if [ "$result" != "" ]; then + echo "Failed to remove $file from inbox" + exit 1 + fi +done + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where file_id = (select id from sda.files where stable_id = 'EGAF74900000002') order by started_at DESC LIMIT 1")" = "ready" ]; do + echo "waiting for files be ready" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for files to be ready" + exit 1 + fi + sleep 2 +done + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.dataset_event_log where dataset_id = 'EGAD74900000101' order by event_date DESC LIMIT 1")" = "registered" ]; do + echo "waiting for dataset be registered" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for dataset to be registered" + exit 1 + fi + sleep 2 +done + +echo "dataset mapped successfully" + +## release dataset +release_payload=$( + jq -r -c -n \ + --arg type release \ + --arg dataset_id EGAD74900000101 \ + '$ARGS.named' +) + +release_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "mappings" \ + --arg payload "$release_payload" \ + --arg payload_encoding string \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$release_body" + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.dataset_event_log where dataset_id = 'EGAD74900000101' order by event_date DESC LIMIT 1")" = "released" ]; do + echo "waiting for dataset be released" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for dataset to be released" + exit 1 + fi + sleep 2 +done + +echo "dataset released successfully" + +## deprecate dataset +deprecate_payload=$( + jq -r -c -n \ + --arg type deprecate \ + --arg dataset_id EGAD74900000101 \ + '$ARGS.named' +) + +deprecate_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "mappings" \ + --arg payload "$deprecate_payload" \ + --arg payload_encoding string \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$deprecate_body" + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.dataset_event_log where dataset_id = 'EGAD74900000101' order by event_date DESC LIMIT 1")" = "deprecated" ]; do + echo "waiting for dataset be deprecated" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for dataset to be deprecated" + exit 1 + fi + sleep 2 +done + +echo "dataset deprecated successfully" \ No newline at end of file