diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 5f0bd3f0a..fb1908a83 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -17,6 +17,7 @@ pip install aiohttp Authlib joserfc requests > /dev/null for n in download finalize inbox ingest mapper sync verify; do echo "creating credentials for: $n" psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" + psql -U postgres -h postgres -d sda -c "GRANT base TO $n;" ## password and permissions for MQ body_data=$(jq -n -c --arg password "$n" --arg tags none '$ARGS.named') @@ -60,6 +61,11 @@ if [ ! -f "/shared/c4gh.sec.pem" ]; then curl -s -L https://github.com/neicnordic/crypt4gh/releases/download/v1.7.4/crypt4gh_linux_x86_64.tar.gz | tar -xz -C /shared/ && chmod +x /shared/crypt4gh /shared/crypt4gh generate -n /shared/c4gh -p c4ghpass fi +if [ ! -f "/shared/sync.sec.pem" ]; then + echo "creating crypth4gh key" + curl -s -L https://github.com/neicnordic/crypt4gh/releases/download/v1.7.4/crypt4gh_linux_x86_64.tar.gz | tar -xz -C /shared/ && chmod +x /shared/crypt4gh + /shared/crypt4gh generate -n /shared/sync -p syncPass +fi if [ ! -f "/shared/keys/ssh" ]; then ssh-keygen -o -a 256 -t ed25519 -f /shared/keys/ssh -N "" diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index d59e6abe7..3b6521efe 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -208,6 +208,30 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared + sync: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-sync ] + container_name: sync + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=sync + - BROKER_USER=sync + - BROKER_QUEUE=completed_stream + - DB_PASSWORD=sync + - DB_USER=sync + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + oidc: container_name: oidc command: @@ -250,6 +274,8 @@ services: condition: service_started s3inbox: condition: service_started + sync: + condition: service_started verify: condition: service_started environment: diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index 15e949f6e..58c097295 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -50,6 +50,7 @@ db: c4gh: filePath: /shared/c4gh.sec.pem passphrase: "c4ghpass" + syncPubKey: /shared/sync.pub.pem server: cert: "" @@ -57,4 +58,12 @@ server: jwtpubkeypath: "/shared/keys/pub/" jwtpubkeyurl: "http://oidc:8080/jwk" - +sync.destination: + type: "s3" + url: "http://s3" + port: 9000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "sync" + region: "us-east-1" diff --git a/.github/integration/tests/sda/50_sync_test.sh b/.github/integration/tests/sda/50_sync_test.sh new file mode 100644 index 000000000..7370aaf79 --- /dev/null +++ b/.github/integration/tests/sda/50_sync_test.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +cd shared || true + +# check DB for archive file names +for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do + size=$(s3cmd -c direct ls s3://sync/test_dummy.org/"$file" | tr -s ' ' | cut -d ' ' -f 3) + if [ -z "$size" ]; then + echo "Failed to get size of $file from sync site" + exit 1 + fi +done + +echo "files synced successfully" \ No newline at end of file