[Integration test] Add case for sync

.github/integration/scripts/make_sda_credentials.sh

@@ -17,6 +17,7 @@ pip install aiohttp Authlib joserfc requests > /dev/null
 for n in download finalize inbox ingest mapper sync verify; do
     echo "creating credentials for: $n"
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
+    psql -U postgres -h postgres -d sda -c "GRANT base TO $n;"
 
     ## password and permissions for MQ
     body_data=$(jq -n -c --arg password "$n" --arg tags none '$ARGS.named')
@@ -60,6 +61,11 @@ if [ ! -f "/shared/c4gh.sec.pem" ]; then
     curl -s -L https://github.com/neicnordic/crypt4gh/releases/download/v1.7.4/crypt4gh_linux_x86_64.tar.gz | tar -xz -C /shared/ && chmod +x /shared/crypt4gh
     /shared/crypt4gh generate -n /shared/c4gh -p c4ghpass
 fi
+if [ ! -f "/shared/sync.sec.pem" ]; then
+    echo "creating crypth4gh key"
+    curl -s -L https://github.com/neicnordic/crypt4gh/releases/download/v1.7.4/crypt4gh_linux_x86_64.tar.gz | tar -xz -C /shared/ && chmod +x /shared/crypt4gh
+    /shared/crypt4gh generate -n /shared/sync -p syncPass
+fi
 
 if [ ! -f "/shared/keys/ssh" ]; then
     ssh-keygen -o -a 256 -t ed25519 -f /shared/keys/ssh -N ""

.github/integration/sda-s3-integration.yml

@@ -208,6 +208,30 @@ services:
       - ./sda/config.yaml:/config.yaml
       - shared:/shared
 
+  sync:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [ sda-sync ]
+    container_name: sync
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      minio:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=sync
+      - BROKER_USER=sync
+      - BROKER_QUEUE=completed_stream
+      - DB_PASSWORD=sync
+      - DB_USER=sync
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
   oidc:
     container_name: oidc
     command:
@@ -250,6 +274,8 @@ services:
         condition: service_started
       s3inbox:
         condition: service_started
+      sync:
+        condition: service_started
       verify:
         condition: service_started
     environment:

.github/integration/sda/config.yaml

@@ -50,11 +50,20 @@ db:
 c4gh:
   filePath: /shared/c4gh.sec.pem
   passphrase: "c4ghpass"
+  syncPubKey: /shared/sync.pub.pem
 
 server:
   cert: ""
   key: ""
   jwtpubkeypath: "/shared/keys/pub/"
   jwtpubkeyurl: "http://oidc:8080/jwk"
 
-
+sync.destination:
+  type: "s3"
+  url: "http://s3"
+  port: 9000
+  readypath: "/minio/health/ready"
+  accessKey: "access"
+  secretKey: "secretKey"
+  bucket: "sync"
+  region: "us-east-1"

.github/integration/tests/sda/50_sync_test.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+cd shared || true
+
+# check DB for archive file names
+for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do
+    size=$(s3cmd -c direct ls s3://sync/test_dummy.org/"$file" | tr -s ' ' | cut -d ' ' -f 3)
+    if [ -z "$size" ]; then
+        echo "Failed to get size of $file from sync site"
+        exit 1
+    fi
+done
+
+echo "files synced successfully"