From b19b796d5df33e204e5965bd6109da83eaf9e8d5 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Fri, 20 Oct 2023 10:37:26 +0200
Subject: [PATCH] [Integration test] Add case for sync

---
 .../scripts/make_sda_credentials.sh           |  6 +++++
 .github/integration/sda-s3-integration.yml    | 26 +++++++++++++++++++
 .github/integration/sda/config.yaml           | 11 +++++++-
 .github/integration/tests/sda/50_sync_test.sh | 15 +++++++++++
 4 files changed, 57 insertions(+), 1 deletion(-)
 create mode 100644 .github/integration/tests/sda/50_sync_test.sh

diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh
index 5f0bd3f0a..fb1908a83 100644
--- a/.github/integration/scripts/make_sda_credentials.sh
+++ b/.github/integration/scripts/make_sda_credentials.sh
@@ -17,6 +17,7 @@ pip install aiohttp Authlib joserfc requests > /dev/null
 for n in download finalize inbox ingest mapper sync verify; do
     echo "creating credentials for: $n"
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
+    psql -U postgres -h postgres -d sda -c "GRANT base TO $n;"
 
     ## password and permissions for MQ
     body_data=$(jq -n -c --arg password "$n" --arg tags none '$ARGS.named')
@@ -60,6 +61,11 @@ if [ ! -f "/shared/c4gh.sec.pem" ]; then
     curl -s -L https://github.com/neicnordic/crypt4gh/releases/download/v1.7.4/crypt4gh_linux_x86_64.tar.gz | tar -xz -C /shared/ && chmod +x /shared/crypt4gh
     /shared/crypt4gh generate -n /shared/c4gh -p c4ghpass
 fi
+if [ ! -f "/shared/sync.sec.pem" ]; then
+    echo "creating crypth4gh key"
+    curl -s -L https://github.com/neicnordic/crypt4gh/releases/download/v1.7.4/crypt4gh_linux_x86_64.tar.gz | tar -xz -C /shared/ && chmod +x /shared/crypt4gh
+    /shared/crypt4gh generate -n /shared/sync -p syncPass
+fi
 
 if [ ! -f "/shared/keys/ssh" ]; then
     ssh-keygen -o -a 256 -t ed25519 -f /shared/keys/ssh -N ""
diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml
index d59e6abe7..3b6521efe 100644
--- a/.github/integration/sda-s3-integration.yml
+++ b/.github/integration/sda-s3-integration.yml
@@ -208,6 +208,30 @@ services:
       - ./sda/config.yaml:/config.yaml
       - shared:/shared
 
+  sync:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [ sda-sync ]
+    container_name: sync
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      minio:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=sync
+      - BROKER_USER=sync
+      - BROKER_QUEUE=completed_stream
+      - DB_PASSWORD=sync
+      - DB_USER=sync
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
   oidc:
     container_name: oidc
     command:
@@ -250,6 +274,8 @@ services:
         condition: service_started
       s3inbox:
         condition: service_started
+      sync:
+        condition: service_started
       verify:
         condition: service_started
     environment:
diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml
index 15e949f6e..58c097295 100644
--- a/.github/integration/sda/config.yaml
+++ b/.github/integration/sda/config.yaml
@@ -50,6 +50,7 @@ db:
 c4gh:
   filePath: /shared/c4gh.sec.pem
   passphrase: "c4ghpass"
+  syncPubKey: /shared/sync.pub.pem
 
 server:
   cert: ""
@@ -57,4 +58,12 @@ server:
   jwtpubkeypath: "/shared/keys/pub/"
   jwtpubkeyurl: "http://oidc:8080/jwk"
 
-
+sync.destination:
+  type: "s3"
+  url: "http://s3"
+  port: 9000
+  readypath: "/minio/health/ready"
+  accessKey: "access"
+  secretKey: "secretKey"
+  bucket: "sync"
+  region: "us-east-1"
diff --git a/.github/integration/tests/sda/50_sync_test.sh b/.github/integration/tests/sda/50_sync_test.sh
new file mode 100644
index 000000000..7370aaf79
--- /dev/null
+++ b/.github/integration/tests/sda/50_sync_test.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+cd shared || true
+
+# check DB for archive file names
+for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do
+    size=$(s3cmd -c direct ls s3://sync/test_dummy.org/"$file" | tr -s ' ' | cut -d ' ' -f 3)
+    if [ -z "$size" ]; then
+        echo "Failed to get size of $file from sync site"
+        exit 1
+    fi
+done
+
+echo "files synced successfully"
\ No newline at end of file