From 39beba68cd2ccbf5937ccc9f3fd3810d6a758699 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:00:50 +0100 Subject: [PATCH 01/18] add config file for running sda-download with go run --- .../dev_utils/config-notls_local.yaml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 sda-download/dev_utils/config-notls_local.yaml diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml new file mode 100644 index 000000000..11f9d768c --- /dev/null +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -0,0 +1,34 @@ +app: + serveUnencryptedData: true + +log: + level: "debug" + format: "json" + +archive: + type: "s3" + # S3 backend + url: "http://localhost" + port: 19000 + accesskey: "access" + secretkey: "secretKey" + bucket: "archive" + region: "us-east-1" + chunksize: 32 + +grpc: + host: localhost + port: 50051 + +db: + host: "localhost" + port: 15432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +oidc: + # oidc configuration API must have values for "userinfo_endpoint" and "jwks_uri" + configuration: + url: "http://localhost:8080/.well-known/openid-configuration" From 5b163879e898d98e3433f43f60ead0ce4006d529 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:13:09 +0100 Subject: [PATCH 02/18] expose API port for rabbitmq --- .github/integration/sda-s3-integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 688ea7dea..a4a900c59 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -52,6 +52,7 @@ services: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: - "15672:15672" + - "15671:5672" restart: always volumes: - rabbitmq_data:/var/lib/rabbitmq From 9591f5f86efd84215f251bcf76fa688cae2b3163 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:14:17 +0100 Subject: [PATCH 03/18] add config file for running sda services (except download) with go run --- sda/config_local.yaml | 107 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 sda/config_local.yaml diff --git a/sda/config_local.yaml b/sda/config_local.yaml new file mode 100644 index 000000000..21fe9b399 --- /dev/null +++ b/sda/config_local.yaml @@ -0,0 +1,107 @@ +log: + format: "json" + level: "debug" +api: + rbacFile: ../.github/integration/sda/rbac.json + +archive: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "archive" + region: "us-east-1" + +auth: + cega: + authUrl: "http://localhost:8443/username/" + id: + secret: + infoText: "About service text" + infoURL: "http://example.org/about" + jwt: + issuer: "https://localhost:8888" + privateKey: "tmp/shared/keys/jwt.key" + signatureAlg: ES256 + tokenTTL: 168 + publicFile: "tmp/shared/c4gh.pub.pem" + resignJwt: + s3Inbox: "http://localhost:18000" + +backup: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "backup" + region: "us-east-1" + +inbox: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "inbox" + region: "us-east-1" + +broker: + host: "localhost" + port: "15671" + user: "" + password: "" + vhost: "/sda" + exchange: "sda" + routingKey: "" + ssl: "false" + +db: + host: "localhost" + port: "15432" + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +c4gh: + filePath: "tmp/shared/c4gh.sec.pem" + passphrase: "c4ghpass" + syncPubKeyPath: "tmp/shared/sync.pub.pem" + +oidc: + configuration: + url: "http://localhost:8080/.well-known/openid-configuration" + +server: + cert: "" + key: "" + jwtpubkeypath: "tmp/shared/keys/pub/" + jwtpubkeyurl: "http://oidc:8080/jwk" + +sync: + api: + password: "pass" + user: "user" + centerPrefix: "SYNC" + destination: + type: "s3" + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "sync" + region: "us-east-1" + remote: + host: "http://sync-api" + port: "8080" + password: "pass" + user: "user" + +schema: + type: "isolated" \ No newline at end of file From 1fe05754961bab7a4fce549a9615aac40276d467 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:14:52 +0100 Subject: [PATCH 04/18] add development instructions using go run to run services --- DEVELOPMENT.md | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 DEVELOPMENT.md diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 000000000..fdd14b826 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,97 @@ +## Run services with `go run` + +This section explains how to run some of the services using `go run` instead of the Docker setup to facilitate development. + +### Running `sda-download` with `go run` +1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: +```sh +make integrationtest-sda-s3-run +``` + +2. Change to the folder `sda-download` and then create the credentials using: +```sh +pushd dev_utils +bash make_certs.sh +chmod 600 certs/*-key.* +popd +``` + +3. Update your `/etc/hosts` file: Add the following line to ensure the service can resolve OIDC locally: +``` +127.0.0.1 oidc +``` + +4. Start the `sda-download` service using: +```sh +APP_PORT=8553 CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go +``` + +5. Check if `sda-download` works as expected using: +```sh +token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') +curl -H "Authorization: Bearer $token" http://localhost:8553/metadata/datasets +``` +If successful, the curl command should output a JSON body containing: +```json +["EGAD74900000101"] +``` + + +### Running other SDA services with `go run` +Running other SDA services located in the `sda` subfolder, such as `ingest` or `verify`, differs slightly from running the `sda-download` service. Here, we'll use `ingest` as an example. + +1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository (this step is the same as for `sda-download`): +```sh +make integrationtest-sda-s3-run +``` + +2. Bring down the `ingest` service using: +```sh +PR_NUMBER=$(/bin/date +%F) docker compose -f .github/integration/sda-s3-integration.yml stop ingest +``` + +3. Change to the folder `sda`, then copy keys and other information from the shared folder of the container using: +```sh +mkdir -p tmp +docker cp verify:/shared tmp/ +``` +This will copy all data from the container's `/shared` folder to `tmp/shared` on your local machine. + +4. Copy schemas to `/schemas` on the localhost using: +```sh +sudo rsync -arvz schemas/ /schemas/ +``` + +5. Start the `ingest` service using: +```sh +export BROKER_PASSWORD=ingest +export BROKER_USER=ingest +export BROKER_QUEUE=ingest +export BROKER_ROUTINGKEY=archived +export DB_PASSWORD=ingest +export DB_USER=ingest +CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go +``` + +6. Check if the `ingest` service works as expected by following these steps +```sh +# create a test file +seq 10 > tmp/t1.txt + +# update the s3cmd config file +sed -i '/host_/s/s3inbox:8000/localhost:18000/g' tmp/shared/s3cfg + +# upload tmp/t1.txt to s3inbox by sda-cli +sda-cli -config tmp/shared/s3cfg upload -encrypt-with-key tmp/shared/c4gh.pub.pem tmp/t1.txt + +# use sda-admin to check if t1.txt has been uploaded +export API_HOST=http://localhost:8090 +export ACCESS_TOKEN=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') +sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'uploaded' + +# use sda-admin to ingest the file t1.txt +sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org + +# verify that t1.txt has been uploaded using sda-admin +sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'verified' +``` From c8a2c4e1aabc573598b039f60579a1882a901736 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:31:06 +0100 Subject: [PATCH 05/18] fix failed integration test with permission error on db-key.pem --- sda-download/dev_utils/compose-sda.yml | 7 ++++--- sda-download/dev_utils/compose.yml | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index 41d3875fd..fad8ca2d5 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -6,13 +6,14 @@ services: - -c - | cp /origcerts/* /certs - chown -R nobody.nobody /certs/* + chown -R nobody:nobody /certs/* + chmod -R 644 /certs/* chmod -R og-rw /certs/*-key.pem - chown -R 70.70 /certs/db* + chown -R 70:70 /certs/db* ls -la /certs/ container_name: certfixer - image: alpine:latest + image: alpine:latest volumes: - ./certs:/origcerts - certs:/certs diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 5ee4d5b0b..d43b3cdea 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -5,14 +5,14 @@ services: - -c - | cp /origcerts/* /certs - chown -R nobody.nobody /certs/* + chown -R nobody:nobody /certs/* chmod -R 644 /certs/* chmod -R og-rw /certs/*-key.pem - chown -R 70.70 /certs/db* + chown -R 70:70 /certs/db* ls -la /certs/ container_name: certfixer - image: alpine:latest + image: alpine:latest volumes: - ./certs:/origcerts - certs:/certs From abfbe1dccb296c1b5d67acb528d7831645bd0577 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 11:16:41 +0100 Subject: [PATCH 06/18] use other ports for rabbitmq --- .github/integration/sda-s3-integration.yml | 2 +- sda/config_local.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index a4a900c59..f8e357849 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -52,7 +52,7 @@ services: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: - "15672:15672" - - "15671:5672" + - "5672:5672" restart: always volumes: - rabbitmq_data:/var/lib/rabbitmq diff --git a/sda/config_local.yaml b/sda/config_local.yaml index 21fe9b399..67008f95a 100644 --- a/sda/config_local.yaml +++ b/sda/config_local.yaml @@ -52,7 +52,7 @@ inbox: broker: host: "localhost" - port: "15671" + port: "5672" user: "" password: "" vhost: "/sda" From f89580662d2d69ba719d622543360a752441493b Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 11:18:08 +0100 Subject: [PATCH 07/18] remove trailing whitespace --- sda-download/dev_utils/compose-sda.yml | 2 +- sda-download/dev_utils/compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index fad8ca2d5..9b6b9f0d2 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -13,7 +13,7 @@ services: ls -la /certs/ container_name: certfixer - image: alpine:latest + image: alpine:latest volumes: - ./certs:/origcerts - certs:/certs diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index d43b3cdea..a3685c878 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -12,7 +12,7 @@ services: ls -la /certs/ container_name: certfixer - image: alpine:latest + image: alpine:latest volumes: - ./certs:/origcerts - certs:/certs From 1ef45b71808721cfb83d18afd606dce646d58582 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 11:34:52 +0100 Subject: [PATCH 08/18] use the system /tmp instead of relative path tmp --- DEVELOPMENT.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index fdd14b826..7b2cb5a8e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -52,10 +52,9 @@ PR_NUMBER=$(/bin/date +%F) docker compose -f .github/integration/sda-s3-integra 3. Change to the folder `sda`, then copy keys and other information from the shared folder of the container using: ```sh -mkdir -p tmp -docker cp verify:/shared tmp/ +docker cp verify:/shared /tmp/ ``` -This will copy all data from the container's `/shared` folder to `tmp/shared` on your local machine. +This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine. 4. Copy schemas to `/schemas` on the localhost using: ```sh @@ -76,13 +75,13 @@ CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go 6. Check if the `ingest` service works as expected by following these steps ```sh # create a test file -seq 10 > tmp/t1.txt +seq 10 > /tmp/t1.txt # update the s3cmd config file -sed -i '/host_/s/s3inbox:8000/localhost:18000/g' tmp/shared/s3cfg +sed -i '/host_/s/s3inbox:8000/localhost:18000/g' /tmp/shared/s3cfg -# upload tmp/t1.txt to s3inbox by sda-cli -sda-cli -config tmp/shared/s3cfg upload -encrypt-with-key tmp/shared/c4gh.pub.pem tmp/t1.txt +# upload /tmp/t1.txt to s3inbox by sda-cli +sda-cli -config /tmp/shared/s3cfg upload -encrypt-with-key /tmp/shared/c4gh.pub.pem /tmp/t1.txt # use sda-admin to check if t1.txt has been uploaded export API_HOST=http://localhost:8090 From 4b2c024ffcd24b7970d2352897b129527de809bf Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 15:27:43 +0100 Subject: [PATCH 09/18] add support for a customizable schemas path --- sda/internal/config/config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index cdc1472c5..984ccf967 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -933,6 +933,10 @@ func (c *Config) configSchemas() { } else { c.Broker.SchemasPath = "/schemas/isolated/" } + + if viper.IsSet("schema.path") { + c.Broker.SchemasPath = viper.GetString("schema.path") + } } // configS3Storage populates and returns a S3Conf from the From 92f23fd23e3a50d7c1e4280d002acf9d4301b5d4 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 15:29:04 +0100 Subject: [PATCH 10/18] update config and instructions for using custom schemas path --- DEVELOPMENT.md | 9 ++------- sda/config_local.yaml | 3 ++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7b2cb5a8e..bad3f628c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -56,12 +56,7 @@ docker cp verify:/shared /tmp/ ``` This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine. -4. Copy schemas to `/schemas` on the localhost using: -```sh -sudo rsync -arvz schemas/ /schemas/ -``` - -5. Start the `ingest` service using: +4. Start the `ingest` service using: ```sh export BROKER_PASSWORD=ingest export BROKER_USER=ingest @@ -72,7 +67,7 @@ export DB_USER=ingest CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go ``` -6. Check if the `ingest` service works as expected by following these steps +5. Check if the `ingest` service works as expected by following these steps ```sh # create a test file seq 10 > /tmp/t1.txt diff --git a/sda/config_local.yaml b/sda/config_local.yaml index 67008f95a..332048b08 100644 --- a/sda/config_local.yaml +++ b/sda/config_local.yaml @@ -104,4 +104,5 @@ sync: user: "user" schema: - type: "isolated" \ No newline at end of file + type: "isolated" + path: "schemas/isolated" \ No newline at end of file From e32734df4e6a96a5c24264eb347a857071403be7 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 10:37:45 +0100 Subject: [PATCH 11/18] add port number in the config file --- sda-download/dev_utils/config-notls_local.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml index 11f9d768c..d425077a8 100644 --- a/sda-download/dev_utils/config-notls_local.yaml +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -1,5 +1,6 @@ app: serveUnencryptedData: true + port: 18080 log: level: "debug" From 8c74f9dd3a9ebd7d5f361d3a22a675f9052af442 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 10:38:51 +0100 Subject: [PATCH 12/18] update instructions based on reviewers comments --- DEVELOPMENT.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index bad3f628c..5b4512a17 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -23,13 +23,19 @@ popd 4. Start the `sda-download` service using: ```sh -APP_PORT=8553 CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go +CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go ``` 5. Check if `sda-download` works as expected using: ```sh +curl -o /dev/null -s -w "%{http_code}\n" http://localhost:18080/health +``` +If successful, the curl command should output the HTTP code `200`. + +You can further check the endpoint `/metadata/datasets` using: +```sh token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') -curl -H "Authorization: Bearer $token" http://localhost:8553/metadata/datasets +curl -H "Authorization: Bearer $token" http://localhost:18080/metadata/datasets ``` If successful, the curl command should output a JSON body containing: ```json @@ -40,7 +46,7 @@ If successful, the curl command should output a JSON body containing: ### Running other SDA services with `go run` Running other SDA services located in the `sda` subfolder, such as `ingest` or `verify`, differs slightly from running the `sda-download` service. Here, we'll use `ingest` as an example. -1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository (this step is the same as for `sda-download`): +1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: ```sh make integrationtest-sda-s3-run ``` From 5e2d09c0bab31a8fbf4524e5c521eddb3393789f Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 10:42:05 +0100 Subject: [PATCH 13/18] change from relative path tmp to /tmp --- sda/config_local.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sda/config_local.yaml b/sda/config_local.yaml index 332048b08..81946299b 100644 --- a/sda/config_local.yaml +++ b/sda/config_local.yaml @@ -23,10 +23,10 @@ auth: infoURL: "http://example.org/about" jwt: issuer: "https://localhost:8888" - privateKey: "tmp/shared/keys/jwt.key" + privateKey: "/tmp/shared/keys/jwt.key" signatureAlg: ES256 tokenTTL: 168 - publicFile: "tmp/shared/c4gh.pub.pem" + publicFile: "/tmp/shared/c4gh.pub.pem" resignJwt: s3Inbox: "http://localhost:18000" @@ -69,9 +69,9 @@ db: sslmode: "disable" c4gh: - filePath: "tmp/shared/c4gh.sec.pem" + filePath: "/tmp/shared/c4gh.sec.pem" passphrase: "c4ghpass" - syncPubKeyPath: "tmp/shared/sync.pub.pem" + syncPubKeyPath: "/tmp/shared/sync.pub.pem" oidc: configuration: @@ -80,7 +80,7 @@ oidc: server: cert: "" key: "" - jwtpubkeypath: "tmp/shared/keys/pub/" + jwtpubkeypath: "/tmp/shared/keys/pub/" jwtpubkeyurl: "http://oidc:8080/jwk" sync: From 0b1261ef106844f9e9a8bdf75ac61aa1180a8d8a Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 12:13:02 +0100 Subject: [PATCH 14/18] remove instructions for creating credentials for notls --- DEVELOPMENT.md | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5b4512a17..86b92ffa6 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -8,25 +8,16 @@ This section explains how to run some of the services using `go run` instead of make integrationtest-sda-s3-run ``` -2. Change to the folder `sda-download` and then create the credentials using: -```sh -pushd dev_utils -bash make_certs.sh -chmod 600 certs/*-key.* -popd -``` - -3. Update your `/etc/hosts` file: Add the following line to ensure the service can resolve OIDC locally: -``` -127.0.0.1 oidc -``` - -4. Start the `sda-download` service using: +2. Change to the folder `sda-download` and start the `sda-download` service using: ```sh CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go ``` +> If the host `oidc` cannot be accessed on the local machine, update your `/etc/hosts` file by adding the following line to ensure the service can resolve `oidc` locally: +> ``` +> 127.0.0.1 oidc +> ``` -5. Check if `sda-download` works as expected using: +3. Check if `sda-download` works as expected using: ```sh curl -o /dev/null -s -w "%{http_code}\n" http://localhost:18080/health ``` From f1a79844a99b2dda8db4c3c5e46316ff51f45fb8 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 10 Dec 2024 13:09:22 +0100 Subject: [PATCH 15/18] Rework oidc.py to work when calling from the users shell --- .github/integration/sda-s3-integration.yml | 8 ++++++++ .github/integration/sda/oidc.py | 24 +++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index f8e357849..59ecdff77 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -221,6 +221,8 @@ services: depends_on: credentials: condition: service_completed_successfully + extra_hosts: + - "localhost:host-gateway" healthcheck: test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] interval: 10s @@ -331,8 +333,12 @@ services: - AUTH_RESIGNJWT=false - OIDC_ID=XC56EL11xx - OIDC_SECRET=wHPVQaYXmdDHg + - OIDC_PROVIDER=http://localhost:8080 + - OIDC_REDIRECTURL=http://localhost:8889/oidc/login - DB_PASSWORD=auth - DB_USER=auth + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8889:8080" @@ -370,6 +376,8 @@ services: condition: service_started reencrypt: condition: service_started + extra_hosts: + - "localhost:host-gateway" environment: - PGPASSWORD=rootpasswd - STORAGETYPE=s3 diff --git a/.github/integration/sda/oidc.py b/.github/integration/sda/oidc.py index c7969b136..a6b387af9 100644 --- a/.github/integration/sda/oidc.py +++ b/.github/integration/sda/oidc.py @@ -51,7 +51,7 @@ def _generate_token() -> Tuple: # See available claims here: http://www.iana.org/assignments/jwt/jwt.xhtml # the important claim is the "authorities" header = { - "jku": f"{HTTP_PROTOCOL}://oidc:8080/jwk", + "jku": f"{HTTP_PROTOCOL}://localhost:8080/jwk", "alg": "ES256", "typ": "JWT", "kid": ec_key1.thumbprint() @@ -61,7 +61,7 @@ def _generate_token() -> Tuple: "aud": ["aud1", "aud2"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", @@ -71,21 +71,21 @@ def _generate_token() -> Tuple: "aud": ["aud2", "aud3"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", } empty_payload = { "sub": "requester@demo.org", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 99999999999, "iat": 1547794655, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", } # Craft passports passport_terms = { - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "sub": "requester@demo.org", "ga4gh_visa_v1": { "type": "AcceptedTermsAndPolicies", @@ -100,7 +100,7 @@ def _generate_token() -> Tuple: } # passport for dataset permissions 1 passport_dataset1 = { - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "sub": "requester@demo.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -165,12 +165,12 @@ def _generate_token() -> Tuple: async def fixed_response(request: web.Request) -> web.Response: global HTTP_PROTOCOL WELL_KNOWN = { - "issuer": f"{HTTP_PROTOCOL}://oidc:8080", - "authorization_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/authorize", - "registration_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/register", - "token_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/token", - "userinfo_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/userinfo", - "jwks_uri": f"{HTTP_PROTOCOL}://oidc:8080/jwk", + "issuer": f"{HTTP_PROTOCOL}://localhost:8080", + "authorization_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/authorize", + "registration_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/register", + "token_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/token", + "userinfo_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/userinfo", + "jwks_uri": f"{HTTP_PROTOCOL}://localhost:8080/jwk", "response_types_supported": [ "code", "id_token", From 2e1d482c2c06ddc50b1a1b0ed014484b2ca9ff95 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 15:41:21 +0100 Subject: [PATCH 16/18] update instructions --- DEVELOPMENT.md | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 86b92ffa6..437849d8e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,59 +1,68 @@ -## Run services with `go run` +# Run services with `go run` This section explains how to run some of the services using `go run` instead of the Docker setup to facilitate development. -### Running `sda-download` with `go run` -1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: +## Running `sda-download` with `go run` + +- Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: + ```sh make integrationtest-sda-s3-run ``` -2. Change to the folder `sda-download` and start the `sda-download` service using: +- Change to the folder `sda-download` and start the `sda-download` service using: + ```sh CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go ``` -> If the host `oidc` cannot be accessed on the local machine, update your `/etc/hosts` file by adding the following line to ensure the service can resolve `oidc` locally: -> ``` -> 127.0.0.1 oidc -> ``` -3. Check if `sda-download` works as expected using: +- Check if `sda-download` works as expected using: + ```sh curl -o /dev/null -s -w "%{http_code}\n" http://localhost:18080/health ``` + If successful, the curl command should output the HTTP code `200`. You can further check the endpoint `/metadata/datasets` using: + ```sh token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') curl -H "Authorization: Bearer $token" http://localhost:18080/metadata/datasets ``` + If successful, the curl command should output a JSON body containing: + ```json ["EGAD74900000101"] ``` +## Running other SDA services with `go run` + +Running any of the SDA services located in the `sda` subfolder requires that the service specific credentials and RabbitMQ configurations are set as ENVs. Here, we'll use `ingest` as an example. -### Running other SDA services with `go run` -Running other SDA services located in the `sda` subfolder, such as `ingest` or `verify`, differs slightly from running the `sda-download` service. Here, we'll use `ingest` as an example. +- Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: -1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: ```sh make integrationtest-sda-s3-run ``` -2. Bring down the `ingest` service using: +- When the previous command is finished, bring down the `ingest` service using: + ```sh -PR_NUMBER=$(/bin/date +%F) docker compose -f .github/integration/sda-s3-integration.yml stop ingest +docker stop ingest ``` -3. Change to the folder `sda`, then copy keys and other information from the shared folder of the container using: +- Copy keys and other information from the shared folder of the container using: + ```sh docker cp verify:/shared /tmp/ ``` -This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine. -4. Start the `ingest` service using: +This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine, so that we have access to all the required auto generated files that will be required. + +- Change to the folder `sda` and start the `ingest` service using: + ```sh export BROKER_PASSWORD=ingest export BROKER_USER=ingest @@ -64,7 +73,8 @@ export DB_USER=ingest CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go ``` -5. Check if the `ingest` service works as expected by following these steps +- Check if the `ingest` service works as expected by following these steps + ```sh # create a test file seq 10 > /tmp/t1.txt @@ -83,6 +93,6 @@ sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'u # use sda-admin to ingest the file t1.txt sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org -# verify that t1.txt has been uploaded using sda-admin +# verify that t1.txt has been ingested using sda-admin sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'verified' ``` From f3c1537c949f0f977f960fda4f3876de0d09e21b Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 21:56:51 +0100 Subject: [PATCH 17/18] update instructions for file ingestions --- DEVELOPMENT.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 437849d8e..c952962dc 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -41,10 +41,10 @@ If successful, the curl command should output a JSON body containing: Running any of the SDA services located in the `sda` subfolder requires that the service specific credentials and RabbitMQ configurations are set as ENVs. Here, we'll use `ingest` as an example. -- Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: +- Bring up all SDA services with the S3 backend by running the following command in the root folder of the repository: ```sh -make integrationtest-sda-s3-run +make sda-s3-up ``` - When the previous command is finished, bring down the `ingest` service using: @@ -88,11 +88,14 @@ sda-cli -config /tmp/shared/s3cfg upload -encrypt-with-key /tmp/shared/c4gh.pub. # use sda-admin to check if t1.txt has been uploaded export API_HOST=http://localhost:8090 export ACCESS_TOKEN=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') -sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'uploaded' +sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'uploaded' + +# register the Crypt4GH key +curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add # use sda-admin to ingest the file t1.txt sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org # verify that t1.txt has been ingested using sda-admin -sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'verified' +sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'verified' ``` From de39a95a781a09d5e73f19ea3a9dffe4c4931444 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 11 Dec 2024 11:07:19 +0100 Subject: [PATCH 18/18] readme: grammatical correction Co-authored-by: Joakim Bygdell --- DEVELOPMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c952962dc..be6bdc999 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -59,7 +59,7 @@ docker stop ingest docker cp verify:/shared /tmp/ ``` -This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine, so that we have access to all the required auto generated files that will be required. +This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine, so that we have access to all the auto generated files that will be required. - Change to the folder `sda` and start the `ingest` service using: