Run services manually with go run

.github/integration/sda-s3-integration.yml

@@ -52,6 +52,7 @@ services:
     image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
     ports:
       - "15672:15672"
+      - "5672:5672"
     restart: always
     volumes:
       - rabbitmq_data:/var/lib/rabbitmq
@@ -220,6 +221,8 @@ services:
     depends_on:
       credentials:
         condition: service_completed_successfully
+    extra_hosts:
+      - "localhost:host-gateway"
     healthcheck:
       test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)']
       interval: 10s
@@ -330,8 +333,12 @@ services:
       - AUTH_RESIGNJWT=false
       - OIDC_ID=XC56EL11xx
       - OIDC_SECRET=wHPVQaYXmdDHg
+      - OIDC_PROVIDER=http://localhost:8080
+      - OIDC_REDIRECTURL=http://localhost:8889/oidc/login
       - DB_PASSWORD=auth
       - DB_USER=auth
+    extra_hosts:
+      - "localhost:host-gateway"
     image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
     ports:
       - "8889:8080"
@@ -369,6 +376,8 @@ services:
          condition: service_started
       reencrypt:
          condition: service_started
+    extra_hosts:
+      - "localhost:host-gateway"
     environment:
       - PGPASSWORD=rootpasswd
       - STORAGETYPE=s3

.github/integration/sda/oidc.py

@@ -51,7 +51,7 @@ def _generate_token() -> Tuple:
     # See available claims here: http://www.iana.org/assignments/jwt/jwt.xhtml
     # the important claim is the "authorities"
     header = {
-        "jku": f"{HTTP_PROTOCOL}://oidc:8080/jwk",
+        "jku": f"{HTTP_PROTOCOL}://localhost:8080/jwk",
         "alg": "ES256",
         "typ": "JWT",
         "kid": ec_key1.thumbprint()
@@ -61,7 +61,7 @@ def _generate_token() -> Tuple:
         "aud": ["aud1", "aud2"],
         "azp": "azp",
         "scope": "openid ga4gh_passport_v1",
-        "iss": "https://oidc:8080/",
+        "iss": "https://localhost:8080/",
         "exp": 9999999999,
         "iat": 1561621913,
         "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102",
@@ -71,21 +71,21 @@ def _generate_token() -> Tuple:
         "aud": ["aud2", "aud3"],
         "azp": "azp",
         "scope": "openid ga4gh_passport_v1",
-        "iss": "https://oidc:8080/",
+        "iss": "https://localhost:8080/",
         "exp": 9999999999,
         "iat": 1561621913,
         "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102",
     }
     empty_payload = {
         "sub": "[email protected]",
-        "iss": "https://oidc:8080/",
+        "iss": "https://localhost:8080/",
         "exp": 99999999999,
         "iat": 1547794655,
         "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102",
     }
     # Craft passports
     passport_terms = {
-        "iss": "https://oidc:8080/",
+        "iss": "https://localhost:8080/",
         "sub": "[email protected]",
         "ga4gh_visa_v1": {
             "type": "AcceptedTermsAndPolicies",
@@ -100,7 +100,7 @@ def _generate_token() -> Tuple:
     }
     # passport for dataset permissions 1
     passport_dataset1 = {
-        "iss": "https://oidc:8080/",
+        "iss": "https://localhost:8080/",
         "sub": "[email protected]",
         "ga4gh_visa_v1": {
             "type": "ControlledAccessGrants",
@@ -165,12 +165,12 @@ def _generate_token() -> Tuple:
 async def fixed_response(request: web.Request) -> web.Response:
     global HTTP_PROTOCOL
     WELL_KNOWN = {
-        "issuer": f"{HTTP_PROTOCOL}://oidc:8080",
-        "authorization_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/authorize",
-        "registration_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/register",
-        "token_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/token",
-        "userinfo_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/userinfo",
-        "jwks_uri": f"{HTTP_PROTOCOL}://oidc:8080/jwk",
+        "issuer": f"{HTTP_PROTOCOL}://localhost:8080",
+        "authorization_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/authorize",
+        "registration_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/register",
+        "token_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/token",
+        "userinfo_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/userinfo",
+        "jwks_uri": f"{HTTP_PROTOCOL}://localhost:8080/jwk",
         "response_types_supported": [
             "code",
             "id_token",

DEVELOPMENT.md

@@ -0,0 +1,101 @@
+# Run services with `go run`
+
+This section explains how to run some of the services using `go run` instead of the Docker setup to facilitate development.
+
+## Running `sda-download` with `go run`
+
+- Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository:
+
+```sh
+make integrationtest-sda-s3-run 
+```
+
+- Change to the folder `sda-download` and start the `sda-download` service using:
+
+```sh
+CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go
+```
+
+- Check if `sda-download` works as expected using:
+
+```sh
+curl -o /dev/null -s -w "%{http_code}\n" http://localhost:18080/health
+```
+
+If successful, the curl command should output the HTTP code `200`.
+
+You can further check the endpoint `/metadata/datasets` using:
+
+```sh
+token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') 
+curl -H "Authorization: Bearer $token" http://localhost:18080/metadata/datasets
+```
+
+If successful, the curl command should output a JSON body containing:
+
+```json
+["EGAD74900000101"]
+```
+
+## Running other SDA services with `go run`
+
+Running any of the SDA services located in the `sda` subfolder requires that the service specific credentials and RabbitMQ configurations are set as ENVs. Here, we'll use `ingest` as an example.
+
+- Bring up all SDA services with the S3 backend by running the following command in the root folder of the repository:
+
+```sh
+make sda-s3-up
+```
+
+- When the previous command is finished, bring down the `ingest` service using:
+
+```sh
+docker stop ingest
+```
+
+- Copy keys and other information from the shared folder of the container using:
+
+```sh
+docker cp verify:/shared /tmp/
+```
+
+This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine, so that we have access to all the auto generated files that will be required.
+
+- Change to the folder `sda` and start the `ingest` service using:
+
+```sh
+export BROKER_PASSWORD=ingest
+export BROKER_USER=ingest
+export BROKER_QUEUE=ingest
+export BROKER_ROUTINGKEY=archived
+export DB_PASSWORD=ingest
+export DB_USER=ingest 
+CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go
+```
+
+- Check if the `ingest` service works as expected by following these steps
+
+```sh
+# create a test file
+seq 10 > /tmp/t1.txt
+
+# update the s3cmd config file
+sed -i '/host_/s/s3inbox:8000/localhost:18000/g' /tmp/shared/s3cfg
+
+# upload /tmp/t1.txt to s3inbox by sda-cli
+sda-cli -config /tmp/shared/s3cfg upload -encrypt-with-key /tmp/shared/c4gh.pub.pem /tmp/t1.txt
+
+# use sda-admin to check if t1.txt has been uploaded
+export API_HOST=http://localhost:8090
+export ACCESS_TOKEN=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') 
+sda-admin file list -user [email protected] # file test_dummy.org/t1.txt.c4gh should have fileStatus 'uploaded'
+
+# register the Crypt4GH key
+curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add
+
+# use sda-admin to ingest the file t1.txt
+sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user [email protected]  
+
+# verify that t1.txt has been ingested using sda-admin
+sda-admin file list -user [email protected] # file test_dummy.org/t1.txt.c4gh should have fileStatus 'verified'
+```

sda-download/dev_utils/compose-sda.yml

@@ -6,9 +6,10 @@ services:
       - -c
       - |
         cp /origcerts/* /certs
-        chown -R nobody.nobody /certs/*
+        chown -R nobody:nobody /certs/*
+        chmod -R 644 /certs/*
         chmod -R og-rw /certs/*-key.pem
-        chown -R 70.70 /certs/db*
+        chown -R 70:70 /certs/db*
         ls -la /certs/
 
     container_name: certfixer

sda-download/dev_utils/compose.yml

@@ -5,10 +5,10 @@ services:
       - -c
       - |
         cp /origcerts/* /certs
-        chown -R nobody.nobody /certs/*
+        chown -R nobody:nobody /certs/*
         chmod -R 644 /certs/*
         chmod -R og-rw /certs/*-key.pem
-        chown -R 70.70 /certs/db*
+        chown -R 70:70 /certs/db*
         ls -la /certs/
 
     container_name: certfixer

sda-download/dev_utils/config-notls_local.yaml

@@ -0,0 +1,35 @@
+app:
+  serveUnencryptedData: true
+  port: 18080
+
+log:
+  level: "debug"
+  format: "json"
+
+archive:
+  type: "s3"
+  # S3 backend
+  url: "http://localhost"
+  port: 19000
+  accesskey: "access"
+  secretkey: "secretKey"
+  bucket: "archive"
+  region: "us-east-1"
+  chunksize: 32
+
+grpc:
+  host: localhost 
+  port: 50051
+
+db:
+  host: "localhost"
+  port: 15432
+  user: "postgres"
+  password: "rootpasswd"
+  database: "sda"
+  sslmode: "disable"
+
+oidc:
+  # oidc configuration API must have values for "userinfo_endpoint" and "jwks_uri"
+  configuration:
+    url: "http://localhost:8080/.well-known/openid-configuration"

sda/config_local.yaml

@@ -0,0 +1,108 @@
+log:
+  format: "json"
+  level: "debug"
+api:
+  rbacFile: ../.github/integration/sda/rbac.json 
+
+archive:
+  type: s3
+  url: "http://localhost"
+  port: 19000
+  readypath: "/minio/health/ready"
+  accessKey: "access"
+  secretKey: "secretKey"
+  bucket: "archive"
+  region: "us-east-1"
+
+auth:
+  cega:
+    authUrl: "http://localhost:8443/username/"
+    id:
+    secret:
+  infoText: "About service text"
+  infoURL: "http://example.org/about"
+  jwt:
+    issuer: "https://localhost:8888"
+    privateKey: "/tmp/shared/keys/jwt.key"
+    signatureAlg: ES256
+    tokenTTL: 168
+  publicFile: "/tmp/shared/c4gh.pub.pem"
+  resignJwt:
+  s3Inbox: "http://localhost:18000"
+
+backup:
+  type: s3
+  url: "http://localhost"
+  port: 19000
+  readypath: "/minio/health/ready"
+  accessKey: "access"
+  secretKey: "secretKey"
+  bucket: "backup"
+  region: "us-east-1"
+
+inbox:
+  type: s3
+  url: "http://localhost"
+  port: 19000
+  readypath: "/minio/health/ready"
+  accessKey: "access"
+  secretKey: "secretKey"
+  bucket: "inbox"
+  region: "us-east-1"
+
+broker:
+  host: "localhost"
+  port: "5672"
+  user: ""
+  password: ""
+  vhost: "/sda"
+  exchange: "sda"
+  routingKey: ""
+  ssl: "false"
+
+db:
+  host: "localhost"
+  port: "15432"
+  user: "postgres"
+  password: "rootpasswd"
+  database: "sda"
+  sslmode: "disable"
+
+c4gh:
+  filePath: "/tmp/shared/c4gh.sec.pem"
+  passphrase: "c4ghpass"
+  syncPubKeyPath: "/tmp/shared/sync.pub.pem"
+
+oidc:
+  configuration:
+    url: "http://localhost:8080/.well-known/openid-configuration"
+
+server:
+  cert: ""
+  key: ""
+  jwtpubkeypath: "/tmp/shared/keys/pub/"
+  jwtpubkeyurl: "http://oidc:8080/jwk"
+
+sync:
+  api:
+    password: "pass"
+    user: "user"
+  centerPrefix: "SYNC"
+  destination:
+    type: "s3"
+    url: "http://localhost"
+    port: 19000
+    readypath: "/minio/health/ready"
+    accessKey: "access"
+    secretKey: "secretKey"
+    bucket: "sync"
+    region: "us-east-1"
+  remote:
+    host: "http://sync-api"
+    port: "8080"
+    password: "pass"
+    user: "user"
+
+schema:
+  type: "isolated"
+  path: "schemas/isolated"

sda/internal/config/config.go

@@ -933,6 +933,10 @@ func (c *Config) configSchemas() {
 	} else {
 		c.Broker.SchemasPath = "/schemas/isolated/"
 	}
+
+	if viper.IsSet("schema.path") {
+		c.Broker.SchemasPath = viper.GetString("schema.path")
+	}
 }
 
 // configS3Storage populates and returns a S3Conf from the