diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 688ea7dea..59ecdff77 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -52,6 +52,7 @@ services: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: - "15672:15672" + - "5672:5672" restart: always volumes: - rabbitmq_data:/var/lib/rabbitmq @@ -220,6 +221,8 @@ services: depends_on: credentials: condition: service_completed_successfully + extra_hosts: + - "localhost:host-gateway" healthcheck: test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] interval: 10s @@ -330,8 +333,12 @@ services: - AUTH_RESIGNJWT=false - OIDC_ID=XC56EL11xx - OIDC_SECRET=wHPVQaYXmdDHg + - OIDC_PROVIDER=http://localhost:8080 + - OIDC_REDIRECTURL=http://localhost:8889/oidc/login - DB_PASSWORD=auth - DB_USER=auth + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8889:8080" @@ -369,6 +376,8 @@ services: condition: service_started reencrypt: condition: service_started + extra_hosts: + - "localhost:host-gateway" environment: - PGPASSWORD=rootpasswd - STORAGETYPE=s3 diff --git a/.github/integration/sda/oidc.py b/.github/integration/sda/oidc.py index c7969b136..a6b387af9 100644 --- a/.github/integration/sda/oidc.py +++ b/.github/integration/sda/oidc.py @@ -51,7 +51,7 @@ def _generate_token() -> Tuple: # See available claims here: http://www.iana.org/assignments/jwt/jwt.xhtml # the important claim is the "authorities" header = { - "jku": f"{HTTP_PROTOCOL}://oidc:8080/jwk", + "jku": f"{HTTP_PROTOCOL}://localhost:8080/jwk", "alg": "ES256", "typ": "JWT", "kid": ec_key1.thumbprint() @@ -61,7 +61,7 @@ def _generate_token() -> Tuple: "aud": ["aud1", "aud2"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", @@ -71,21 +71,21 @@ def _generate_token() -> Tuple: "aud": ["aud2", "aud3"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", } empty_payload = { "sub": "requester@demo.org", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 99999999999, "iat": 1547794655, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", } # Craft passports passport_terms = { - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "sub": "requester@demo.org", "ga4gh_visa_v1": { "type": "AcceptedTermsAndPolicies", @@ -100,7 +100,7 @@ def _generate_token() -> Tuple: } # passport for dataset permissions 1 passport_dataset1 = { - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "sub": "requester@demo.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -165,12 +165,12 @@ def _generate_token() -> Tuple: async def fixed_response(request: web.Request) -> web.Response: global HTTP_PROTOCOL WELL_KNOWN = { - "issuer": f"{HTTP_PROTOCOL}://oidc:8080", - "authorization_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/authorize", - "registration_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/register", - "token_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/token", - "userinfo_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/userinfo", - "jwks_uri": f"{HTTP_PROTOCOL}://oidc:8080/jwk", + "issuer": f"{HTTP_PROTOCOL}://localhost:8080", + "authorization_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/authorize", + "registration_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/register", + "token_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/token", + "userinfo_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/userinfo", + "jwks_uri": f"{HTTP_PROTOCOL}://localhost:8080/jwk", "response_types_supported": [ "code", "id_token", diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 000000000..be6bdc999 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,101 @@ +# Run services with `go run` + +This section explains how to run some of the services using `go run` instead of the Docker setup to facilitate development. + +## Running `sda-download` with `go run` + +- Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: + +```sh +make integrationtest-sda-s3-run +``` + +- Change to the folder `sda-download` and start the `sda-download` service using: + +```sh +CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go +``` + +- Check if `sda-download` works as expected using: + +```sh +curl -o /dev/null -s -w "%{http_code}\n" http://localhost:18080/health +``` + +If successful, the curl command should output the HTTP code `200`. + +You can further check the endpoint `/metadata/datasets` using: + +```sh +token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') +curl -H "Authorization: Bearer $token" http://localhost:18080/metadata/datasets +``` + +If successful, the curl command should output a JSON body containing: + +```json +["EGAD74900000101"] +``` + +## Running other SDA services with `go run` + +Running any of the SDA services located in the `sda` subfolder requires that the service specific credentials and RabbitMQ configurations are set as ENVs. Here, we'll use `ingest` as an example. + +- Bring up all SDA services with the S3 backend by running the following command in the root folder of the repository: + +```sh +make sda-s3-up +``` + +- When the previous command is finished, bring down the `ingest` service using: + +```sh +docker stop ingest +``` + +- Copy keys and other information from the shared folder of the container using: + +```sh +docker cp verify:/shared /tmp/ +``` + +This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine, so that we have access to all the auto generated files that will be required. + +- Change to the folder `sda` and start the `ingest` service using: + +```sh +export BROKER_PASSWORD=ingest +export BROKER_USER=ingest +export BROKER_QUEUE=ingest +export BROKER_ROUTINGKEY=archived +export DB_PASSWORD=ingest +export DB_USER=ingest +CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go +``` + +- Check if the `ingest` service works as expected by following these steps + +```sh +# create a test file +seq 10 > /tmp/t1.txt + +# update the s3cmd config file +sed -i '/host_/s/s3inbox:8000/localhost:18000/g' /tmp/shared/s3cfg + +# upload /tmp/t1.txt to s3inbox by sda-cli +sda-cli -config /tmp/shared/s3cfg upload -encrypt-with-key /tmp/shared/c4gh.pub.pem /tmp/t1.txt + +# use sda-admin to check if t1.txt has been uploaded +export API_HOST=http://localhost:8090 +export ACCESS_TOKEN=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') +sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'uploaded' + +# register the Crypt4GH key +curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add + +# use sda-admin to ingest the file t1.txt +sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org + +# verify that t1.txt has been ingested using sda-admin +sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'verified' +``` diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index 41d3875fd..9b6b9f0d2 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -6,9 +6,10 @@ services: - -c - | cp /origcerts/* /certs - chown -R nobody.nobody /certs/* + chown -R nobody:nobody /certs/* + chmod -R 644 /certs/* chmod -R og-rw /certs/*-key.pem - chown -R 70.70 /certs/db* + chown -R 70:70 /certs/db* ls -la /certs/ container_name: certfixer diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 5ee4d5b0b..a3685c878 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -5,10 +5,10 @@ services: - -c - | cp /origcerts/* /certs - chown -R nobody.nobody /certs/* + chown -R nobody:nobody /certs/* chmod -R 644 /certs/* chmod -R og-rw /certs/*-key.pem - chown -R 70.70 /certs/db* + chown -R 70:70 /certs/db* ls -la /certs/ container_name: certfixer diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml new file mode 100644 index 000000000..d425077a8 --- /dev/null +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -0,0 +1,35 @@ +app: + serveUnencryptedData: true + port: 18080 + +log: + level: "debug" + format: "json" + +archive: + type: "s3" + # S3 backend + url: "http://localhost" + port: 19000 + accesskey: "access" + secretkey: "secretKey" + bucket: "archive" + region: "us-east-1" + chunksize: 32 + +grpc: + host: localhost + port: 50051 + +db: + host: "localhost" + port: 15432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +oidc: + # oidc configuration API must have values for "userinfo_endpoint" and "jwks_uri" + configuration: + url: "http://localhost:8080/.well-known/openid-configuration" diff --git a/sda/config_local.yaml b/sda/config_local.yaml new file mode 100644 index 000000000..81946299b --- /dev/null +++ b/sda/config_local.yaml @@ -0,0 +1,108 @@ +log: + format: "json" + level: "debug" +api: + rbacFile: ../.github/integration/sda/rbac.json + +archive: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "archive" + region: "us-east-1" + +auth: + cega: + authUrl: "http://localhost:8443/username/" + id: + secret: + infoText: "About service text" + infoURL: "http://example.org/about" + jwt: + issuer: "https://localhost:8888" + privateKey: "/tmp/shared/keys/jwt.key" + signatureAlg: ES256 + tokenTTL: 168 + publicFile: "/tmp/shared/c4gh.pub.pem" + resignJwt: + s3Inbox: "http://localhost:18000" + +backup: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "backup" + region: "us-east-1" + +inbox: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "inbox" + region: "us-east-1" + +broker: + host: "localhost" + port: "5672" + user: "" + password: "" + vhost: "/sda" + exchange: "sda" + routingKey: "" + ssl: "false" + +db: + host: "localhost" + port: "15432" + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +c4gh: + filePath: "/tmp/shared/c4gh.sec.pem" + passphrase: "c4ghpass" + syncPubKeyPath: "/tmp/shared/sync.pub.pem" + +oidc: + configuration: + url: "http://localhost:8080/.well-known/openid-configuration" + +server: + cert: "" + key: "" + jwtpubkeypath: "/tmp/shared/keys/pub/" + jwtpubkeyurl: "http://oidc:8080/jwk" + +sync: + api: + password: "pass" + user: "user" + centerPrefix: "SYNC" + destination: + type: "s3" + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "sync" + region: "us-east-1" + remote: + host: "http://sync-api" + port: "8080" + password: "pass" + user: "user" + +schema: + type: "isolated" + path: "schemas/isolated" \ No newline at end of file diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index cdc1472c5..984ccf967 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -933,6 +933,10 @@ func (c *Config) configSchemas() { } else { c.Broker.SchemasPath = "/schemas/isolated/" } + + if viper.IsSet("schema.path") { + c.Broker.SchemasPath = viper.GetString("schema.path") + } } // configS3Storage populates and returns a S3Conf from the