test: add rucio to the integration_tests.py

integration_tests.py

@@ -253,7 +253,7 @@ def prepare_environment(
     typer.secho("Running docker compose to create containers", fg=c.GREEN)
     with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn:
         subprocess.run(
-            ["docker", "compose", "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client", "dirac-pilot"]
+            ["docker", "compose", "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client", "dirac-pilot", "rucio-server"]
             + extra_services,
             check=True,
             env=docker_compose_env,
@@ -655,7 +655,7 @@ def _gen_docker_compose(modules, *, diracx_dist_dir=None):
     input_fn = Path(__file__).parent / "tests/CI/docker-compose.yml"
     docker_compose = yaml.safe_load(input_fn.read_text())
     # diracx-wait-for-db needs the volume to be able to run the witing script
-    for ctn in ("dirac-server", "dirac-client", "dirac-pilot", "diracx-wait-for-db"):
+    for ctn in ("dirac-server", "dirac-client", "dirac-pilot", "diracx-wait-for-db", "rucio-wait-for-db"):
         if "volumes" not in docker_compose["services"][ctn]:
             docker_compose["services"][ctn]["volumes"] = []
     volumes = [f"{path}:/home/dirac/LocalRepo/ALTERNATIVE_MODULES/{name}" for name, path in modules.items()]
@@ -664,6 +664,7 @@ def _gen_docker_compose(modules, *, diracx_dist_dir=None):
     docker_compose["services"]["dirac-client"]["volumes"].extend(volumes[:])
     docker_compose["services"]["dirac-pilot"]["volumes"].extend(volumes[:])
     docker_compose["services"]["diracx-wait-for-db"]["volumes"].extend(volumes[:])
+    docker_compose["services"]["rucio-wait-for-db"]["volumes"].extend(volumes[:])
 
     module_configs = _load_module_configs(modules)
     if diracx_dist_dir is not None:

tests/CI/check_rucio_db_initialised.sh

@@ -0,0 +1,24 @@
+# Should be called with PGPASSWORD=mysecretpassword
+
+#!/bin/bash
+dbMissing=true;
+allDBs=(rucio)
+while ${dbMissing};
+do
+    dbMissing=false;
+    allExistingDBs=$(psql -U postgres -h rucio-db -p 5432 -c "\l");
+    for db in "${allDBs[@]}";
+    do
+        if grep -q "${db}" <<< "${allExistingDBs}";
+        then
+            echo "${db} OK";
+        else
+            echo "${db} not created";
+            dbMissing=true;
+        fi;
+    done;
+    if ${dbMissing};
+    then
+        sleep 1;
+    fi
+done

tests/CI/docker-compose.yml

@@ -5,8 +5,90 @@ volumes:
   diracx-key-store:
   # Volume used to store the jwks of the IAM service
   diracx-iam-key-store:
+  # Volume used to store the rucio config
+  rucio-server-config:
 
 services:
+  rucio-db:
+    hostname: rucio-db
+    image: "postgres:15"
+    environment:
+      POSTGRES_USER: rucio
+      POSTGRES_DB: rucio
+      POSTGRES_PASSWORD: secret
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"]
+      timeout: 10s
+      start_period: 30s
+    pull_policy: always
+
+  rucio-wait-for-db:
+    image: "postgres:15"
+    container_name: rucio-wait-for-db
+    environment:
+      POSTGRES_USER: rucio
+      POSTGRES_DB: rucio
+      PGPASSWORD: secret
+    depends_on:
+      rucio-db:
+        condition: service_healthy
+    command: /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/check_rucio_db_initialized.sh
+    pull_policy: always
+
+  rucio-init:
+    profiles:
+      - init
+    image: rucio/rucio-init:release-36.1.0
+    environment:
+      RUCIO_CFG_DATABASE_DEFAULT: "postgresql://rucio:secret@rucio-db/rucio"
+      RUCIO_CFG_BOOTSTRAP_USERPASS_IDENTITY: "test-user"
+      RUCIO_CFG_BOOTSTRAP_USERPASS_PWD: "secret"
+      RUCIO_CFG_BOOTSTRAP_X509_IDENTITY: "CN=Test User"
+      RUCIO_CFG_BOOTSTRAP_X509_EMAIL: "[email protected]"
+      RUCIO_CFG_POLICY_PACKAGE: "dirac_rucio_policy"
+      RUCIO_CFG_COMMON_EXTRACT_SCOPE: "dirac"
+      RUCIO_CFG_POLICY_LFN2PFN_ALGORITHM_DEFAULT: "dirac"
+      PYTHONPATH: /opt/rucio
+    depends_on:
+      rucio-db:
+        condition: service_healthy
+    volumes:
+      - rucio-server-config:/opt/rucio/etc/
+      - ./rucio_policy:/opt/rucio/dirac_rucio_policy
+
+  rucio-server:
+    hostname: rucio-server
+    image: rucio/rucio-server:release-36.1.0
+    environment:
+      RUCIO_ENABLE_SSL: "True"
+      RUCIO_HTTPD_ENCODED_SLASHES: "True"
+      RUCIO_HTTPD_GRID_SITE_ENABLED: "True"
+    depends_on:
+      rucio-db:
+        condition: service_healthy
+      rucio-storage-1:
+        condition: service_started
+    volumes:
+      - rucio-server-config:/opt/rucio/etc/
+      - ./certs/hostcert_rucio-server.pem:/etc/grid-security/hostcert.pem:z
+      - ./certs/hostcert_rucio-server.key.pem:/etc/grid-security/hostkey.pem:z
+      - ./certs/test_ca.pem:/etc/grid-security/ca.pem:z
+      - ${RUCIO_REPOSITORY:-/dev/null}:/src/rucio
+
+  rucio-storage-1:
+    hostname: rucio-storage-1
+    image: rucio/xrootd
+    environment:
+      XRDPORT: "1094"
+    volumes:
+      - ./certs/test_ca.pem:/etc/grid-security/certificates/37cc428d.0
+      - ./certs/hostcert_rucio-storage-1.pem:/tmp/xrdcert.pem:z
+      - ./certs/hostcert_rucio-storage-1.key.pem:/tmp/xrdkey.pem:z
+    ulimits:
+      nofile:
+        soft: 1024
+        hard: 2048
+
   mysql:
     image: ${MYSQL_VER}
     container_name: mysql