From d26b4c310cb9770e226b1887416b9d70397e5616 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sun, 26 Nov 2023 21:17:26 +0100 Subject: [PATCH] test: Support split diracx package for integration tests --- integration_tests.py | 20 +++++++++++-------- tests/CI/docker-compose.yml | 15 ++++++++++---- .../opensearchproject/opensearch:2.1.0.env | 2 +- tests/CI/exportCSLoop.sh | 3 +++ 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/integration_tests.py b/integration_tests.py index 6377a48fc9c..de5c05206ae 100755 --- a/integration_tests.py +++ b/integration_tests.py @@ -146,12 +146,13 @@ def create( flags: Optional[list[str]] = typer.Argument(None), editable: Optional[bool] = None, extra_module: Optional[list[str]] = None, + diracx_dist_dir: Optional[str] = None, release_var: Optional[str] = None, run_server_tests: bool = True, run_client_tests: bool = True, ): """Start a local instance of the integration tests""" - prepare_environment(flags, editable, extra_module, release_var) + prepare_environment(flags, editable, extra_module, diracx_dist_dir, release_var) install_server() install_client() exit_code = 0 @@ -191,6 +192,7 @@ def prepare_environment( flags: Optional[list[str]] = typer.Argument(None), editable: Optional[bool] = None, extra_module: Optional[list[str]] = None, + diracx_dist_dir: Optional[str] = None, release_var: Optional[str] = None, ): """Prepare the local environment for installing DIRAC.""" @@ -227,7 +229,7 @@ def prepare_environment( extra_services = list(chain(*[config["extra-services"] for config in module_configs.values()])) typer.secho("Running docker-compose to create containers", fg=c.GREEN) - with _gen_docker_compose(modules) as docker_compose_fn: + with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn: subprocess.run( ["docker-compose", "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client"] + extra_services, check=True, @@ -322,7 +324,7 @@ def prepare_environment( typer.secho("Running docker-compose to create DiracX containers", fg=c.GREEN) typer.secho(f"Will leave a folder behind: {docker_compose_fn_final}", fg=c.YELLOW) - with _gen_docker_compose(modules) as docker_compose_fn: + with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn: # We cannot use the temporary directory created in the context manager because # we don't stay in the contect manager (Popen) # So we need something that outlives it. @@ -545,7 +547,7 @@ class TestExit(typer.Exit): @contextmanager -def _gen_docker_compose(modules): +def _gen_docker_compose(modules, *, diracx_dist_dir=None): # Load the docker-compose configuration and mount the necessary volumes input_fn = Path(__file__).parent / "tests/CI/docker-compose.yml" docker_compose = yaml.safe_load(input_fn.read_text()) @@ -560,10 +562,12 @@ def _gen_docker_compose(modules): docker_compose["services"]["diracx-wait-for-db"]["volumes"].extend(volumes[:]) module_configs = _load_module_configs(modules) - if "diracx" in module_configs: - docker_compose["services"]["diracx"]["volumes"].append( - f"{modules['diracx']}/src/diracx:{module_configs['diracx']['install-location']}" - ) + if diracx_dist_dir is not None: + for container_name in ["diracx-init-cs", "diracx-wait-for-db", "diracx"]: + docker_compose["services"][container_name]["volumes"].append(f"{diracx_dist_dir}:/diracx_sources") + docker_compose["services"][container_name].setdefault("environment", []).append( + "DIRACX_CUSTOM_SOURCE_PREFIXES=/diracx_sources" + ) # Add any extension services for module_name, module_configs in module_configs.items(): diff --git a/tests/CI/docker-compose.yml b/tests/CI/docker-compose.yml index 97c4dadaf5d..e35437bd0bb 100644 --- a/tests/CI/docker-compose.yml +++ b/tests/CI/docker-compose.yml @@ -120,15 +120,20 @@ services: - DIRACX_SERVICE_AUTH_TOKEN_KEY="file:///signing-key/rs256.key" volumes: - diracx-key-store:/signing-key/ + # As the diracx images don't run as root we need to change the permissions of the /cs_store/ directory as well + - diracx-cs-store:/cs_store/ # We need to allow everybody to read the private keys # Because the users are different between the DIRAC and DiracX containers entrypoint: | - bash -c "ssh-keygen -P '' -trsa -b4096 -mPEM -f/signing-key/rs256.key && chmod o+r /signing-key/rs256.*" + bash -xc "ssh-keygen -P '' -trsa -b4096 -mPEM -f/signing-key/rs256.key && chmod o+r /signing-key/rs256.* && chmod -R o=u /cs_store" pull_policy: always diracx-init-cs: - image: ghcr.io/diracgrid/diracx/server + image: ghcr.io/diracgrid/diracx/client:dev container_name: diracx-init-cs + depends_on: + diracx-init-key: + condition: service_completed_successfully # Let the init container set the permission on /cs_store/ environment: - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key @@ -140,7 +145,7 @@ services: pull_policy: always diracx-init-db: - image: ghcr.io/diracgrid/diracx/server + image: ghcr.io/diracgrid/diracx/services:dev container_name: diracx-init-db depends_on: mysql: @@ -152,7 +157,7 @@ services: pull_policy: always diracx: - image: ghcr.io/diracgrid/diracx/server + image: ghcr.io/diracgrid/diracx/services:dev container_name: diracx environment: - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo @@ -175,6 +180,8 @@ services: volumes: - diracx-cs-store:/cs_store/ - diracx-key-store:/signing-key/ + entrypoint: | + /entrypoint.sh bash -xc 'uvicorn --factory diracx.routers:create_app --host=0.0.0.0' healthcheck: test: ["CMD", "/entrypoint.sh", "curl", "-f", "http://localhost:8000/.well-known/openid-configuration"] diff --git a/tests/CI/envs/opensearchproject/opensearch:2.1.0.env b/tests/CI/envs/opensearchproject/opensearch:2.1.0.env index 2e19968618a..0eb9691f87f 100644 --- a/tests/CI/envs/opensearchproject/opensearch:2.1.0.env +++ b/tests/CI/envs/opensearchproject/opensearch:2.1.0.env @@ -6,4 +6,4 @@ cluster.routing.allocation.disk.watermark.high=300mb plugins.security.disabled=true # Elasticsearch allocates 1GB of memory by default. As resources are limited # and elasticsearch performance isn't critical in CI, limit this to 256MB -"OPENSEARCH_JAVA_OPTS=-Xms256m -Xmx256m" # minimum and maximum Java heap size +OPENSEARCH_JAVA_OPTS=-Xms256m -Xmx256m diff --git a/tests/CI/exportCSLoop.sh b/tests/CI/exportCSLoop.sh index 093794bdbd6..7a9ba5b4946 100755 --- a/tests/CI/exportCSLoop.sh +++ b/tests/CI/exportCSLoop.sh @@ -13,6 +13,9 @@ source /home/dirac/ServerInstallDIR/bashrc git config --global user.name "DIRAC Server CI" git config --global user.email "dirac-server-ci@invalid" +mkdir -p /home/dirac/TestCode/diracx/tests/cli/legacy/cs_sync/ +curl -L https://raw.githubusercontent.com/DIRACGrid/diracx/main/diracx-cli/tests/legacy/cs_sync/convert_integration_test.yaml > /home/dirac/TestCode/diracx/tests/cli/legacy/cs_sync/convert_integration_test.yaml + while true; do DIRAC_COMPAT_ENABLE_CS_CONVERSION=x dirac internal legacy cs-sync \ "$DIRACOS/etc/Production.cfg" \