Merge pull request #103 from i-VRESSE/quick-close

Dont close a connection that is not there

.github/workflows/dirac.yml

@@ -5,6 +5,7 @@ on:
     # Run tests only if dirac code is changed or workflow is triggerd manually
     paths:
     - '**dirac**'
+    - .github/workflows/dirac.yml
   workflow_dispatch:
 
 jobs:

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
   - id: add-trailing-comma
 
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.1.0
+  rev: v2.14.0
   hooks:
   - id: pretty-format-yaml
     args:

README.md

@@ -35,16 +35,18 @@ at <https://i-vresse-bartender.readthedocs.io> .
 
 ## Quickstart
 
-1. Install bartender via github:
+1. Install bartender via GitHub:
 
     ```bash
-    pip install git+https://github.com/i-VRESSE/bartender.git
+    # Inside an empty directory
+    git clone https://github.com/i-VRESSE/bartender.git .
+    pip install .
     ```
 
 1. Obtain a copy of the example configuration file
 
     ```bash
-    curl -o config.yaml https://raw.githubusercontent.com/i-VRESSE/bartender/main/config-example.yaml
+    cp config-example.yaml config.yaml
     ```
 
 1. In another terminal, start up a database for storing jobs.
@@ -109,8 +111,8 @@ Bartender is by default configured with a word count applicaton. Use the
 following steps to run a job:
 
 1. Create an archive to submit. The zip file should contain a file called
-   `README.md`. A zip file could be created in a clone of this repo with `zip
-   README.zip README.md`.
+   `README.md`. A zip file could be created in a clone of this repo with
+   `zip README.zip README.md`.
 2. Start [bartender web service and postgresql
    server](https://i-vresse-bartender.readthedocs.io/en/latest/index.html#quickstart)
 3. Generate token & authorize
@@ -123,7 +125,7 @@ following steps to run a job:
     4. Try out the `GET /api/whoami` route.
        It should return JSON document with `someone` as username.
 4. Submit archive.
-    1. Try out the `PUT /api/application/wc/job` route.
+    1. Try out the `PUT /api/application/wc` route.
     2. Upload the `README.zip` as request body.
     3. Press execute button
     4. The response contains a job identifier (`id` property) that can be used

deploy/Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.10-slim-buster
 # Install healthcheck dependency
 RUN apt update && apt install -y curl unzip
 
-RUN pip install poetry==1.7.0
+RUN pip install poetry==1.8.3
 
 # Configuring poetry
 RUN poetry config virtualenvs.create false

deploy/Dockerfile.dirac

@@ -8,7 +8,7 @@ RUN apt install -y unzip
 
 USER diracuser
 
-RUN pip install poetry==1.4.2
+RUN pip install poetry==1.8.3
 
 # Configuring poetry
 RUN poetry config virtualenvs.create false

docs/configuration.md

@@ -340,7 +340,7 @@ If you need [DIRAC](http://diracgrid.org/) support create a conda environment
 instead of creating a virtual environment.
 
 ```bash
-mamba create --name bartender dirac-grid python=3.10 poetry=1.5.1
+mamba create --name bartender dirac-grid python=3.10 poetry=1.8.3
 conda activate bartender
 poetry install
 ```

docs/develop.md

@@ -203,7 +203,7 @@ the `WorkloadManagement_SiteDirector` service starting a pilot.
 To look around inside the DIRAC server use
 
 ```shell
-docker-compose -f tests_dirac/docker-compose.yml exec dirac-tuto bash
+docker compose -f tests_dirac/docker-compose.yml exec dirac-tuto bash
 ```
 
 Sometimes the DIRAC server needs clearing of its state,

pyproject.toml

@@ -21,7 +21,7 @@ yarl = "^1.7.2"
 ujson = "^5.1.0"
 sqlalchemy = {extras = ["asyncio"], version = "^2.0.7"}
 alembic = "^1.10.2"
-asyncpg = "^0.27.0"
+asyncpg = "^0.29.0"
 python-multipart = "^0.0.6"
 aiofiles = "^23.1.0"
 asyncssh = "^2.13.1"
@@ -38,12 +38,13 @@ redis = {extras = ["hiredis"], version ="5.0.8"}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.0"
-flake8 = "^4.0.1"
+flake8 = "^7.1.1"
 mypy = "^1.1.1"
 isort = "^5.10.1"
 yesqa = "^1.3.0"
 pre-commit = "^2.19.0"
-wemake-python-styleguide = "^0.16.1"
+# released version not python 3.12 compatible due to flake8-commas, use git version
+wemake-python-styleguide = {git = "https://github.com/wemake-services/wemake-python-styleguide.git", rev = "56121b39a190c0a0fb575f4066cd8bafddfe2211"}
 black = "^22.3.0"
 autoflake = "^1.4"
 pytest-cov = "^3.0.0"
@@ -99,5 +100,5 @@ env = [
 ]
 
 [build-system]
-requires = ["poetry-core>=1.5.2"]
+requires = ["poetry-core>=1.8.3"]
 build-backend = "poetry.core.masonry.api"

src/bartender/schedulers/arq.py

@@ -42,10 +42,11 @@ class ArqSchedulerConfig(BaseModel):
 
     Default is one hour.
 
-    In seconds or string in `ISO 8601 duration format <https://en.wikipedia.org/wiki/ISO_8601#Durations>`_.
+    In seconds or string in
+    `ISO 8601 duration format <https://en.wikipedia.org/wiki/ISO_8601#Durations>`_.
 
     For example, "PT12H" represents a max runtime of "twelve hours".
-    """  # noqa: E501, WPS428
+    """  # noqa: WPS428
 
     @property
     def redis_settings(self) -> RedisSettings:
@@ -73,8 +74,8 @@ def __init__(self, config: ArqSchedulerConfig) -> None:
         self.connection: Optional[ArqRedis] = None
 
     async def close(self) -> None:  # noqa: D102
-        pool = await self._pool()
-        await pool.close()
+        if self.connection is not None:
+            await self.connection.close()
 
     async def submit(self, description: JobDescription) -> str:  # noqa: D102
         pool = await self._pool()

src/bartender/schedulers/dirac.py

@@ -298,7 +298,7 @@ async def _jdl_script(self, description: JobDescription, scriptdir: Path) -> str
             StdOutput = "jobstdout.txt";
             StdError = "jobstderr.txt";
             OutputSandbox = {{ "jobstdout.txt", "jobstderr.txt" }};
-            """,  # noqa: E501, WPS237
+            """,  # noqa: E501, WPS237, WPS318
         )
 
 

tests_dirac/Dockerfile

@@ -1,7 +1,7 @@
-FROM ghcr.io/xenon-middleware/diracclient:8.0.39
+FROM ghcr.io/xenon-middleware/diracclient:8.0.49
 
 RUN pip install --upgrade pip wheel setuptools && \
-    pip install poetry==1.7.0
+    pip install poetry==1.8.3
 
 # Configuring poetry
 RUN poetry config virtualenvs.create false
@@ -12,7 +12,9 @@ WORKDIR /workspace
 COPY pyproject.toml poetry.lock ./
 
 # Installing requirements
-RUN poetry install --no-root --no-interaction --no-ansi --with dev
+RUN poetry install --no-root --no-interaction --no-ansi --without dev
+
+RUN pip install pytest anyio pytest-env
 
 # Workaround `ImportWarning: _SixMetaPathImporter.find_spec() not found; falling back to find_module()` error
 RUN /home/diracuser/diracos/bin/micromamba install m2crypto==0.041.0

tests_dirac/docker-compose.yml

@@ -1,8 +1,6 @@
-version: '3.9'
-
 services:
   dirac-tuto:
-    image: ghcr.io/xenon-middleware/dirac:8.0.39
+    image: ghcr.io/xenon-middleware/dirac:8.0.49
     privileged: true
     hostname: dirac-tuto
   test:

tests_dirac/test_it.py

@@ -127,7 +127,20 @@ async def apptainer_image() -> Path:
     return image
 
 
+# Skip test
+# su - diracpilot
+# . /opt/dirac/bashrc
+# apptainer run docker://alpine cat /etc/os-release
+# Gives
+# ERROR   [U=65534,P=5871]   setup_userns_mappings()
+#        Could not write info to setgroups: Permission denied
+# ERROR   [U=65534,P=5892]   init()
+# Error while waiting event for user namespace mappings: no event received
+# image does not run bashrc or has apptainer suid installed
 @pytest.mark.anyio
+@pytest.mark.skip(
+    reason="diracpilot user in DIRAC server container does not have permission to run apptainer",  # noqa: E501
+)
 async def test_it_with_apptainer(  # noqa: WPS217 single piece of code for readablilty
     apptainer_image: Path,
     tmp_path: Path,

tests_dirac/test_renewer.py

@@ -66,7 +66,6 @@ async def test_make_valid_dirac_proxy_given_bad_cert(tmp_path: Path) -> None:
 @pytest.mark.anyio
 async def test_renewer_setup_teardown() -> None:
     setup_proxy_renewer(ProxyConfig(valid="00:03", min_life=30))
-
     # Give renewer task time to run
     await asyncio.sleep(0.1)