Remove the environment provider router (#53)

* Remove the environment provider router

We'll now just let the ETOS API save the providers in the database
instead of sending HTTP requests to the environment provider.

python/requirements.txt

@@ -12,16 +12,17 @@
 # pip install -r requirements.txt
 # Remember to also add them in setup.cfg but unpinned.
 
-etos_lib==3.2.1
+etos_lib==4.0.0
+etcd3gw~=2.3
 pyscaffold~=4.4
 uvicorn~=0.22
-fastapi~=0.96.0
+fastapi~=0.109.1
 aiohttp[speedups]~=3.8
 gql[requests]~=3.4
 httpx~=0.24
 kubernetes~=26.1
 sse-starlette~=1.6
 opentelemetry-api~=1.21
 opentelemetry-exporter-otlp~=1.21
-opentelemetry-instrumentation-fastapi==0.42b0
+opentelemetry-instrumentation-fastapi==0.43b0
 opentelemetry-sdk~=1.21

python/setup.cfg

@@ -24,10 +24,11 @@ package_dir =
 # DON'T CHANGE THE FOLLOWING LINE! IT WILL BE UPDATED BY PYSCAFFOLD!
 setup_requires = pyscaffold>=3.2a0,<3.3a0
 install_requires =
-    etos_lib==3.2.1
+    etos_lib==4.0.0
+    etcd3gw~=2.3
     pyscaffold~=4.4
     uvicorn~=0.22
-    fastapi~=0.96.0
+    fastapi~=0.109.1
     aiohttp[speedups]~=3.8
     gql[requests]~=3.4
     httpx~=0.24

python/src/etos_api/library/database.py

@@ -0,0 +1,96 @@
+# Copyright Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ETCD helpers."""
+import os
+from threading import Event
+from typing import Any, Iterator, Optional, Union
+
+from etcd3gw import client
+from etos_lib.lib.config import Config as ETOSConfig
+
+
+class ETCDPath:
+    """An ETCD path is like a filesystem path, but it works with keys in ETCD."""
+
+    def __init__(self, path: Union[str, bytes] = "/") -> None:
+        """Initialize."""
+        if ETOSConfig().get("database") is None:
+            ETOSConfig().set(
+                "database",
+                client(
+                    host=os.getenv("ETOS_ETCD_HOST", "etcd-client"),
+                    port=int(os.getenv("ETOS_ETCD_PORT", "2379")),
+                ),
+            )
+        self.database: client = ETOSConfig().get("database")
+        if isinstance(path, bytes):
+            path = path.decode()
+        self.path = path
+
+    def join(self, new: str) -> "ETCDPath":
+        """Join this path with another path.
+
+        :param new: New child path 'below' current.
+        """
+        if new.startswith("/"):
+            new = new[1:]
+        return ETCDPath("/".join((self.path, new)))
+
+    def write(self, value: Any, expire: Optional[int] = None) -> None:
+        """Write a value to an ETCD path.
+
+        :param value: Value to write to database.
+        :param expire: Optional expiration time in seconds.
+        """
+        lease = None
+        if expire is not None:
+            lease = self.database.lease(expire)
+        self.database.put(self.path, value, lease)
+
+    def read(self) -> Optional[bytes]:
+        """Read the values from an ETCD path."""
+        try:
+            return self.database.get(self.path)[0]
+        except IndexError:
+            return None
+
+    def read_all(self) -> list[tuple[bytes, dict]]:
+        """Read values of all keys "below" a path."""
+        return self.database.get_prefix(self.path)
+
+    def watch(self) -> tuple[Event, Iterator[dict]]:
+        """Watch an ETCD path for any changes."""
+        return self.database.watch(self.path)
+
+    def watch_all(self) -> tuple[Event, Iterator[dict]]:
+        """Watch an ETCD path for any changes to itself or its children."""
+        return self.database.watch(self.path, range_end="\0")
+
+    def delete(self) -> None:
+        """Delete the ETCD path."""
+        self.database.delete(self.path)
+
+    def delete_all(self) -> None:
+        """Delete the ETCD path and paths "below"."""
+        self.database.delete_prefix(self.path)
+
+    def __str__(self) -> str:
+        """Represent the ETCD path as a string."""
+        return self.path
+
+    def __repr__(self) -> str:
+        """Represent the ETCD path as a string."""
+        return self.path

python/src/etos_api/library/environment.py

@@ -0,0 +1,92 @@
+# Copyright Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Environment for ETOS testruns."""
+import json
+from collections import OrderedDict
+from typing import Optional, Union
+
+from pydantic import BaseModel  # pylint:disable=no-name-in-module
+
+from .database import ETCDPath
+
+
+class Configuration(BaseModel):
+    """Model for the ETOS testrun configuration."""
+
+    suite_id: str
+    dataset: Union[dict, list]
+    execution_space_provider: str
+    iut_provider: str
+    log_area_provider: str
+
+
+async def configure_testrun(configuration: Configuration) -> None:
+    """Configure an ETOS testrun with the configuration passed by user.
+
+    :param configuration: The configuration to save.
+    """
+    testrun = ETCDPath(f"/testrun/{configuration.suite_id}")
+    providers = ETCDPath("/environment/provider")
+
+    await do_configure(
+        providers.join(f"log-area/{configuration.log_area_provider}"),
+        configuration.log_area_provider,
+        testrun.join("provider/log-area"),
+    )
+    await do_configure(
+        providers.join(f"execution-space/{configuration.execution_space_provider}"),
+        configuration.execution_space_provider,
+        testrun.join("provider/execution-space"),
+    )
+    await do_configure(
+        providers.join(f"iut/{configuration.iut_provider}"),
+        configuration.iut_provider,
+        testrun.join("provider/iut"),
+    )
+    await save_json(testrun.join("provider/dataset"), configuration.dataset)
+
+
+async def do_configure(path: ETCDPath, provider_id: str, testrun: ETCDPath) -> None:
+    """Configure a provider based on provider ID and save it to a testrun.
+
+    :param path: Path to load provider from.
+    :param provider_id: The ID of the provider to load.
+    :param testrun: Where to store the loaded provider.
+    """
+    if (provider := await load(path)) is None:
+        raise AssertionError(f"{provider_id} does not exist")
+    await save_json(testrun, provider)
+
+
+async def load(path: ETCDPath) -> Optional[dict]:
+    """Load a provider from an ETCD path.
+
+    :param path: Path to load data from. Will assume it's JSON and load is as such.
+    """
+    provider = path.read()
+    if provider:
+        return json.loads(provider, object_pairs_hook=OrderedDict)
+    return None
+
+
+async def save_json(path: ETCDPath, data: dict, expire=3600) -> None:
+    """Save data as json to an ETCD path.
+
+    :param path: The path to store data on.
+    :param data: The data to save. Will be dumped to JSON before saving.
+    :param expire: How long, in seconds, to set the expiration to.
+    """
+    path.write(json.dumps(data), expire=expire)

python/src/etos_api/library/validator.py

@@ -15,13 +15,16 @@
 # limitations under the License.
 """ETOS API suite validator module."""
 import logging
+from typing import List, Union
 from uuid import UUID
-from typing import Union, List
 
-# Pylint refrains from linting C extensions due to arbitrary code execution.
-from pydantic import BaseModel, constr, conlist  # pylint:disable=no-name-in-module
-from pydantic import validator, ValidationError
 import requests
+
+# Pylint refrains from linting C extensions due to arbitrary code execution.
+from pydantic import BaseModel  # pylint:disable=no-name-in-module
+from pydantic import ValidationError, conlist, constr, field_validator
+from pydantic.fields import PrivateAttr
+
 from etos_api.library.docker import Docker
 
 # pylint:disable=too-few-public-methods
@@ -45,7 +48,7 @@ class Checkout(BaseModel):
     """ETOS suite definion 'CHECKOUT' constraint."""
 
     key: str
-    value: conlist(str, min_items=1)
+    value: conlist(str, min_length=1)
 
 
 class Parameters(BaseModel):
@@ -91,16 +94,18 @@ class Recipe(BaseModel):
     id: UUID
     testCase: TestCase
 
-    __constraint_models = {
-        "ENVIRONMENT": Environment,
-        "COMMAND": Command,
-        "CHECKOUT": Checkout,
-        "PARAMETERS": Parameters,
-        "EXECUTE": Execute,
-        "TEST_RUNNER": TestRunner,
-    }
-
-    @validator("constraints")
+    __constraint_models = PrivateAttr(
+        {
+            "ENVIRONMENT": Environment,
+            "COMMAND": Command,
+            "CHECKOUT": Checkout,
+            "PARAMETERS": Parameters,
+            "EXECUTE": Execute,
+            "TEST_RUNNER": TestRunner,
+        }
+    )
+
+    @field_validator("constraints")
     def validate_constraints(cls, value):  # Pydantic requires cls. pylint:disable=no-self-argument
         """Validate the constraints fields for each recipe.
 
@@ -118,14 +123,15 @@ def validate_constraints(cls, value):  # Pydantic requires cls. pylint:disable=n
         :return: Same as value, if validated.
         :rtype: Any
         """
-        count = dict.fromkeys(cls.__constraint_models.keys(), 0)
+        keys = cls.__constraint_models.default.keys()
+        count = dict.fromkeys(keys, 0)
         for constraint in value:
-            model = cls.__constraint_models.get(constraint.key)
+            model = cls.__constraint_models.default.get(constraint.key)
             if model is None:
-                keys = tuple(cls.__constraint_models.keys())
+                keys = tuple(keys)
                 raise TypeError(f"Unknown key {constraint.key}, valid keys: {keys}")
             try:
-                model(**constraint.dict())
+                model(**constraint.model_dump())
             except ValidationError as exception:
                 raise ValueError(str(exception)) from exception
             count[constraint.key] += 1

python/src/etos_api/main.py

@@ -53,5 +53,4 @@ async def redirect_head_to_root():
 
 APP.include_router(routers.etos.ROUTER)
 APP.include_router(routers.selftest.ROUTER)
-APP.include_router(routers.environment_provider.ROUTER)
 APP.include_router(routers.logs.ROUTER)

python/src/etos_api/routers/__init__.py

@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ETOS API routers module."""
-from . import environment_provider, etos, selftest, logs
+from . import etos, logs, selftest