From 24f27121a5416571fb1c6b0893d2c10303355543 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Tue, 6 Feb 2024 08:45:40 +0100 Subject: [PATCH 1/3] Remove the environment provider router We'll now just let the ETOS API save the providers in the database instead of sending HTTP requests to the environment provider. --- python/requirements.txt | 7 +- python/setup.cfg | 7 +- python/src/etos_api/library/database.py | 96 ++++++++ python/src/etos_api/library/environment.py | 92 ++++++++ python/src/etos_api/library/validator.py | 44 ++-- python/src/etos_api/main.py | 1 - python/src/etos_api/routers/__init__.py | 2 +- .../routers/environment_provider/__init__.py | 18 -- .../routers/environment_provider/router.py | 130 ----------- .../routers/environment_provider/schemas.py | 32 --- python/src/etos_api/routers/etos/router.py | 11 +- python/src/etos_api/routers/etos/schemas.py | 15 +- python/tests/fake_database.py | 124 +++++++++++ python/tests/library/test_validator.py | 4 +- python/tests/test_routers.py | 209 ++++++++---------- 15 files changed, 459 insertions(+), 333 deletions(-) create mode 100644 python/src/etos_api/library/database.py create mode 100644 python/src/etos_api/library/environment.py delete mode 100644 python/src/etos_api/routers/environment_provider/__init__.py delete mode 100644 python/src/etos_api/routers/environment_provider/router.py delete mode 100644 python/src/etos_api/routers/environment_provider/schemas.py create mode 100644 python/tests/fake_database.py diff --git a/python/requirements.txt b/python/requirements.txt index e23100e..576c2e4 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -12,10 +12,11 @@ # pip install -r requirements.txt # Remember to also add them in setup.cfg but unpinned. -etos_lib==3.2.1 +etos_lib==4.0.0 +etcd3gw~=2.3 pyscaffold~=4.4 uvicorn~=0.22 -fastapi~=0.96.0 +fastapi~=0.109.1 aiohttp[speedups]~=3.8 gql[requests]~=3.4 httpx~=0.24 @@ -23,5 +24,5 @@ kubernetes~=26.1 sse-starlette~=1.6 opentelemetry-api~=1.21 opentelemetry-exporter-otlp~=1.21 -opentelemetry-instrumentation-fastapi==0.42b0 +opentelemetry-instrumentation-fastapi==0.43b0 opentelemetry-sdk~=1.21 diff --git a/python/setup.cfg b/python/setup.cfg index a600eba..4fcc8d5 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -24,10 +24,11 @@ package_dir = # DON'T CHANGE THE FOLLOWING LINE! IT WILL BE UPDATED BY PYSCAFFOLD! setup_requires = pyscaffold>=3.2a0,<3.3a0 install_requires = - etos_lib==3.2.1 + etos_lib==4.0.0 + etcd3gw~=2.3 pyscaffold~=4.4 uvicorn~=0.22 - fastapi~=0.96.0 + fastapi~=0.109.1 aiohttp[speedups]~=3.8 gql[requests]~=3.4 httpx~=0.24 @@ -35,7 +36,7 @@ install_requires = sse-starlette~=1.6 opentelemetry-api~=1.21 opentelemetry-exporter-otlp~=1.21 - opentelemetry-instrumentation-fastapi==0.42b0 + opentelemetry-instrumentation-fastapi==0.43b0 opentelemetry-sdk~=1.21 # Require a specific Python version, e.g. Python 2.7 or >= 3.4 diff --git a/python/src/etos_api/library/database.py b/python/src/etos_api/library/database.py new file mode 100644 index 0000000..7a84b36 --- /dev/null +++ b/python/src/etos_api/library/database.py @@ -0,0 +1,96 @@ +# Copyright Axis Communications AB. +# +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ETCD helpers.""" +import os +from threading import Event +from typing import Any, Iterator, Optional, Union + +from etcd3gw import client +from etos_lib.lib.config import Config as ETOSConfig + + +class ETCDPath: + """An ETCD path is like a filesystem path, but it works with keys in ETCD.""" + + def __init__(self, path: Union[str, bytes] = "/") -> None: + """Initialize.""" + if ETOSConfig().get("database") is None: + ETOSConfig().set( + "database", + client( + host=os.getenv("ETOS_ETCD_HOST", "etcd-client"), + port=int(os.getenv("ETOS_ETCD_PORT", "2379")), + ), + ) + self.database: client = ETOSConfig().get("database") + if isinstance(path, bytes): + path = path.decode() + self.path = path + + def join(self, new: str) -> "ETCDPath": + """Join this path with another path. + + :param new: New child path 'below' current. + """ + if new.startswith("/"): + new = new[1:] + return ETCDPath("/".join((self.path, new))) + + def write(self, value: Any, expire: Optional[int] = None) -> None: + """Write a value to an ETCD path. + + :param value: Value to write to database. + :param expire: Optional expiration time in seconds. + """ + lease = None + if expire is not None: + lease = self.database.lease(expire) + self.database.put(self.path, value, lease) + + def read(self) -> Optional[bytes]: + """Read the values from an ETCD path.""" + try: + return self.database.get(self.path)[0] + except IndexError: + return None + + def read_all(self) -> list[tuple[bytes, dict]]: + """Read values of all keys "below" a path.""" + return self.database.get_prefix(self.path) + + def watch(self) -> tuple[Event, Iterator[dict]]: + """Watch an ETCD path for any changes.""" + return self.database.watch(self.path) + + def watch_all(self) -> tuple[Event, Iterator[dict]]: + """Watch an ETCD path for any changes to itself or its children.""" + return self.database.watch(self.path, range_end="\0") + + def delete(self) -> None: + """Delete the ETCD path.""" + self.database.delete(self.path) + + def delete_all(self) -> None: + """Delete the ETCD path and paths "below".""" + self.database.delete_prefix(self.path) + + def __str__(self) -> str: + """Represent the ETCD path as a string.""" + return self.path + + def __repr__(self) -> str: + """Represent the ETCD path as a string.""" + return self.path diff --git a/python/src/etos_api/library/environment.py b/python/src/etos_api/library/environment.py new file mode 100644 index 0000000..07fb72f --- /dev/null +++ b/python/src/etos_api/library/environment.py @@ -0,0 +1,92 @@ +# Copyright Axis Communications AB. +# +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Environment for ETOS testruns.""" +import json +from collections import OrderedDict +from typing import Optional, Union + +from pydantic import BaseModel # pylint:disable=no-name-in-module + +from .database import ETCDPath + + +class Configuration(BaseModel): + """Model for the ETOS testrun configuration.""" + + suite_id: str + dataset: Union[dict, list] + execution_space_provider: str + iut_provider: str + log_area_provider: str + + +async def configure_testrun(configuration: Configuration) -> None: + """Configure an ETOS testrun with the configuration passed by user. + + :param configuration: The configuration to save. + """ + testrun = ETCDPath(f"/testrun/{configuration.suite_id}") + providers = ETCDPath("/environment/provider") + + await do_configure( + providers.join(f"log-area/{configuration.log_area_provider}"), + configuration.log_area_provider, + testrun.join("provider/log-area"), + ) + await do_configure( + providers.join(f"execution-space/{configuration.execution_space_provider}"), + configuration.execution_space_provider, + testrun.join("provider/execution-space"), + ) + await do_configure( + providers.join(f"iut/{configuration.iut_provider}"), + configuration.iut_provider, + testrun.join("provider/iut"), + ) + await save_json(testrun.join("provider/dataset"), json.dumps(configuration.dataset)) + + +async def do_configure(path: ETCDPath, provider_id: str, testrun: ETCDPath) -> None: + """Configure a provider based on provider ID and save it to a testrun. + + :param path: Path to load provider from. + :param provider_id: The ID of the provider to load. + :param testrun: Where to store the loaded provider. + """ + if (provider := await load(path)) is None: + raise AssertionError(f"{provider_id} does not exist") + await save_json(testrun, provider) + + +async def load(path: ETCDPath) -> Optional[dict]: + """Load a provider from an ETCD path. + + :param path: Path to load data from. Will assume it's JSON and load is as such. + """ + provider = path.read() + if provider: + return json.loads(provider, object_pairs_hook=OrderedDict) + return None + + +async def save_json(path: ETCDPath, data: dict, expire=3600) -> None: + """Save data as json to an ETCD path. + + :param path: The path to store data on. + :param data: The data to save. Will be dumped to JSON before saving. + :param expire: How long, in seconds, to set the expiration to. + """ + path.write(json.dumps(data), expire=expire) diff --git a/python/src/etos_api/library/validator.py b/python/src/etos_api/library/validator.py index 5f959da..5aad911 100644 --- a/python/src/etos_api/library/validator.py +++ b/python/src/etos_api/library/validator.py @@ -15,13 +15,16 @@ # limitations under the License. """ETOS API suite validator module.""" import logging +from typing import List, Union from uuid import UUID -from typing import Union, List -# Pylint refrains from linting C extensions due to arbitrary code execution. -from pydantic import BaseModel, constr, conlist # pylint:disable=no-name-in-module -from pydantic import validator, ValidationError import requests + +# Pylint refrains from linting C extensions due to arbitrary code execution. +from pydantic import BaseModel # pylint:disable=no-name-in-module +from pydantic import ValidationError, conlist, constr, field_validator +from pydantic.fields import PrivateAttr + from etos_api.library.docker import Docker # pylint:disable=too-few-public-methods @@ -45,7 +48,7 @@ class Checkout(BaseModel): """ETOS suite definion 'CHECKOUT' constraint.""" key: str - value: conlist(str, min_items=1) + value: conlist(str, min_length=1) class Parameters(BaseModel): @@ -91,16 +94,18 @@ class Recipe(BaseModel): id: UUID testCase: TestCase - __constraint_models = { - "ENVIRONMENT": Environment, - "COMMAND": Command, - "CHECKOUT": Checkout, - "PARAMETERS": Parameters, - "EXECUTE": Execute, - "TEST_RUNNER": TestRunner, - } - - @validator("constraints") + __constraint_models = PrivateAttr( + { + "ENVIRONMENT": Environment, + "COMMAND": Command, + "CHECKOUT": Checkout, + "PARAMETERS": Parameters, + "EXECUTE": Execute, + "TEST_RUNNER": TestRunner, + } + ) + + @field_validator("constraints") def validate_constraints(cls, value): # Pydantic requires cls. pylint:disable=no-self-argument """Validate the constraints fields for each recipe. @@ -118,14 +123,15 @@ def validate_constraints(cls, value): # Pydantic requires cls. pylint:disable=n :return: Same as value, if validated. :rtype: Any """ - count = dict.fromkeys(cls.__constraint_models.keys(), 0) + keys = cls.__constraint_models.default.keys() + count = dict.fromkeys(keys, 0) for constraint in value: - model = cls.__constraint_models.get(constraint.key) + model = cls.__constraint_models.default.get(constraint.key) if model is None: - keys = tuple(cls.__constraint_models.keys()) + keys = tuple(keys) raise TypeError(f"Unknown key {constraint.key}, valid keys: {keys}") try: - model(**constraint.dict()) + model(**constraint.model_dump()) except ValidationError as exception: raise ValueError(str(exception)) from exception count[constraint.key] += 1 diff --git a/python/src/etos_api/main.py b/python/src/etos_api/main.py index 723e34d..b441211 100644 --- a/python/src/etos_api/main.py +++ b/python/src/etos_api/main.py @@ -53,5 +53,4 @@ async def redirect_head_to_root(): APP.include_router(routers.etos.ROUTER) APP.include_router(routers.selftest.ROUTER) -APP.include_router(routers.environment_provider.ROUTER) APP.include_router(routers.logs.ROUTER) diff --git a/python/src/etos_api/routers/__init__.py b/python/src/etos_api/routers/__init__.py index 6f1d75b..9e3bb14 100644 --- a/python/src/etos_api/routers/__init__.py +++ b/python/src/etos_api/routers/__init__.py @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. """ETOS API routers module.""" -from . import environment_provider, etos, selftest, logs +from . import etos, logs, selftest diff --git a/python/src/etos_api/routers/environment_provider/__init__.py b/python/src/etos_api/routers/environment_provider/__init__.py deleted file mode 100644 index 8dec8f8..0000000 --- a/python/src/etos_api/routers/environment_provider/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2020 Axis Communications AB. -# -# For a full list of individual contributors, please see the commit history. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""ETOS API environment provider module.""" -from .router import ROUTER -from . import schemas diff --git a/python/src/etos_api/routers/environment_provider/router.py b/python/src/etos_api/routers/environment_provider/router.py deleted file mode 100644 index 6b3b48f..0000000 --- a/python/src/etos_api/routers/environment_provider/router.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2020-2021 Axis Communications AB. -# -# For a full list of individual contributors, please see the commit history. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Environment provider proxy API.""" -import asyncio -import logging -import os -import time - -import aiohttp -from etos_lib import ETOS -from fastapi import APIRouter, HTTPException -from opentelemetry import trace - -from .schemas import ConfigureEnvironmentProviderRequest - -ROUTER = APIRouter() -TRACER = trace.get_tracer("etos_api.routers.environment_provider.router") -LOGGER = logging.getLogger(__name__) - - -async def _wait_for_configuration(etos_library, environment): - """Wait for the environment provider configuration to apply. - - :param etos_library: An ETOS library instance for requesting the environment provider. - :type etos_library: :obj:`etos_lib.ETOS` - :param environment: Environment that has been configured. - :type environment: :obj:`etos_api.routers.etos.schemas.ConfigureEnvironmentProviderRequest` - """ - with TRACER.start_as_current_span("wait-for-configuration") as span: - LOGGER.info("Waiting for configuration to be applied in the environment provider.") - end_time = time.time() + etos_library.debug.default_http_timeout - LOGGER.debug("Timeout: %r", etos_library.debug.default_http_timeout) - span.set_attribute("timeout", etos_library.debug.default_http_timeout) - async with aiohttp.ClientSession() as session: - while time.time() < end_time: - try: - async with session.get( - f"{etos_library.debug.environment_provider}/configure", - params={"suite_id": environment.suite_id}, - headers={ - "Content-Type": "application/json", - "Accept": "application/json", - }, - ) as response: - assert 200 <= response.status < 400 - response_json = await response.json() - LOGGER.info("Configuration: %r", response_json) - assert response_json.get("dataset") is not None - assert response_json.get("iut_provider") is not None - assert response_json.get("log_area_provider") is not None - assert response_json.get("execution_space_provider") is not None - break - except AssertionError: - if response.status < 400: - LOGGER.warning("Configuration not ready yet.") - else: - LOGGER.warning( - "Configuration verification request failed: %r, %r", - response.status, - response.reason, - ) - await asyncio.sleep(2) - else: - raise HTTPException( - status_code=400, - detail="Environment provider configuration did not apply properly", - ) - - -@ROUTER.post("/environment_provider/configure", tags=["environment_provider"], status_code=204) -async def configure_environment_provider( - environment: ConfigureEnvironmentProviderRequest, -): - """Configure environment provider request. - - :param environment: Environment to configure. - :type environment: :obj:`etos_api.routers.etos.schemas.ConfigureEnvironmentProviderRequest` - """ - with TRACER.start_as_current_span("configure-environment-provider") as span: - LOGGER.identifier.set(environment.suite_id) - span.set_attribute("etos.id", environment.suite_id) - span.set_attribute("etos.iut_provider", environment.iut_provider) - span.set_attribute("etos.execution_space_provider", environment.execution_space_provider) - span.set_attribute("etos.log_area_provider", environment.log_area_provider) - span.set_attribute("etos.dataset", str(environment.dataset)) - - LOGGER.info("Configuring environment provider using %r", environment) - etos_library = ETOS("ETOS API", os.getenv("HOSTNAME"), "ETOS API") - - end_time = time.time() + etos_library.debug.default_http_timeout - LOGGER.debug("HTTP Timeout: %r", etos_library.debug.default_http_timeout) - async with aiohttp.ClientSession() as session: - while time.time() < end_time: - try: - async with session.post( - f"{etos_library.debug.environment_provider}/configure", - json=environment.dict(), - headers={ - "Content-Type": "application/json", - "Accept": "application/json", - }, - ) as response: - assert 200 <= response.status < 400 - break - except AssertionError: - LOGGER.warning( - "Configuration request failed: %r, %r", - response.status, - response.reason, - ) - await asyncio.sleep(2) - else: - raise HTTPException( - status_code=400, - detail=f"Unable to configure environment provider with '{environment.json()}'", - ) - await _wait_for_configuration(etos_library, environment) diff --git a/python/src/etos_api/routers/environment_provider/schemas.py b/python/src/etos_api/routers/environment_provider/schemas.py deleted file mode 100644 index 30968ad..0000000 --- a/python/src/etos_api/routers/environment_provider/schemas.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2020-2022 Axis Communications AB. -# -# For a full list of individual contributors, please see the commit history. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Schemas for the environment provider endpoint.""" -from typing import Union - -# Pylint refrains from linting C extensions due to arbitrary code execution. -from pydantic import BaseModel # pylint:disable=no-name-in-module - -# pylint:disable=too-few-public-methods - - -class ConfigureEnvironmentProviderRequest(BaseModel): - """Model for the configure environment provider API.""" - - suite_id: str - dataset: Union[dict, list] - execution_space_provider: str - iut_provider: str - log_area_provider: str diff --git a/python/src/etos_api/routers/etos/router.py b/python/src/etos_api/routers/etos/router.py index d9ef055..a5d0ccb 100644 --- a/python/src/etos_api/routers/etos/router.py +++ b/python/src/etos_api/routers/etos/router.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Axis Communications AB. +# Copyright Axis Communications AB. # # For a full list of individual contributors, please see the commit history. # @@ -23,10 +23,9 @@ from fastapi import APIRouter, HTTPException from opentelemetry import trace +from etos_api.library.environment import Configuration, configure_testrun from etos_api.library.utilities import sync_to_async from etos_api.library.validator import SuiteValidator -from etos_api.routers.environment_provider.router import configure_environment_provider -from etos_api.routers.environment_provider.schemas import ConfigureEnvironmentProviderRequest from .schemas import StartEtosRequest, StartEtosResponse from .utilities import wait_for_artifact_created @@ -101,7 +100,7 @@ async def _start(etos: StartEtosRequest, span: "Span") -> dict: "selectionStrategy": {"tracker": "Suite Builder", "id": str(uuid4())}, "batchesUri": etos.test_suite_url, } - request = ConfigureEnvironmentProviderRequest( + config = Configuration( suite_id=tercc.meta.event_id, dataset=etos.dataset, execution_space_provider=etos.execution_space_provider, @@ -109,8 +108,8 @@ async def _start(etos: StartEtosRequest, span: "Span") -> dict: log_area_provider=etos.log_area_provider, ) try: - await configure_environment_provider(request) - except Exception as exception: # pylint:disable=broad-except + await configure_testrun(config) + except AssertionError as exception: LOGGER.critical(exception) raise HTTPException( status_code=400, diff --git a/python/src/etos_api/routers/etos/schemas.py b/python/src/etos_api/routers/etos/schemas.py index 0098968..cd9e67e 100644 --- a/python/src/etos_api/routers/etos/schemas.py +++ b/python/src/etos_api/routers/etos/schemas.py @@ -15,11 +15,11 @@ # limitations under the License. """Schemas for the ETOS endpoint.""" import os -from uuid import UUID from typing import Optional, Union +from uuid import UUID # Pylint refrains from linting C extensions due to arbitrary code execution. -from pydantic import BaseModel, validator # pylint:disable=no-name-in-module +from pydantic import BaseModel, Field, field_validator # pylint:disable=no-name-in-module # pylint: disable=too-few-public-methods # pylint: disable=no-self-argument @@ -29,7 +29,7 @@ class StartEtosRequest(BaseModel): """Request model for the ETOS start API.""" artifact_identity: Optional[str] - artifact_id: Optional[UUID] + artifact_id: Optional[UUID] = Field(default=None, validate_default=True) test_suite_url: str dataset: Optional[Union[dict, list]] = {} execution_space_provider: Optional[str] = os.getenv( @@ -38,17 +38,18 @@ class StartEtosRequest(BaseModel): iut_provider: Optional[str] = os.getenv("DEFAULT_IUT_PROVIDER", "default") log_area_provider: Optional[str] = os.getenv("DEFAULT_LOG_AREA_PROVIDER", "default") - @validator("artifact_id", always=True) - def validate_id_or_identity(cls, artifact_id, values): + @field_validator("artifact_id") + def validate_id_or_identity(cls, artifact_id, info): """Validate that at least one and only one of id and identity are set. :param artifact_id: The value of 'artifact_id' to validate. :value artifact_id: str or None - :param values: The list of values set on the model. - :type values: list + :param info: The information about the model. + :type info: FieldValidationInfo :return: The value of artifact_id. :rtype: str or None """ + values = info.data if values.get("artifact_identity") is None and not artifact_id: raise ValueError("At least one of 'artifact_identity' or 'artifact_id' is required.") if values.get("artifact_identity") is not None and artifact_id: diff --git a/python/tests/fake_database.py b/python/tests/fake_database.py new file mode 100644 index 0000000..c1d9756 --- /dev/null +++ b/python/tests/fake_database.py @@ -0,0 +1,124 @@ +# Copyright Axis Communications AB. +# +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fake database library helpers.""" +from queue import Queue +from threading import Event, RLock, Timer +from typing import Any, Iterator, Optional + +from etcd3gw.lease import Lease + +# pylint:disable=unused-argument + + +class FakeDatabase: + """A fake database that follows the etcd client.""" + + lock = RLock() + + def __init__(self): + """Initialize fake reader and writer.""" + self.db_dict = {} + self.expire = [] + self.leases = {} + self.watchers = [] + + def __call__(self): + """Database instantiation faker.""" + return self + + def watch(self, path: str, range_end: Optional[str] = None) -> (Event, Iterator[dict]): + """Watch for changes of a path.""" + canceled = Event() + queue = Queue() + + def cancel(): + canceled.set() + queue.put(None) + + def iterator(): + self.watchers.append(queue) + try: + while not canceled.is_set(): + event = queue.get() + if event is None: + canceled.set() + if not canceled.is_set(): + yield event + finally: + self.watchers.remove(queue) + + return iterator(), cancel + + def __event(self, event: dict) -> None: + """Send an event to all watchers.""" + for watcher in self.watchers: + watcher.put_nowait(event) + + def put(self, item: str, value: Any, lease: Optional[Lease] = None) -> None: + """Put an item into database.""" + with self.lock: + self.db_dict.setdefault(item, []) + if lease is not None: + self.leases[item] = lease + timer = Timer(self.expire[lease.id], self.delete, args=(item,)) + timer.daemon = True + timer.start() + self.db_dict[item].append(str(value).encode()) + self.__event({"kv": {"key": item.encode(), "value": str(value).encode()}}) + + def lease(self, ttl=30) -> Lease: + """Create a lease.""" + with self.lock: + self.expire.append(ttl) + # ttl is unused since we do not actually make the post request that the regular + # etcd client does. First argument to `Lease` is the ID that was returned by the + # etcd server. + return Lease(len(self.expire) - 1) + + def get(self, path: str) -> list[bytes]: + """Get an item from database.""" + if isinstance(path, bytes): + path = path.decode() + with self.lock: + return list(reversed(self.db_dict.get(path, []))) + + def get_prefix(self, prefix: str) -> list[tuple[bytes, dict]]: + """Get items based on prefix.""" + if isinstance(prefix, bytes): + prefix = prefix.decode() + paths = [] + with self.lock: + for key, value in self.db_dict.items(): + if key.startswith(prefix): + paths.append((value[-1], {"key": key.encode()})) + return paths + + def delete(self, path: str) -> None: + """Delete a single item.""" + with self.lock: + del self.db_dict[path] + if self.leases.get(path): + self.expire.pop(self.leases.get(path).id) + del self.leases[path] + self.__event({"kv": {"key": path.encode()}, "type": "DELETE"}) + + def delete_prefix(self, prefix: str) -> None: + """Delete items based on prefix.""" + with self.lock: + db_dict = self.db_dict.copy() + for key in db_dict: + if key.startswith(prefix): + self.delete(key) diff --git a/python/tests/library/test_validator.py b/python/tests/library/test_validator.py index b8dd7be..61b7763 100644 --- a/python/tests/library/test_validator.py +++ b/python/tests/library/test_validator.py @@ -17,7 +17,9 @@ import logging import sys from unittest.mock import patch + import pytest + from etos_api.library.validator import SuiteValidator, ValidationError logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) @@ -297,7 +299,7 @@ async def test_validate_unknown_constraint(self, download_suite_mock): try: await validator.validate("url") exception = False - except ValidationError: + except TypeError: exception = True self.logger.info("STEP: Verify that the validator raises ValidationError.") assert exception is True diff --git a/python/tests/test_routers.py b/python/tests/test_routers.py index 71e4084..f519608 100644 --- a/python/tests/test_routers.py +++ b/python/tests/test_routers.py @@ -14,25 +14,83 @@ # See the License for the specific language governing permissions and # limitations under the License. """ETOS API routers.""" +import json import logging import sys -from unittest.mock import patch, AsyncMock -from fastapi.testclient import TestClient +from unittest import TestCase +from unittest.mock import patch + +from etos_lib.lib.config import Config from etos_lib.lib.debug import Debug +from fastapi.testclient import TestClient + from etos_api.main import APP +from tests.fake_database import FakeDatabase logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) +IUT_PROVIDER = { + "iut": { + "id": "default", + "list": { + "possible": { + "$expand": { + "value": { + "type": "$identity.type", + "namespace": "$identity.namespace", + "name": "$identity.name", + "version": "$identity.version", + "qualifiers": "$identity.qualifiers", + "subpath": "$identity.subpath", + }, + "to": "$amount", + } + }, + "available": "$this.possible", + }, + } +} + +EXECUTION_SPACE_PROVIDER = { + "execution_space": { + "id": "default", + "list": { + "possible": { + "$expand": { + "value": {"instructions": "$execution_space_instructions"}, + "to": "$amount", + } + }, + "available": "$this.possible", + }, + } +} + + +LOG_AREA_PROVIDER = { + "log": { + "id": "default", + "list": { + "possible": { + "$expand": { + "value": {"upload": {"url": "$dataset.host", "method": "GET"}}, + "to": "$amount", + } + }, + "available": "$this.possible", + }, + } +} -class TestRouters: +class TestRouters(TestCase): """Test the routers in etos-api.""" logger = logging.getLogger(__name__) client = TestClient(APP) - @staticmethod - def teardown_method(): + def tearDown(self): """Cleanup events from ETOS library debug.""" + Config().reset() Debug().events_received.clear() Debug().events_published.clear() @@ -43,11 +101,11 @@ def test_head_on_root_without_redirect(self): - A HEAD request on root shall return 308. Test steps:: - 1. Send a HEAD request to root without allow_redirects. + 1. Send a HEAD request to root without follow_redirects. 2. Verify that status code is 308. """ - self.logger.info("STEP: Send a HEAD request to root without allow_redirects.") - response = self.client.head("/", allow_redirects=False) + self.logger.info("STEP: Send a HEAD request to root without follow_redirects.") + response = self.client.head("/", follow_redirects=False) self.logger.info("STEP: Verify that status code is 308.") assert response.status_code == 308 @@ -58,11 +116,11 @@ def test_head_on_root_with_redirect(self): - A redirected HEAD request on root shall return 204. Test steps:: - 1. Send a HEAD request to root with allow_redirects. + 1. Send a HEAD request to root with follow_redirects. 2. Verify that status code is 204. """ - self.logger.info("STEP: Send a HEAD request to root with allow_redirects.") - response = self.client.head("/", allow_redirects=True) + self.logger.info("STEP: Send a HEAD request to root with follow_redirects.") + response = self.client.head("/", follow_redirects=True) self.logger.info("STEP: Verify that status code is 204.") assert response.status_code == 204 @@ -73,20 +131,19 @@ def test_post_on_root_without_redirect(self): - A POST request on root shall return 308. Test steps:: - 1. Send a POST request to root without allow_redirects. + 1. Send a POST request to root without follow_redirects. 2. Verify that status code is 308. """ - self.logger.info("STEP: Send a POST request to root without allow_redirects.") - response = self.client.post("/", allow_redirects=False) + self.logger.info("STEP: Send a POST request to root without follow_redirects.") + response = self.client.post("/", follow_redirects=False) self.logger.info("STEP: Verify that status code is 308.") assert response.status_code == 308 @patch("etos_api.library.validator.Docker.digest") @patch("etos_api.library.validator.SuiteValidator._download_suite") @patch("etos_api.library.graphql.GraphqlQueryHandler.execute") - @patch("etos_api.routers.environment_provider.router.aiohttp.ClientSession") def test_post_on_root_with_redirect( - self, mock_client, graphql_execute_mock, download_suite_mock, digest_mock + self, graphql_execute_mock, download_suite_mock, digest_mock ): """Test that POST requests to / redirects and starts ETOS tests. @@ -94,27 +151,19 @@ def test_post_on_root_with_redirect( - A redirected POST requests to root shall return 200. Test steps:: - 1. Send a POST request to root with allow_redirects. + 1. Send a POST request to root with follow_redirects. 2. Verify that the status code is 200. """ digest_mock.return_value = ( "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ) - - mock_client().__aenter__.return_value = mock_client - mock_client.post().__aenter__.return_value = mock_client - mock_client.get().__aenter__.return_value = mock_client - mock_client.json = AsyncMock( - return_value={ - "dataset": "{}", - "iut_provider": "iut", - "log_area_provider": "log_area", - "execution_space_provider": "execution_space", - } + fake_database = FakeDatabase() + Config().set("database", fake_database) + fake_database.put("/environment/provider/log-area/default", json.dumps(LOG_AREA_PROVIDER)) + fake_database.put("/environment/provider/iut/default", json.dumps(IUT_PROVIDER)) + fake_database.put( + "/environment/provider/execution-space/default", json.dumps(EXECUTION_SPACE_PROVIDER) ) - mock_client.status = 200 - # post is called above when adding the __aenter__ return_value. - mock_client.post.reset_mock() graphql_execute_mock.return_value = { "artifactCreated": { @@ -153,14 +202,14 @@ def test_post_on_root_with_redirect( } ] - self.logger.info("STEP: Send a POST request to root with allow_redirects.") + self.logger.info("STEP: Send a POST request to root with follow_redirects.") response = self.client.post( "/", json={ "artifact_identity": "pkg:testing/etos", "test_suite_url": "http://localhost/my_test.json", }, - allow_redirects=True, + follow_redirects=True, ) self.logger.info("STEP: Verify that the status code is 200.") assert response.status_code == 200 @@ -168,8 +217,7 @@ def test_post_on_root_with_redirect( @patch("etos_api.library.validator.Docker.digest") @patch("etos_api.library.validator.SuiteValidator._download_suite") @patch("etos_api.library.graphql.GraphqlQueryHandler.execute") - @patch("etos_api.routers.environment_provider.router.aiohttp.ClientSession") - def test_start_etos(self, mock_client, graphql_execute_mock, download_suite_mock, digest_mock): + def test_start_etos(self, graphql_execute_mock, download_suite_mock, digest_mock): """Test that POST requests to /etos attempts to start ETOS tests. Approval criteria: @@ -186,20 +234,13 @@ def test_start_etos(self, mock_client, graphql_execute_mock, download_suite_mock digest_mock.return_value = ( "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ) - mock_client().__aenter__.return_value = mock_client - mock_client.post().__aenter__.return_value = mock_client - mock_client.get().__aenter__.return_value = mock_client - mock_client.json = AsyncMock( - return_value={ - "dataset": "{}", - "iut_provider": "iut", - "log_area_provider": "log_area", - "execution_space_provider": "execution_space", - } + fake_database = FakeDatabase() + Config().set("database", fake_database) + fake_database.put("/environment/provider/log-area/default", json.dumps(LOG_AREA_PROVIDER)) + fake_database.put("/environment/provider/iut/default", json.dumps(IUT_PROVIDER)) + fake_database.put( + "/environment/provider/execution-space/default", json.dumps(EXECUTION_SPACE_PROVIDER) ) - mock_client.status = 200 - # post is called above when adding the __aenter__ return_value. - mock_client.post.reset_mock() graphql_execute_mock.return_value = { "artifactCreated": { @@ -257,73 +298,17 @@ def test_start_etos(self, mock_client, graphql_execute_mock, download_suite_mock assert tercc is not None assert response.json().get("tercc") == tercc.meta.event_id self.logger.info("STEP: Verify that the environment provider was configured.") - mock_client.post.assert_called_once_with( - f"{debug.environment_provider}/configure", - json={ - "suite_id": tercc.meta.event_id, - "dataset": {}, - "execution_space_provider": "default", - "iut_provider": "default", - "log_area_provider": "default", - }, - headers={"Content-Type": "application/json", "Accept": "application/json"}, - ) - - @patch("etos_api.routers.environment_provider.router.aiohttp.ClientSession") - def test_configure_environment_provider(self, mock_client): - """Test that configure requests are proxied to the environment provider. - - Approval criteria: - - Requests to configure shall be redirected to the environment provider. - - HTTP status code shall be 204. - Test steps:: - 1. Send a POST request to configure. - 2. Verify that the status code is 204. - 3. Verify that the request was sent to the environment provider. - """ - mock_client().__aenter__.return_value = mock_client - mock_client.post().__aenter__.return_value = mock_client - mock_client.get().__aenter__.return_value = mock_client - mock_client.json = AsyncMock( - return_value={ - "dataset": "{}", - "iut_provider": "iut", - "log_area_provider": "log_area", - "execution_space_provider": "execution_space", - } - ) - mock_client.status = 200 - mock_client.post.reset_mock() - - self.logger.info("STEP: Send a POST request to configure.") - response = self.client.post( - "environment_provider/configure", - json={ - "suite_id": "f5d5bc7b-c6b8-406f-a997-43c8217e32c1", - "dataset": {}, - "iut_provider": "iut", - "execution_space_provider": "execution_space", - "log_area_provider": "log_area", - }, + log_area = json.loads( + fake_database.get(f"/testrun/{tercc.meta.event_id}/provider/log-area")[0] ) - - self.logger.info("STEP: Verify that the status code is 204.") - assert response.status_code == 204 - - self.logger.info("STEP: Verify that the request was sent to the environment provider.") - debug = Debug() - mock_client.post.assert_called_once_with( - f"{debug.environment_provider}/configure", - json={ - "suite_id": "f5d5bc7b-c6b8-406f-a997-43c8217e32c1", - "dataset": {}, - "execution_space_provider": "execution_space", - "iut_provider": "iut", - "log_area_provider": "log_area", - }, - headers={"Content-Type": "application/json", "Accept": "application/json"}, + iut = json.loads(fake_database.get(f"/testrun/{tercc.meta.event_id}/provider/iut")[0]) + execution_space = json.loads( + fake_database.get(f"/testrun/{tercc.meta.event_id}/provider/execution-space")[0] ) + self.assertDictEqual(log_area, LOG_AREA_PROVIDER) + self.assertDictEqual(iut, IUT_PROVIDER) + self.assertDictEqual(execution_space, EXECUTION_SPACE_PROVIDER) def test_selftest_get_ping(self): """Test that selftest ping with HTTP GET pings the system. From ab88b76bbf28cfd9df40b59341ef68aedc243816 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Tue, 6 Feb 2024 10:17:47 +0100 Subject: [PATCH 2/3] Update versions of the SSE requirements --- deploy/etos-sse/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/etos-sse/Dockerfile b/deploy/etos-sse/Dockerfile index 55ed7e8..2ece284 100644 --- a/deploy/etos-sse/Dockerfile +++ b/deploy/etos-sse/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.20-alpine AS build WORKDIR /tmp/sse COPY . . -RUN apk add --no-cache make=4.4.1-r1 git=2.40.1-r0 && make build +RUN apk add --no-cache make=4.4.1-r2 git=2.43.0-r0 && make build FROM alpine:3.17.3 ARG TZ @@ -11,7 +11,7 @@ LABEL org.opencontainers.image.source=https://github.com/eiffel-community/etos-a LABEL org.opencontainers.image.authors=etos-maintainers@googlegroups.com LABEL org.opencontainers.image.licenses=Apache-2.0 -RUN apk add --no-cache tzdata=2023c-r0 +RUN apk add --no-cache tzdata=2024a-r0 ENTRYPOINT ["/app/etos-sse"] COPY --from=build /tmp/sse/bin/etos-sse /app/etos-sse From 798ee32eb017f2427e94f1765f52f3e66faa6ffe Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Tue, 6 Feb 2024 13:51:34 +0100 Subject: [PATCH 3/3] Dont dump to json string twice --- python/src/etos_api/library/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/etos_api/library/environment.py b/python/src/etos_api/library/environment.py index 07fb72f..f7a458a 100644 --- a/python/src/etos_api/library/environment.py +++ b/python/src/etos_api/library/environment.py @@ -56,7 +56,7 @@ async def configure_testrun(configuration: Configuration) -> None: configuration.iut_provider, testrun.join("provider/iut"), ) - await save_json(testrun.join("provider/dataset"), json.dumps(configuration.dataset)) + await save_json(testrun.join("provider/dataset"), configuration.dataset) async def do_configure(path: ETCDPath, provider_id: str, testrun: ETCDPath) -> None: