From b49ccae48bd371149aa8a5e2f170095c0b20e148 Mon Sep 17 00:00:00 2001 From: Baptiste O'Jeanson Date: Wed, 2 Aug 2023 18:10:26 +0200 Subject: [PATCH] wip pydantic --- docs/edge_orchestrator.md | 2 - edge_orchestrator/config/.active_config | 0 edge_orchestrator/config/.active_config.json | 65 +++++++++++++ edge_orchestrator/config/inventory.json | 21 ++--- ...ker_classification_with_1_fake_camera.json | 92 +++++++++++-------- ...marker_classification_with_1_picamera.json | 22 ++++- ...arker_classification_with_1_usbcamera.json | 20 +++- ...er_classification_with_2_fake_cameras.json | 28 +++++- .../person_detection_with_1_fake_camera.json | 6 +- .../edge_orchestrator/api_config.py | 11 +-- .../application/api_routes.py | 46 +++++----- .../edge_orchestrator/application/config.py | 73 +++++++++++++-- .../application/dto/binary_storage_config.py | 16 ++++ .../application/dto/camera_config.py | 55 +++++++++++ .../application/dto/edge_station_config.py | 14 +++ .../dto/metadata_storage_config.py | 17 ++++ .../application/dto/model_config.py | 81 ++++++++++++++++ .../application/dto/model_forward_config.py | 14 +++ .../application/dto/station_config.py | 58 ++++++++++++ .../application/dto/telemetry_sink_config.py | 15 +++ .../application/trigger_routes.py | 2 +- .../edge_orchestrator/constants.py | 5 + .../domain/models/edge_station.py | 2 +- .../domain/models/model_infos.py | 40 +++----- .../domain/ports/inventory.py | 8 -- .../domain/ports/station_config.py | 30 ------ .../domain/use_cases/supervisor.py | 2 +- .../edge_orchestrator/environment/__init__.py | 0 .../environment/abstract_config.py | 43 --------- .../edge_orchestrator/environment/default.py | 35 ------- .../edge_orchestrator/environment/docker.py | 45 --------- .../edge_with_azure_container_storage.py | 39 -------- .../edge_with_filesystem_metadata_storage.py | 43 --------- .../edge_with_mongo_db_metadata_storage.py | 43 --------- .../edge_orchestrator/environment/test.py | 49 ---------- .../environment/upload_with_gcp_bucket.py | 40 -------- .../infrastructure/camera/fake_camera.py | 3 +- .../infrastructure/filesystem_helpers.py | 2 +- .../infrastructure/inventory/__init__.py | 0 .../inventory/json_inventory.py | 30 ------ .../model_forward/model_forward_factory.py | 4 +- .../station_config/json_station_config.py | 18 ++-- edge_orchestrator/pyproject.toml | 1 + .../tests/config/inventory_TEST.json | 7 +- edge_orchestrator/tests/conftest.py | 1 - .../tests/fixtures/items_config.py | 8 -- .../fixtures/supervisor_and_collaborators.py | 11 +-- .../supervisor_inventory_route.feature | 5 - .../test_json_station_config.py | 8 +- .../domain/models/test_edge_station.py | 4 +- .../unit_tests/domain/test_supervisor.py | 16 +--- 51 files changed, 592 insertions(+), 608 deletions(-) delete mode 100644 edge_orchestrator/config/.active_config create mode 100644 edge_orchestrator/config/.active_config.json create mode 100644 edge_orchestrator/edge_orchestrator/application/dto/binary_storage_config.py create mode 100644 edge_orchestrator/edge_orchestrator/application/dto/camera_config.py create mode 100644 edge_orchestrator/edge_orchestrator/application/dto/edge_station_config.py create mode 100644 edge_orchestrator/edge_orchestrator/application/dto/metadata_storage_config.py create mode 100644 edge_orchestrator/edge_orchestrator/application/dto/model_config.py create mode 100644 edge_orchestrator/edge_orchestrator/application/dto/model_forward_config.py create mode 100644 edge_orchestrator/edge_orchestrator/application/dto/station_config.py create mode 100644 edge_orchestrator/edge_orchestrator/application/dto/telemetry_sink_config.py create mode 100644 edge_orchestrator/edge_orchestrator/constants.py delete mode 100644 edge_orchestrator/edge_orchestrator/domain/ports/inventory.py delete mode 100644 edge_orchestrator/edge_orchestrator/environment/__init__.py delete mode 100644 edge_orchestrator/edge_orchestrator/environment/abstract_config.py delete mode 100644 edge_orchestrator/edge_orchestrator/environment/default.py delete mode 100644 edge_orchestrator/edge_orchestrator/environment/docker.py delete mode 100644 edge_orchestrator/edge_orchestrator/environment/edge_with_azure_container_storage.py delete mode 100644 edge_orchestrator/edge_orchestrator/environment/edge_with_filesystem_metadata_storage.py delete mode 100644 edge_orchestrator/edge_orchestrator/environment/edge_with_mongo_db_metadata_storage.py delete mode 100644 edge_orchestrator/edge_orchestrator/environment/test.py delete mode 100644 edge_orchestrator/edge_orchestrator/environment/upload_with_gcp_bucket.py delete mode 100644 edge_orchestrator/edge_orchestrator/infrastructure/inventory/__init__.py delete mode 100644 edge_orchestrator/edge_orchestrator/infrastructure/inventory/json_inventory.py diff --git a/docs/edge_orchestrator.md b/docs/edge_orchestrator.md index e52653a0..72f9fba3 100644 --- a/docs/edge_orchestrator.md +++ b/docs/edge_orchestrator.md @@ -213,7 +213,6 @@ Inside this folder should be the .tflite model and if needed a .txt file with th "your_new_model_name": { "category": "classification", "version": 1, - "pb_file_path": "modelforward/your_new_model_name", "class_names": [ "class name 1", "class name 2", @@ -230,7 +229,6 @@ Inside this folder should be the .tflite model and if needed a .txt file with th "your_new_model_name": { "category": "object_detection", "version": 1, - "pb_file_path": "modelforward/your_new_model_name", "class_names_path": "{name of file with the class names}.txt", "output": { "boxes_coordinates": "{name of the boxes_coordinates variable in your model}", diff --git a/edge_orchestrator/config/.active_config b/edge_orchestrator/config/.active_config deleted file mode 100644 index e69de29b..00000000 diff --git a/edge_orchestrator/config/.active_config.json b/edge_orchestrator/config/.active_config.json new file mode 100644 index 00000000..d91b003a --- /dev/null +++ b/edge_orchestrator/config/.active_config.json @@ -0,0 +1,65 @@ +{ + "infra": { + "binary_storage": { + "type": "filesystem", + "params": { + "src_directory": "/tmp/vio/edge_orchestrator/data/storage" + } + }, + "metadata_storage": { + "type": "filesystem" + }, + "mode1_forward": { + "type": "fake" + }, + "telemetry_sink": { + "type": "fake" + }, + "cameras": [ + { + "name": "camera_#1", + "type": "fake", + "source": "marker_images", + "position": "back", + "exposition": 100 + } + ] + }, + "domain": { + "camera_rules": [ + { + "name": "camera_#1", + "models_graph": [ + { + "name": "marker_quality_control", + "category": "classification", + "version": "1", + "class_names": [ + "OK", + "KO" + ], + "image_resolution": [ + 224, + 224 + ], + "depends_on": [] + } + ], + "rule": { + "name": "expected_label_rule", + "parameters": { + "expected_label": [ + "OK" + ] + } + } + } + ], + "item_rule": { + "name": "min_threshold_KO_rule", + "params": { + "threshold": 1 + } + } + } +} diff --git a/edge_orchestrator/config/inventory.json b/edge_orchestrator/config/inventory.json index 92880c2d..fbc7aba6 100644 --- a/edge_orchestrator/config/inventory.json +++ b/edge_orchestrator/config/inventory.json @@ -29,8 +29,7 @@ "models": { "inception": { "category": "classification", - "version": 1, - "pb_file_path": "modelforward/inception", + "version": "1", "image_resolution": [ 224, 224 @@ -38,8 +37,7 @@ }, "marker_quality_control": { "category": "classification", - "version": 1, - "pb_file_path": "modelforward/marker_quality_control", + "version": "1", "class_names": [ "OK", "KO" @@ -51,8 +49,7 @@ }, "mask_classification_model": { "category": "classification", - "version": 1, - "pb_file_path": "modelforward/mask_classification_model", + "version": "1", "class_names": [ "OK", "KO" @@ -64,8 +61,7 @@ }, "mobilenet_v1_640x640": { "category": "object_detection", - "version": 1, - "pb_file_path": "modelforward/mobilenet_v1_640x640", + "version": "1", "class_names_path": "coco_labels_originals.txt", "output": { "boxes_coordinates": "detection_boxes", @@ -81,8 +77,7 @@ }, "mobilenet_ssd_v2_coco": { "category": "object_detection", - "version": 1, - "pb_file_path": "modelforward/mobilenet_ssd_v2_coco", + "version": "1", "class_names_path": "coco_labels_originals.txt", "output": { "boxes_coordinates": "detection_boxes", @@ -97,8 +92,7 @@ }, "mobilenet_ssd_v2_face": { "category": "object_detection", - "version": 1, - "pb_file_path": "modelforward/mobilenet_ssd_v2_face", + "version": "1", "class_names_path": "face_labels.txt", "output": { "boxes_coordinates": "detection_boxes", @@ -113,8 +107,7 @@ }, "cellphone_connection_control": { "category": "classification", - "version": 1, - "pb_file_path": "modelforward/cellphone_connection_control", + "version": "1", "class_names": [ "unconnected", "connected" diff --git a/edge_orchestrator/config/station_configs/marker_classification_with_1_fake_camera.json b/edge_orchestrator/config/station_configs/marker_classification_with_1_fake_camera.json index 5baac647..fa8d2b44 100644 --- a/edge_orchestrator/config/station_configs/marker_classification_with_1_fake_camera.json +++ b/edge_orchestrator/config/station_configs/marker_classification_with_1_fake_camera.json @@ -1,45 +1,63 @@ { - "binary_storage": { - "type": "filesystem", - "params": { - "src_directory": "/tmp/vio/edge_orchestrator/data/storage" + "infra": { + "binary_storage": { + "type": "filesystem", + "params": { + "src_directory": "/tmp/vio/edge_orchestrator/data/storage" + } + }, + "metadata_storage": { + "type": "filesystem" + }, + "model_forward": { + "type": "fake" + }, + "telemetry_sink": { + "type": "fake" + }, + "cameras": { + "camera_#1": { + "type": "fake", + "source": "marker_images", + "position": "back", + "exposition": 100 + } } }, - "metadata_storage": { - "type": "filesystem" - }, - "model_forward": { - "type": "fake" - }, - "telemetry_sink": { - "type": "fake" - }, - "cameras": { - "camera_#1": { - "type": "fake", - "source": "marker_images", - "position": "back", - "exposition": 100, - "models_graph": { - "model_#4": { - "metadata": "marker_quality_control", - "depends_on": [] - } - }, - "camera_rule": { - "name": "expected_label_rule", - "parameters": { - "expected_label": [ - "OK" - ] + "domain": { + "cameras": { + "camera_#1": { + "models_graph": { + "model_#4": { + "name": "marker_quality_control", + "category": "classification", + "version": "1", + "class_names": [ + "OK", + "KO" + ], + "image_resolution": [ + 224, + 224 + ], + "depends_on": [] + } + }, + "camera_rule": { + "name": "expected_label_rule", + "parameters": { + "expected_label": [ + "OK" + ] + } } } - } - }, - "item_rule": { - "name": "min_threshold_KO_rule", - "parameters": { - "threshold": 1 + }, + "item_rule": { + "name": "min_threshold_KO_rule", + "parameters": { + "threshold": 1 + } } } } diff --git a/edge_orchestrator/config/station_configs/marker_classification_with_1_picamera.json b/edge_orchestrator/config/station_configs/marker_classification_with_1_picamera.json index bbf474b7..a00878cf 100644 --- a/edge_orchestrator/config/station_configs/marker_classification_with_1_picamera.json +++ b/edge_orchestrator/config/station_configs/marker_classification_with_1_picamera.json @@ -1,19 +1,31 @@ { "cameras": { - "camera_id3": { + "camera_#3": { "type": "pi_camera", "position": "front", "exposition": 100, "models_graph": { - "model_id4": { - "metadata": "marker_quality_control", + "model_#4": { + "name": "marker_quality_control", + "category": "classification", + "version": "1", + "class_names": [ + "OK", + "KO" + ], + "image_resolution": [ + 224, + 224 + ], "depends_on": [] } }, "camera_rule": { "name": "expected_label_rule", "parameters": { - "expected_label": ["OK"] + "expected_label": [ + "OK" + ] } } } @@ -24,4 +36,4 @@ "threshold": 1 } } -} \ No newline at end of file +} diff --git a/edge_orchestrator/config/station_configs/marker_classification_with_1_usbcamera.json b/edge_orchestrator/config/station_configs/marker_classification_with_1_usbcamera.json index fe33eb1b..6204f221 100644 --- a/edge_orchestrator/config/station_configs/marker_classification_with_1_usbcamera.json +++ b/edge_orchestrator/config/station_configs/marker_classification_with_1_usbcamera.json @@ -1,20 +1,32 @@ { "cameras": { - "camera_id3": { + "camera_#3": { "type": "usb_camera", "position": "front", "exposition": 100, "source": "/dev/video0", "models_graph": { - "model_id4": { - "metadata": "marker_quality_control", + "model_#4": { + "name": "marker_quality_control", + "category": "classification", + "version": "1", + "class_names": [ + "OK", + "KO" + ], + "image_resolution": [ + 224, + 224 + ], "depends_on": [] } }, "camera_rule": { "name": "expected_label_rule", "parameters": { - "expected_label": ["OK"] + "expected_label": [ + "OK" + ] } } } diff --git a/edge_orchestrator/config/station_configs/marker_classification_with_2_fake_cameras.json b/edge_orchestrator/config/station_configs/marker_classification_with_2_fake_cameras.json index f4e3dd24..2acaf7ca 100644 --- a/edge_orchestrator/config/station_configs/marker_classification_with_2_fake_cameras.json +++ b/edge_orchestrator/config/station_configs/marker_classification_with_2_fake_cameras.json @@ -21,8 +21,18 @@ "position": "back", "exposition": 100, "models_graph": { - "model_id4": { - "metadata": "marker_quality_control", + "model_#4": { + "name": "marker_quality_control", + "category": "classification", + "version": "1", + "class_names": [ + "OK", + "KO" + ], + "image_resolution": [ + 224, + 224 + ], "depends_on": [] } }, @@ -41,8 +51,18 @@ "position": "back", "exposition": 100, "models_graph": { - "model_id4": { - "metadata": "marker_quality_control", + "model_#4": { + "name": "marker_quality_control", + "category": "classification", + "version": "1", + "class_names": [ + "OK", + "KO" + ], + "image_resolution": [ + 224, + 224 + ], "depends_on": [] } }, diff --git a/edge_orchestrator/config/station_configs/person_detection_with_1_fake_camera.json b/edge_orchestrator/config/station_configs/person_detection_with_1_fake_camera.json index 894c4fbd..55732e5f 100644 --- a/edge_orchestrator/config/station_configs/person_detection_with_1_fake_camera.json +++ b/edge_orchestrator/config/station_configs/person_detection_with_1_fake_camera.json @@ -1,13 +1,13 @@ { "cameras": { - "camera_id3": { + "camera_#3": { "type": "fake", "source": "people_dataset", "position": "front", "exposition": 100, "models_graph": { - "model_id5": { - "metadata": "mobilenet_ssd_v2_coco", + "model_#5": { + "name": "mobilenet_ssd_v2_coco", "depends_on": [], "class_to_detect": ["person"] } diff --git a/edge_orchestrator/edge_orchestrator/api_config.py b/edge_orchestrator/edge_orchestrator/api_config.py index 27382e51..197d035e 100644 --- a/edge_orchestrator/edge_orchestrator/api_config.py +++ b/edge_orchestrator/edge_orchestrator/api_config.py @@ -2,19 +2,17 @@ from pathlib import Path from application.config import get_settings +from application.dto.station_config import StationConfig from edge_orchestrator.domain.models.edge_station import EdgeStation from edge_orchestrator.domain.ports.binary_storage import BinaryStorage -from edge_orchestrator.domain.ports.inventory import Inventory from edge_orchestrator.domain.ports.metadata_storage import MetadataStorage from edge_orchestrator.domain.ports.model_forward import ModelForward -from edge_orchestrator.domain.ports.station_config import StationConfig from edge_orchestrator.domain.ports.telemetry_sink import TelemetrySink from edge_orchestrator.domain.use_cases.supervisor import Supervisor from edge_orchestrator.domain.use_cases.uploader import Uploader from edge_orchestrator.infrastructure.binary_storage.binary_storage_factory import ( BinaryStorageFactory, ) -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory from edge_orchestrator.infrastructure.metadata_storage.metadata_storage_factory import ( MetadataStorageFactory, ) @@ -57,18 +55,13 @@ def _get_active_config_name(active_config_path: Path) -> str: return "no_active_configuration" -@lru_cache() -def get_inventory() -> Inventory: - return JsonInventory(get_settings().inventory_path) - - @lru_cache() def get_station_config( try_setting_station_config_from_file: bool = True, ) -> StationConfig: settings = get_settings() station_config = JsonStationConfig( - settings.station_configs_folder, get_inventory(), settings.data_folder + settings.station_configs_folder, settings.data_folder ) if try_setting_station_config_from_file: station_config = set_station_config_from_file( diff --git a/edge_orchestrator/edge_orchestrator/application/api_routes.py b/edge_orchestrator/edge_orchestrator/application/api_routes.py index 9d31e246..8577d170 100644 --- a/edge_orchestrator/edge_orchestrator/application/api_routes.py +++ b/edge_orchestrator/edge_orchestrator/application/api_routes.py @@ -1,27 +1,27 @@ from http import HTTPStatus +from typing import List from fastapi import APIRouter, Depends from starlette.responses import Response from typing_extensions import Annotated +from edge_orchestrator import logger from edge_orchestrator.api_config import ( get_metadata_storage, get_station_config, get_binary_storage, - get_inventory, ) from edge_orchestrator.application.config import ( Settings, get_settings, ) +from edge_orchestrator.application.dto.station_config import StationConfig from edge_orchestrator.application.dto.station_config_dto import StationConfigDto from edge_orchestrator.application.no_active_configuration_exception import ( NoActiveConfigurationException, ) from edge_orchestrator.domain.ports.binary_storage import BinaryStorage -from edge_orchestrator.domain.ports.inventory import Inventory from edge_orchestrator.domain.ports.metadata_storage import MetadataStorage -from edge_orchestrator.domain.ports.station_config import StationConfig api_router = APIRouter() @@ -35,24 +35,24 @@ async def home(settings: Annotated[Settings, Depends(get_settings)]): @api_router.get("/items") def read_all( - metadata_storage: Annotated[MetadataStorage, Depends(get_metadata_storage)] + metadata_storage: Annotated[MetadataStorage, Depends(get_metadata_storage)] ): return metadata_storage.get_all_items_metadata() @api_router.get("/items/{item_id}") def get_item( - item_id: str, - metadata_storage: Annotated[MetadataStorage, Depends(get_metadata_storage)], + item_id: str, + metadata_storage: Annotated[MetadataStorage, Depends(get_metadata_storage)], ): return metadata_storage.get_item_metadata(item_id) @api_router.get("/items/{item_id}/binaries/{camera_id}") def get_item_binary( - item_id: str, - camera_id: str, - binary_storage: BinaryStorage = Depends(get_binary_storage), + item_id: str, + camera_id: str, + binary_storage: BinaryStorage = Depends(get_binary_storage), ): content_binary = binary_storage.get_item_binary(item_id, camera_id) return Response( @@ -63,27 +63,22 @@ def get_item_binary( # @api_router.get("/items/{item_id}/binaries") def get_item_binaries( - item_id: str, - binary_storage: BinaryStorage = Depends(get_binary_storage), + item_id: str, + binary_storage: BinaryStorage = Depends(get_binary_storage), ): return binary_storage.get_item_binaries(item_id) @api_router.get("/items/{item_id}/state") def get_item_state( - item_id: str, - metadata_storage: MetadataStorage = Depends(get_metadata_storage), + item_id: str, + metadata_storage: MetadataStorage = Depends(get_metadata_storage), ): return metadata_storage.get_item_state(item_id) -@api_router.get("/inventory") -def get_inventory(inventory: Inventory = Depends(get_inventory)): - return inventory - - @api_router.get("/configs") -def get_all_configs(station_config: StationConfig = Depends(get_station_config)): +def get_all_configs(station_config: StationConfig = Depends(get_station_config)) -> List[StationConfig]: station_config.load() return station_config.all_configs @@ -96,9 +91,16 @@ def get_active_config(station_config: StationConfig = Depends(get_station_config @api_router.post("/configs/active") -def set_station_config( - station_config_dto: StationConfigDto, - station_config: StationConfig = Depends(get_station_config), +def set_active_config( + station_config_dto: StationConfigDto, + station_config: StationConfig = Depends(get_station_config), ): station_config.set_station_config(station_config_dto.config_name) return station_config.active_config + + +@api_router.post("/config") +def set_config( + station_config_dto: StationConfig, +): + logger.info(f"set config {station_config_dto.to_model()} to") diff --git a/edge_orchestrator/edge_orchestrator/application/config.py b/edge_orchestrator/edge_orchestrator/application/config.py index 72257d5b..df830d01 100644 --- a/edge_orchestrator/edge_orchestrator/application/config.py +++ b/edge_orchestrator/edge_orchestrator/application/config.py @@ -1,22 +1,79 @@ +import json +from abc import ABC, abstractmethod from functools import lru_cache from pathlib import Path +from typing import Dict, Any -from pydantic_settings import BaseSettings, SettingsConfigDict +import yaml +from pydantic_settings import BaseSettings + +from application.dto.station_config import StationConfig +from constants import ACTIVE_CONFIG_FILE_PATH class Settings(BaseSettings): app_name: str = "edge-orchestrator API" url_prefix: str = "/api/v1" - root_dir: Path = Path(__file__).resolve().parents[2] - station_configs_folder: Path = root_dir / "config" / "station_configs" - data_folder: Path = root_dir / "data" - inventory_path: Path = root_dir / "config" / "inventory.json" - active_config_path: Path = root_dir / "config" / ".active_config" + # root_dir: Path = Path(__file__).resolve().parents[2] + # station_configs_folder: Path = root_dir / "config" / "station_configs" + # data_folder: Path = root_dir / "data" + # active_config_path: Path = root_dir / "config" / ".active_config" + # + # model_config = SettingsConfigDict(env_file=".env") + + +class _AbstractConfigReader(ABC): + def __init__(self, path: Path): + self._path = path + self.config = self.get_config() + + def get_config(self) -> StationConfig: + return self._read_file() + + @abstractmethod + def _read_file(self) -> StationConfig: + raise NotImplementedError("Not implemented") + + +class YamlConfigReader(_AbstractConfigReader): + def _read_file(self) -> StationConfig: + content = yaml.load(self._path.read_text(encoding="utf-8"), yaml.SafeLoader) + return StationConfig.from_payload(content) + + +class JsonConfigReader(_AbstractConfigReader): + def _read_file(self) -> StationConfig: + _content = json.loads(self._path.read_text(encoding="utf-8")) + return self.read_content(_content) + + @staticmethod + def read_content(content: Dict[str, Any]) -> StationConfig: + return StationConfig.from_payload(content) + + +class ConfigReader: + def __init__(self, path: Path): + self._path = path + self._reader = self._define_reader() + + def _define_reader(self) -> _AbstractConfigReader: + if self._path.suffixes[0] == ".json": + return JsonConfigReader(self._path) + elif self._path.suffixes[0] in [".yaml", ".yml"]: + return YamlConfigReader(self._path) + raise Exception( + f"Unexpected extension of the deployment file: {self._path}. " + f"Please check the documentation for supported extensions." + ) - model_config = SettingsConfigDict(env_file=".env") + def get_config(self) -> StationConfig: + return self._reader.config @lru_cache() def get_settings() -> Settings: - return Settings() + settings = Settings() + config_reader = ConfigReader(ACTIVE_CONFIG_FILE_PATH) + config = config_reader.get_config() + return settings diff --git a/edge_orchestrator/edge_orchestrator/application/dto/binary_storage_config.py b/edge_orchestrator/edge_orchestrator/application/dto/binary_storage_config.py new file mode 100644 index 00000000..523f2a93 --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/application/dto/binary_storage_config.py @@ -0,0 +1,16 @@ +from enum import Enum +from typing import Optional, Dict, Any + +from pydantic import BaseModel + + +class BinaryStorageTypeEnum(str, Enum): + azure_container = "azure_container" + filesystem = "filesystem" + gcp_bucket = "gcp_bucket" + in_memory = "in_memory" + + +class BinaryStorageConfig(BaseModel): + type: BinaryStorageTypeEnum = BinaryStorageTypeEnum.filesystem + params: Optional[Dict[str, Any]] = None diff --git a/edge_orchestrator/edge_orchestrator/application/dto/camera_config.py b/edge_orchestrator/edge_orchestrator/application/dto/camera_config.py new file mode 100644 index 00000000..ca3359b3 --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/application/dto/camera_config.py @@ -0,0 +1,55 @@ +from enum import Enum +from typing import Dict, Any, List + +from pydantic import ( + BaseModel, + StringConstraints, PositiveInt, +) +from typing_extensions import Annotated + +from edge_orchestrator.application.dto.model_config import ModelConfig + +CameraName = Annotated[ + str, + StringConstraints( + strip_whitespace=True, to_lower=True, pattern=r"camera_#[0-9]+" + ), +] + + +class CameraTypeEnum(str, Enum): + fake = "fake" + pi_camera = "pi_camera" + usb_camera = "usb_camera" + + +class CameraPositionEnum(str, Enum): + front = "front" + back = "back" + left = "left" + right = "right" + + +class CameraRuleNameEnum(str, Enum): + expected_label_rule = "expected_label_rule" + max_nb_objects_rule = "max_nb_objects_rule" + min_nb_objects_rule = "min_nb_objects_rule" + + +class CameraRule(BaseModel): + name: CameraRuleNameEnum = CameraRuleNameEnum.expected_label_rule + params: Dict[str, Any] = None + + +class CameraConfig(BaseModel): + name: CameraName = "camera_#1" + type: CameraTypeEnum = CameraTypeEnum.fake + source: str + position: CameraPositionEnum = CameraPositionEnum.front + exposition: PositiveInt = 100 + + +class CameraLogic(BaseModel): + name: CameraName = "camera_#1" + models_graph: List[ModelConfig] + rule: CameraRule = CameraRule() diff --git a/edge_orchestrator/edge_orchestrator/application/dto/edge_station_config.py b/edge_orchestrator/edge_orchestrator/application/dto/edge_station_config.py new file mode 100644 index 00000000..049670f8 --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/application/dto/edge_station_config.py @@ -0,0 +1,14 @@ +from enum import Enum +from typing import Optional, Dict, Any + +from pydantic import BaseModel + + +class ItemRuleNameEnum(str, Enum): + min_threshold_KO_rule = "min_threshold_KO_rule" + threshold_ratio_rule = "threshold_ratio_rule" + + +class ItemRule(BaseModel): + name: ItemRuleNameEnum = ItemRuleNameEnum.min_threshold_KO_rule + params: Optional[Dict[str, Any]] = None diff --git a/edge_orchestrator/edge_orchestrator/application/dto/metadata_storage_config.py b/edge_orchestrator/edge_orchestrator/application/dto/metadata_storage_config.py new file mode 100644 index 00000000..cc90fbf3 --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/application/dto/metadata_storage_config.py @@ -0,0 +1,17 @@ +from enum import Enum +from typing import Optional, Dict, Any + +from pydantic import BaseModel + + +class MetadataStorageTypeEnum(str, Enum): + azure_container = "azure_container" + filesystem = "filesystem" + gcp_bucket = "gcp_bucket" + in_memory = "in_memory" + mongo_db = "mongo_db" + + +class MetadataStorageConfig(BaseModel): + type: MetadataStorageTypeEnum = MetadataStorageTypeEnum.filesystem + params: Optional[Dict[str, Any]] = None diff --git a/edge_orchestrator/edge_orchestrator/application/dto/model_config.py b/edge_orchestrator/edge_orchestrator/application/dto/model_config.py new file mode 100644 index 00000000..899a5c25 --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/application/dto/model_config.py @@ -0,0 +1,81 @@ +from enum import Enum +from typing import Optional, List + +import annotated_types +from pydantic import ( + StringConstraints, + BaseModel, + FilePath, + PositiveInt, + Field, + model_validator, +) +from typing_extensions import Annotated + + +class ModelNameEnum(str, Enum): + inception = "inception" + mask_classification_model = "mask_classification_model" + marker_quality_control = "marker_quality_control" + mobilenet_v1_640x640 = "mobilenet_v1_640x640" + mobilenet_ssd_v2_coco = "mobilenet_ssd_v2_coco" + mobilenet_ssd_v2_face = "mobilenet_ssd_v2_face" + cellphone_connection_control = "cellphone_connection_control" + + +class ModelCategoryEnum(str, Enum): + classification = "classification" + object_detection = "object_detection" + + +Version = Annotated[ + str, + StringConstraints(pattern=r"^(0|[1-9]\d*)(\.(0|[1-9]\d*)){0,2}"), +] + + +class ModelOutput(BaseModel): + boxes_coordinates: str + objectness_scores: str + number_of_boxes: Optional[str] = None + detection_classes: str + + +class ModelConfig(BaseModel): + name: ModelNameEnum + category: ModelCategoryEnum + version: Version + class_names: Optional[List[str]] = None + class_names_path: Optional[FilePath] = None + output: Optional[ModelOutput] = None + image_resolution: List[PositiveInt] = Field(min_items=2, max_items=2) + objectness_threshold: Optional[ + Annotated[float, annotated_types.Interval(gt=0, le=1.0)] + ] = None + depends_on: Optional[List[str]] = None + class_to_detect: Optional[List[str]] = None + + @model_validator(mode="after") + def check_class_names_or_class_names_path(self) -> "ModelConfig": + if (not self.class_names and not self.class_names_path) or ( + self.class_names and self.class_names_path + ): + raise ValueError( + "Either class_names or class_names_path is required (not both)" + ) + return self + + @model_validator(mode="after") + def check_object_detection_model(self) -> "ModelConfig": + if self.category is ModelCategoryEnum.object_detection: + if not self.output: + raise ValueError("output is required with object_detection category") + if not self.objectness_threshold: + raise ValueError( + "objectness_threshold is required with object_detection category" + ) + if not self.class_to_detect: + raise ValueError( + "class_to_detect is required with object_detection category" + ) + return self diff --git a/edge_orchestrator/edge_orchestrator/application/dto/model_forward_config.py b/edge_orchestrator/edge_orchestrator/application/dto/model_forward_config.py new file mode 100644 index 00000000..8c2732f3 --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/application/dto/model_forward_config.py @@ -0,0 +1,14 @@ +from enum import Enum +from typing import Optional, Dict, Any + +from pydantic import BaseModel + + +class ModelForwardTypeEnum(str, Enum): + fake = "fake" + tf_serving = "tf_serving" + + +class ModelForwardConfig(BaseModel): + type: ModelForwardTypeEnum + params: Optional[Dict[str, Any]] = None diff --git a/edge_orchestrator/edge_orchestrator/application/dto/station_config.py b/edge_orchestrator/edge_orchestrator/application/dto/station_config.py new file mode 100644 index 00000000..2dde348c --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/application/dto/station_config.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import logging +from typing import Dict, Any, Optional, List + +from pydantic import BaseModel + +from edge_orchestrator.application.dto.binary_storage_config import BinaryStorageConfig +from edge_orchestrator.application.dto.camera_config import CameraConfig, CameraLogic +from edge_orchestrator.application.dto.edge_station_config import ItemRule +from edge_orchestrator.application.dto.metadata_storage_config import MetadataStorageConfig +from edge_orchestrator.application.dto.model_forward_config import ModelForwardConfig +from edge_orchestrator.application.dto.telemetry_sink_config import TelemetrySinkDto + + +class InfrastructureConfig(BaseModel): + binary_storage: BinaryStorageConfig + metadata_storage: MetadataStorageConfig + mode1_forward: ModelForwardConfig + telemetry_sink: TelemetrySinkDto + cameras: List[CameraConfig] + + +class DomainRulesConfig(BaseModel): + camera_rules: List[CameraLogic] + item_rule: ItemRule = ItemRule() + + +class StationConfig(BaseModel): + infra_config: Optional[InfrastructureConfig] + domain_config: Optional[DomainRulesConfig] + + def to_model(self): + print(self) + return + + @staticmethod + def _prepare_infra(payload: Dict[str, Any]) -> InfrastructureConfig: + _infra_payload = payload.get("infra", {}) + if not _infra_payload: + logging.info("No infra logic defined in the config file. Default infra logic will be used.") + _infra_payload = {"binary_storage": BinaryStorageConfig(), "metadata_storage": MetadataStorageConfig(), + "mode1_forward": ModelForwardConfig(), "telemetry": TelemetrySinkDto(), + "cameras": [CameraConfig()]} + return InfrastructureConfig(**_infra_payload) + + @staticmethod + def _prepare_domain_rules(payload: Dict[str, Any]) -> DomainRulesConfig: + _domain_rules_payload = payload.get("domain", {}) + if not _domain_rules_payload: + logging.info("No domain logic defined in the config file. Default domain logic will be used.") + _domain_rules_payload = {"camera_rules": [CameraLogic()], "item_rule": ItemRule()} + return DomainRulesConfig(**_domain_rules_payload) + + @classmethod + def from_payload(cls, payload: Dict[str, Any]) -> StationConfig: + return StationConfig(infra_config=cls._prepare_infra(payload), + domain_config=cls._prepare_domain_rules(payload)) diff --git a/edge_orchestrator/edge_orchestrator/application/dto/telemetry_sink_config.py b/edge_orchestrator/edge_orchestrator/application/dto/telemetry_sink_config.py new file mode 100644 index 00000000..14880012 --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/application/dto/telemetry_sink_config.py @@ -0,0 +1,15 @@ +from enum import Enum +from typing import Optional, Dict, Any + +from pydantic import BaseModel + + +class TelemetrySinkTypeEnum(str, Enum): + azure_iot_hub = "azure_iot_hub" + fake = "fake" + postgres = "postgres" + + +class TelemetrySinkDto(BaseModel): + type: TelemetrySinkTypeEnum = TelemetrySinkTypeEnum.fake + params: Optional[Dict[str, Any]] = None diff --git a/edge_orchestrator/edge_orchestrator/application/trigger_routes.py b/edge_orchestrator/edge_orchestrator/application/trigger_routes.py index e4e2bc5d..5876193d 100644 --- a/edge_orchestrator/edge_orchestrator/application/trigger_routes.py +++ b/edge_orchestrator/edge_orchestrator/application/trigger_routes.py @@ -2,13 +2,13 @@ from fastapi.responses import JSONResponse from typing_extensions import Annotated +from application.dto.station_config import StationConfig from edge_orchestrator.api_config import ( get_supervisor, get_station_config, get_uploader, ) from edge_orchestrator.domain.models.item import Item -from edge_orchestrator.domain.ports.station_config import StationConfig trigger_router = APIRouter() diff --git a/edge_orchestrator/edge_orchestrator/constants.py b/edge_orchestrator/edge_orchestrator/constants.py new file mode 100644 index 00000000..29adc544 --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/constants.py @@ -0,0 +1,5 @@ +from pathlib import Path + +ORCHESTRATOR_PATH = Path(".") +CONFIG_PATH = ORCHESTRATOR_PATH / "config" +ACTIVE_CONFIG_FILE_PATH = CONFIG_PATH / ".active_config.json" diff --git a/edge_orchestrator/edge_orchestrator/domain/models/edge_station.py b/edge_orchestrator/edge_orchestrator/domain/models/edge_station.py index d227a6e3..2b963aa4 100644 --- a/edge_orchestrator/edge_orchestrator/domain/models/edge_station.py +++ b/edge_orchestrator/edge_orchestrator/domain/models/edge_station.py @@ -1,7 +1,7 @@ from typing import Dict, Tuple, Any, List +from application.dto.station_config import StationConfig from domain.models.camera import Camera -from edge_orchestrator.domain.ports.station_config import StationConfig class EdgeStation: diff --git a/edge_orchestrator/edge_orchestrator/domain/models/model_infos.py b/edge_orchestrator/edge_orchestrator/domain/models/model_infos.py index 8ebff8ae..e86817ea 100644 --- a/edge_orchestrator/edge_orchestrator/domain/models/model_infos.py +++ b/edge_orchestrator/edge_orchestrator/domain/models/model_infos.py @@ -1,10 +1,7 @@ -import os from enum import Enum from pathlib import Path from typing import Dict, List, Optional -from edge_orchestrator.domain.ports.inventory import Inventory - class ModelInfos: def __init__( @@ -47,30 +44,21 @@ def from_model_graph_node( camera_id: str, model_id: str, model: Dict, - inventory: Inventory, data_folder: Path, ): - model_type = model["metadata"] - class_names = inventory.models[model_type].get("class_names") + model_name = model["name"] + class_names = model.get("class_names") class_to_detect = model.get("class_to_detect") - class_names_path = inventory.models[model_type].get("class_names_path") - objectness_threshold = inventory.models[model_type].get("objectness_threshold") + class_names_path = model.get("class_names_path") + objectness_threshold = model.get("objectness_threshold") - if inventory.models[model_type].get("class_names_path") is not None: - class_names_path = os.path.join(data_folder, class_names_path) + if class_names_path is not None: + class_names_path = data_folder / class_names_path try: - boxes_coordinates = ( - inventory.models[model_type].get("output").get("boxes_coordinates") - ) - objectness_scores = ( - inventory.models[model_type].get("output").get("objectness_scores") - ) - number_of_boxes = ( - inventory.models[model_type].get("output").get("number_of_boxes") - ) - detection_classes = ( - inventory.models[model_type].get("output").get("detection_classes") - ) + boxes_coordinates = model.get("output").get("boxes_coordinates") + objectness_scores = model.get("output").get("objectness_scores") + number_of_boxes = model.get("output").get("number_of_boxes") + detection_classes = model.get("output").get("detection_classes") except AttributeError: boxes_coordinates = None objectness_scores = None @@ -79,9 +67,9 @@ def from_model_graph_node( return ModelInfos( id=model_id, - name=model_type, - category=inventory.models[model_type]["category"], - version=str(inventory.models[model_type]["version"]), + name=model_name, + category=model["category"], + version=model["version"], depends_on=model["depends_on"], camera_id=camera_id, class_names=class_names, @@ -90,7 +78,7 @@ def from_model_graph_node( objectness_scores=objectness_scores, number_of_boxes=number_of_boxes, detection_classes=detection_classes, - image_resolution=inventory.models[model_type].get("image_resolution"), + image_resolution=model.get("image_resolution"), class_to_detect=class_to_detect, objectness_threshold=objectness_threshold, ) diff --git a/edge_orchestrator/edge_orchestrator/domain/ports/inventory.py b/edge_orchestrator/edge_orchestrator/domain/ports/inventory.py deleted file mode 100644 index c5d0959a..00000000 --- a/edge_orchestrator/edge_orchestrator/domain/ports/inventory.py +++ /dev/null @@ -1,8 +0,0 @@ -from typing import Dict, List, Union - - -class Inventory: - cameras: List[str] - models: Dict[str, Dict[str, Union[str, int]]] - camera_rules: List[str] - item_rules: List[str] diff --git a/edge_orchestrator/edge_orchestrator/domain/ports/station_config.py b/edge_orchestrator/edge_orchestrator/domain/ports/station_config.py index 741c94d7..8b137891 100644 --- a/edge_orchestrator/edge_orchestrator/domain/ports/station_config.py +++ b/edge_orchestrator/edge_orchestrator/domain/ports/station_config.py @@ -1,31 +1 @@ -from abc import abstractmethod -from typing import Dict, List, Optional, Type, Union -from edge_orchestrator.domain.models.camera import Camera -from edge_orchestrator.domain.models.model_infos import ModelInfos - - -class StationConfig: - all_configs: dict - active_config_name: Optional[str] - active_config: Optional[dict] - - @abstractmethod - def get_model_pipeline_for_camera(self, camera_id: str) -> List[ModelInfos]: - pass - - @abstractmethod - def get_cameras(self) -> List[str]: - pass - - @abstractmethod - def get_camera_type(self, camera_id: str) -> Type[Camera]: - pass - - @abstractmethod - def get_camera_settings(self, camera_id: str) -> Dict[str, Union[str, int]]: - pass - - @abstractmethod - def set_station_config(self, config_name: str) -> None: - pass diff --git a/edge_orchestrator/edge_orchestrator/domain/use_cases/supervisor.py b/edge_orchestrator/edge_orchestrator/domain/use_cases/supervisor.py index 485be216..e5742ee1 100644 --- a/edge_orchestrator/edge_orchestrator/domain/use_cases/supervisor.py +++ b/edge_orchestrator/edge_orchestrator/domain/use_cases/supervisor.py @@ -6,6 +6,7 @@ from PIL import Image +from application.dto.station_config import StationConfig from edge_orchestrator import logger from edge_orchestrator.domain.models.camera import ( get_camera_rule, @@ -20,7 +21,6 @@ from edge_orchestrator.domain.ports.binary_storage import BinaryStorage from edge_orchestrator.domain.ports.metadata_storage import MetadataStorage from edge_orchestrator.domain.ports.model_forward import ModelForward -from edge_orchestrator.domain.ports.station_config import StationConfig from edge_orchestrator.domain.ports.telemetry_sink import TelemetrySink diff --git a/edge_orchestrator/edge_orchestrator/environment/__init__.py b/edge_orchestrator/edge_orchestrator/environment/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/edge_orchestrator/edge_orchestrator/environment/abstract_config.py b/edge_orchestrator/edge_orchestrator/environment/abstract_config.py deleted file mode 100644 index 825ce8ac..00000000 --- a/edge_orchestrator/edge_orchestrator/environment/abstract_config.py +++ /dev/null @@ -1,43 +0,0 @@ -from abc import ABC -from pathlib import Path - -from edge_orchestrator.domain.models.edge_station import EdgeStation -from edge_orchestrator.domain.ports.binary_storage import BinaryStorage -from edge_orchestrator.domain.ports.inventory import Inventory -from edge_orchestrator.domain.ports.metadata_storage import MetadataStorage -from edge_orchestrator.domain.ports.model_forward import ModelForward -from edge_orchestrator.domain.ports.station_config import StationConfig -from edge_orchestrator.domain.ports.telemetry_sink import TelemetrySink - - -class AbstractConfig(ABC): - ROOT_PATH = Path(__file__).parents[2] - - metadata_storage: MetadataStorage = None - model_forward: ModelForward = None - binary_storage: BinaryStorage = None - inventory: Inventory = None - station_config: StationConfig = None - edge_station: EdgeStation = None - telemetry_sink: TelemetrySink = None - - def get_metadata_storage(self): - return self.metadata_storage - - def get_binary_storage(self): - return self.binary_storage - - def get_model_forward(self): - return self.model_forward - - def get_edge_station(self): - return self.edge_station - - def get_inventory(self): - return self.inventory - - def get_station_config(self): - return self.station_config - - def get_telemetry_sink(self): - return self.telemetry_sink diff --git a/edge_orchestrator/edge_orchestrator/environment/default.py b/edge_orchestrator/edge_orchestrator/environment/default.py deleted file mode 100644 index 835888f4..00000000 --- a/edge_orchestrator/edge_orchestrator/environment/default.py +++ /dev/null @@ -1,35 +0,0 @@ -from edge_orchestrator.domain.models.edge_station import EdgeStation -from edge_orchestrator.environment.abstract_config import AbstractConfig -from edge_orchestrator.infrastructure.binary_storage.filesystem_binary_storage import ( - FileSystemBinaryStorage, -) -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory -from edge_orchestrator.infrastructure.metadata_storage.in_memory_metadata_storage import ( - InMemoryMetadataStorage, -) -from edge_orchestrator.infrastructure.model_forward.fake_model_forward import ( - FakeModelForward, -) -from edge_orchestrator.infrastructure.station_config.json_station_config import ( - JsonStationConfig, -) -from edge_orchestrator.infrastructure.telemetry_sink.fake_telemetry_sink import ( - FakeTelemetrySink, -) - - -class Default(AbstractConfig): - def __init__(self): - self.metadata_storage = InMemoryMetadataStorage() - self.model_forward = FakeModelForward() - self.binary_storage = FileSystemBinaryStorage( - self.ROOT_PATH / "data" / "storage" - ) - self.inventory = JsonInventory(self.ROOT_PATH / "config" / "inventory.json") - self.station_config = JsonStationConfig( - self.ROOT_PATH / "config" / "station_configs", - self.inventory, - self.ROOT_PATH / "data", - ) - self.edge_station = EdgeStation(self.station_config) - self.telemetry_sink = FakeTelemetrySink() diff --git a/edge_orchestrator/edge_orchestrator/environment/docker.py b/edge_orchestrator/edge_orchestrator/environment/docker.py deleted file mode 100644 index c4ee0ca6..00000000 --- a/edge_orchestrator/edge_orchestrator/environment/docker.py +++ /dev/null @@ -1,45 +0,0 @@ -import os - -from edge_orchestrator.domain.models.edge_station import EdgeStation -from edge_orchestrator.environment.abstract_config import AbstractConfig -from edge_orchestrator.infrastructure.binary_storage.filesystem_binary_storage import ( - FileSystemBinaryStorage, -) -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory -from edge_orchestrator.infrastructure.metadata_storage.mongo_db_metadata_storage import ( - MongoDbMetadataStorage, -) -from edge_orchestrator.infrastructure.model_forward.tf_serving_wrapper import ( - TFServingWrapper, -) -from edge_orchestrator.infrastructure.station_config.json_station_config import ( - JsonStationConfig, -) -from edge_orchestrator.infrastructure.telemetry_sink.postgres_telemetry_sink import ( - PostgresTelemetrySink, -) - - -class Docker(AbstractConfig): - MONGO_DB_URI = os.environ.get("MONGO_DB_URI", "mongodb://edge_db:27017/") - POSTGRES_DB_URI = os.environ.get( - "POSTGRES_DB_URI", "postgresql://vio:vio@hub_monitoring_db:5432/vio" - ) - SERVING_MODEL_URL = os.environ.get( - "SERVING_MODEL_URL", "http://edge_model_serving:8501" - ) - - def __init__(self): - self.metadata_storage = MongoDbMetadataStorage(self.MONGO_DB_URI) - self.binary_storage = FileSystemBinaryStorage( - self.ROOT_PATH / "data" / "storage" - ) - self.inventory = JsonInventory(self.ROOT_PATH / "config" / "inventory.json") - self.station_config = JsonStationConfig( - station_configs_folder=self.ROOT_PATH / "config" / "station_configs", - inventory=self.inventory, - data_folder=self.ROOT_PATH / "data", - ) - self.edge_station = EdgeStation(self.station_config) - self.model_forward = TFServingWrapper(self.SERVING_MODEL_URL) - self.telemetry_sink = PostgresTelemetrySink(self.POSTGRES_DB_URI) diff --git a/edge_orchestrator/edge_orchestrator/environment/edge_with_azure_container_storage.py b/edge_orchestrator/edge_orchestrator/environment/edge_with_azure_container_storage.py deleted file mode 100644 index 731258e6..00000000 --- a/edge_orchestrator/edge_orchestrator/environment/edge_with_azure_container_storage.py +++ /dev/null @@ -1,39 +0,0 @@ -import os - -from edge_orchestrator.domain.models.edge_station import EdgeStation -from edge_orchestrator.environment.abstract_config import AbstractConfig -from edge_orchestrator.infrastructure.binary_storage.azure_container_binary_storage import ( - AzureContainerBinaryStorage, -) -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory -from edge_orchestrator.infrastructure.metadata_storage.azure_container_metadata_storage import ( - AzureContainerMetadataStorage, -) -from edge_orchestrator.infrastructure.model_forward.tf_serving_wrapper import ( - TFServingWrapper, -) -from edge_orchestrator.infrastructure.station_config.json_station_config import ( - JsonStationConfig, -) -from edge_orchestrator.infrastructure.telemetry_sink.azure_iot_hub_telemetry_sink import ( - AzureIotHubTelemetrySink, -) - - -class EdgeWithAzureContainerStorage(AbstractConfig): - SERVING_MODEL_URL = os.environ.get( - "SERVING_MODEL_URL", "http://edge_model_serving:8501" - ) - - def __init__(self): - self.metadata_storage = AzureContainerMetadataStorage() - self.binary_storage = AzureContainerBinaryStorage() - self.inventory = JsonInventory(self.ROOT_PATH / "config" / "inventory.json") - self.station_config = JsonStationConfig( - self.ROOT_PATH / "config" / "station_configs", - self.inventory, - self.ROOT_PATH / "data", - ) - self.edge_station = EdgeStation(self.station_config) - self.model_forward = TFServingWrapper(self.SERVING_MODEL_URL) - self.telemetry_sink = AzureIotHubTelemetrySink() diff --git a/edge_orchestrator/edge_orchestrator/environment/edge_with_filesystem_metadata_storage.py b/edge_orchestrator/edge_orchestrator/environment/edge_with_filesystem_metadata_storage.py deleted file mode 100644 index b2c5e11b..00000000 --- a/edge_orchestrator/edge_orchestrator/environment/edge_with_filesystem_metadata_storage.py +++ /dev/null @@ -1,43 +0,0 @@ -import os - -from edge_orchestrator.domain.models.edge_station import EdgeStation -from edge_orchestrator.environment.abstract_config import AbstractConfig -from edge_orchestrator.infrastructure.binary_storage.filesystem_binary_storage import ( - FileSystemBinaryStorage, -) -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory -from edge_orchestrator.infrastructure.metadata_storage.filesystem_metadata_storage import ( - FileSystemMetadataStorage, -) -from edge_orchestrator.infrastructure.model_forward.tf_serving_wrapper import ( - TFServingWrapper, -) -from edge_orchestrator.infrastructure.station_config.json_station_config import ( - JsonStationConfig, -) -from edge_orchestrator.infrastructure.telemetry_sink.azure_iot_hub_telemetry_sink import ( - AzureIotHubTelemetrySink, -) - - -class EdgeWithFileSystemMetadataStorage(AbstractConfig): - SERVING_MODEL_URL = os.environ.get( - "SERVING_MODEL_URL", "http://edge_model_serving:8501" - ) - - def __init__(self): - self.metadata_storage = FileSystemMetadataStorage( - self.ROOT_PATH / "data" / "storage" - ) - self.binary_storage = FileSystemBinaryStorage( - self.ROOT_PATH / "data" / "storage" - ) - self.inventory = JsonInventory(self.ROOT_PATH / "config" / "inventory.json") - self.station_config = JsonStationConfig( - self.ROOT_PATH / "config" / "station_configs", - self.inventory, - self.ROOT_PATH / "data", - ) - self.edge_station = EdgeStation(self.station_config) - self.model_forward = TFServingWrapper(self.SERVING_MODEL_URL) - self.telemetry_sink = AzureIotHubTelemetrySink() diff --git a/edge_orchestrator/edge_orchestrator/environment/edge_with_mongo_db_metadata_storage.py b/edge_orchestrator/edge_orchestrator/environment/edge_with_mongo_db_metadata_storage.py deleted file mode 100644 index 60edce41..00000000 --- a/edge_orchestrator/edge_orchestrator/environment/edge_with_mongo_db_metadata_storage.py +++ /dev/null @@ -1,43 +0,0 @@ -import os - -from edge_orchestrator.infrastructure.metadata_storage.mongodb_metadata_storage import ( - MongoDbMetadataStorage, -) - -from edge_orchestrator.domain.models.edge_station import EdgeStation -from edge_orchestrator.environment.abstract_config import AbstractConfig -from edge_orchestrator.infrastructure.binary_storage.filesystem_binary_storage import ( - FileSystemBinaryStorage, -) -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory -from edge_orchestrator.infrastructure.model_forward.tf_serving_wrapper import ( - TFServingWrapper, -) -from edge_orchestrator.infrastructure.station_config.json_station_config import ( - JsonStationConfig, -) -from edge_orchestrator.infrastructure.telemetry_sink.azure_iot_hub_telemetry_sink import ( - AzureIotHubTelemetrySink, -) - - -class EdgeWithMongoDbMetadataStorage(AbstractConfig): - MONGO_DB_URI = os.environ.get("MONGO_DB_URI", "mongodb://edge_db:27017/") - SERVING_MODEL_URL = os.environ.get( - "SERVING_MODEL_URL", "http://edge_model_serving:8501" - ) - - def __init__(self): - self.metadata_storage = MongoDbMetadataStorage(self.MONGO_DB_URI) - self.binary_storage = FileSystemBinaryStorage( - self.ROOT_PATH / "data" / "storage" - ) - self.inventory = JsonInventory(self.ROOT_PATH / "config" / "inventory.json") - self.station_config = JsonStationConfig( - self.ROOT_PATH / "config" / "station_configs", - self.inventory, - self.ROOT_PATH / "data", - ) - self.edge_station = EdgeStation(self.station_config) - self.model_forward = TFServingWrapper(self.SERVING_MODEL_URL) - self.telemetry_sink = AzureIotHubTelemetrySink() diff --git a/edge_orchestrator/edge_orchestrator/environment/test.py b/edge_orchestrator/edge_orchestrator/environment/test.py deleted file mode 100644 index 5cf58f2c..00000000 --- a/edge_orchestrator/edge_orchestrator/environment/test.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -from pathlib import Path - -from edge_orchestrator.infrastructure.metadata_storage.mongodb_metadata_storage import ( - MongoDbMetadataStorage, -) - -from edge_orchestrator.domain.models.edge_station import EdgeStation -from edge_orchestrator.environment.abstract_config import AbstractConfig -from edge_orchestrator.infrastructure.binary_storage.filesystem_binary_storage import ( - FileSystemBinaryStorage, -) -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory -from edge_orchestrator.infrastructure.model_forward.tf_serving_wrapper import ( - TFServingWrapper, -) -from edge_orchestrator.infrastructure.station_config.json_station_config import ( - JsonStationConfig, -) -from edge_orchestrator.infrastructure.telemetry_sink.postgres_telemetry_sink import ( - PostgresTelemetrySink, -) -from tests.conftest import ( - TEST_DATA_FOLDER_PATH, - TEST_INVENTORY_PATH, - TEST_STATION_CONFIGS_FOLDER_PATH, -) - - -class Test(AbstractConfig): - ROOT_PATH = Path("/tests") - MONGO_DB_URI = os.environ.get("MONGO_DB_URI", "mongodb://edge_db:27017/") - POSTGRES_DB_URI = os.environ.get( - "POSTGRES_DB_URI", "postgresql://vio:vio@hub_monitoring_db:5432/vio" - ) - SERVING_MODEL_URL = os.environ.get( - "SERVING_MODEL_URL", "http://edge_model_serving:8501" - ) - - def __init__(self): - self.metadata_storage = MongoDbMetadataStorage(self.MONGO_DB_URI) - self.binary_storage = FileSystemBinaryStorage(TEST_DATA_FOLDER_PATH / "storage") - self.inventory = JsonInventory(TEST_INVENTORY_PATH) - self.station_config = JsonStationConfig( - TEST_STATION_CONFIGS_FOLDER_PATH, self.inventory, TEST_DATA_FOLDER_PATH - ) - self.edge_station = EdgeStation(self.station_config) - self.model_forward = TFServingWrapper(self.SERVING_MODEL_URL) - self.telemetry_sink = PostgresTelemetrySink(self.POSTGRES_DB_URI) diff --git a/edge_orchestrator/edge_orchestrator/environment/upload_with_gcp_bucket.py b/edge_orchestrator/edge_orchestrator/environment/upload_with_gcp_bucket.py deleted file mode 100644 index f32f13d9..00000000 --- a/edge_orchestrator/edge_orchestrator/environment/upload_with_gcp_bucket.py +++ /dev/null @@ -1,40 +0,0 @@ -import os - -from edge_orchestrator.infrastructure.binary_storage.gcp_binary_storage import ( - GCPBinaryStorage, -) -from edge_orchestrator.infrastructure.metadata_storage.gcp_metadata_storage import ( - GCPMetadataStorage, -) - -from edge_orchestrator.domain.models.edge_station import EdgeStation -from edge_orchestrator.environment.abstract_config import AbstractConfig -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory -from edge_orchestrator.infrastructure.model_forward.tf_serving_wrapper import ( - TFServingWrapper, -) -from edge_orchestrator.infrastructure.station_config.json_station_config import ( - JsonStationConfig, -) -from edge_orchestrator.infrastructure.telemetry_sink.fake_telemetry_sink import ( - FakeTelemetrySink, -) - - -class UploadWithGCPBucket(AbstractConfig): - SERVING_MODEL_URL = os.environ.get( - "SERVING_MODEL_URL", "http://edge_model_serving:8501" - ) - - def __init__(self): - self.metadata_storage = GCPMetadataStorage() - self.binary_storage = GCPBinaryStorage() - self.inventory = JsonInventory(self.ROOT_PATH / "config" / "inventory.json") - self.station_config = JsonStationConfig( - self.ROOT_PATH / "config" / "station_configs", - self.inventory, - self.ROOT_PATH / "data", - ) - self.edge_station = EdgeStation(self.station_config) - self.model_forward = TFServingWrapper(self.SERVING_MODEL_URL) - self.telemetry_sink = FakeTelemetrySink() diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/camera/fake_camera.py b/edge_orchestrator/edge_orchestrator/infrastructure/camera/fake_camera.py index 7e1d0b57..6a69617d 100644 --- a/edge_orchestrator/edge_orchestrator/infrastructure/camera/fake_camera.py +++ b/edge_orchestrator/edge_orchestrator/infrastructure/camera/fake_camera.py @@ -4,14 +4,13 @@ from edge_orchestrator import logger from edge_orchestrator.domain.models.camera import Camera -from edge_orchestrator.environment.abstract_config import AbstractConfig class FakeCamera(Camera): def __init__(self, id: str, settings: Dict[str, Union[str, Dict]]): self.id = id self.settings = settings - self.data_folder_path = AbstractConfig.ROOT_PATH / "data" + self.data_folder_path = Path(__file__).parents[2] / "data" self.image_extensions = ["*.jpg", "*.png"] def capture(self) -> bytes: diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/filesystem_helpers.py b/edge_orchestrator/edge_orchestrator/infrastructure/filesystem_helpers.py index 1e0472be..b898a104 100644 --- a/edge_orchestrator/edge_orchestrator/infrastructure/filesystem_helpers.py +++ b/edge_orchestrator/edge_orchestrator/infrastructure/filesystem_helpers.py @@ -2,6 +2,6 @@ def get_tmp_path() -> Path: - tmp_path = Path(f"/tmp/vio/edge_orchestrator/data/storage") + tmp_path = Path("/tmp/vio/edge_orchestrator/data/storage") tmp_path.mkdir(parents=True, exist_ok=True) return tmp_path diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/inventory/__init__.py b/edge_orchestrator/edge_orchestrator/infrastructure/inventory/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/inventory/json_inventory.py b/edge_orchestrator/edge_orchestrator/infrastructure/inventory/json_inventory.py deleted file mode 100644 index 1f1435e9..00000000 --- a/edge_orchestrator/edge_orchestrator/infrastructure/inventory/json_inventory.py +++ /dev/null @@ -1,30 +0,0 @@ -import json -from pathlib import Path -from typing import Dict, List, Union - -from edge_orchestrator.domain.ports.inventory import Inventory - - -class JsonInventory(Inventory): - def __init__(self, inventory_path: Path): - if not inventory_path.exists(): - raise FileNotFoundError(f'No inventory file found at "{inventory_path}"') - - with open(inventory_path, "r") as inventory_file: - content = json.load(inventory_file) - self.cameras = content["cameras"] - self.models = content["models"] - self.camera_rules = content["camera_rules"] - self.item_rules = content["item_rules"] - - def get_cameras(self) -> List[str]: - return self.inventory["cameras"] - - def get_models(self) -> Dict[str, Dict[str, Union[str, int]]]: - return self.inventory["models"] - - def get_camera_rules(self) -> List[str]: - return self.inventory["camera_rules"] - - def get_item_rules(self) -> List[str]: - return self.inventory["item_rules"] diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/model_forward/model_forward_factory.py b/edge_orchestrator/edge_orchestrator/infrastructure/model_forward/model_forward_factory.py index 8063dba9..b4a54b6c 100644 --- a/edge_orchestrator/edge_orchestrator/infrastructure/model_forward/model_forward_factory.py +++ b/edge_orchestrator/edge_orchestrator/infrastructure/model_forward/model_forward_factory.py @@ -20,10 +20,10 @@ class ModelForwardFactory: @lru_cache() def get_model_forward( model_forward_type: Optional[str] = "fake", - **model_forward_config: Optional[Dict[str, Any]], + **model_forward_params: Optional[Dict[str, Any]], ) -> ModelForward: try: - return AVAILABLE_MODEL_FORWARD[model_forward_type](**model_forward_config) + return AVAILABLE_MODEL_FORWARD[model_forward_type](**model_forward_params) except KeyError as err: raise ValueError( f"Unknown model forward type: {model_forward_type}" diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/station_config/json_station_config.py b/edge_orchestrator/edge_orchestrator/infrastructure/station_config/json_station_config.py index ab5411ca..44e3c55b 100644 --- a/edge_orchestrator/edge_orchestrator/infrastructure/station_config/json_station_config.py +++ b/edge_orchestrator/edge_orchestrator/infrastructure/station_config/json_station_config.py @@ -3,12 +3,11 @@ from pathlib import Path from typing import Dict, List, Type, Union +from application.dto.station_config import StationConfig from application.no_active_configuration_exception import NoActiveConfigurationException from edge_orchestrator import logger from edge_orchestrator.domain.models.camera import Camera from edge_orchestrator.domain.models.model_infos import ModelInfos -from edge_orchestrator.domain.ports.inventory import Inventory -from edge_orchestrator.domain.ports.station_config import StationConfig from edge_orchestrator.infrastructure.camera.fake_camera import FakeCamera from edge_orchestrator.infrastructure.camera.raspberry_pi_camera import ( RaspberryPiCamera, @@ -23,10 +22,7 @@ class JsonStationConfig(StationConfig): - def __init__( - self, station_configs_folder: Path, inventory: Inventory, data_folder: Path - ): - self.inventory = inventory + def __init__(self, station_configs_folder: Path, data_folder: Path): self.data_folder = data_folder if not station_configs_folder.exists(): @@ -36,14 +32,14 @@ def __init__( self.station_configs_folder = station_configs_folder self.all_configs = {} - self.load() + self._load() self.active_config = None - config_name = os.environ.get("ACTIVE_CONFIG_NAME", None) + config_name = os.environ.get("ACTIVE_CONFIG_NAME") if config_name is not None: self.set_station_config(config_name) - def load(self): + def _load(self): self.all_configs = {} for config in self.station_configs_folder.glob("*.json"): with open(config, "r") as station_config_file: @@ -72,7 +68,7 @@ def get_model_pipeline_for_camera(self, camera_id: str) -> List[ModelInfos]: if model_pipeline_config: for model_id, model in model_pipeline_config.items(): model_infos = ModelInfos.from_model_graph_node( - camera_id, model_id, model, self.inventory, self.data_folder + camera_id, model_id, model, self.data_folder ) model_pipeline.append(model_infos) else: @@ -108,7 +104,7 @@ def _check_station_config_based_on_inventory(self, content): raise ValueError(f"Camera type {camera_type} is not supported.") self._check_business_rule(camera_conf, "camera") for model_id, model_conf in camera_conf["models_graph"].items(): - model = model_conf["metadata"] + model = model_conf["name"] if model not in self.inventory.models: raise ValueError(f"Model type {model} is not supported.") self._check_business_rule(model_conf, "model") diff --git a/edge_orchestrator/pyproject.toml b/edge_orchestrator/pyproject.toml index d84d5297..e632e748 100644 --- a/edge_orchestrator/pyproject.toml +++ b/edge_orchestrator/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "numpy==1.24.1", "Pillow==9.3.0", "psycopg2-binary==2.9.5", + "pydantic==2.1.1", "pymongo==4.3.3", "smart_open[azure]==6.3.0", "google-cloud-storage==2.2.1", diff --git a/edge_orchestrator/tests/config/inventory_TEST.json b/edge_orchestrator/tests/config/inventory_TEST.json index b7a1b29e..c31e5790 100644 --- a/edge_orchestrator/tests/config/inventory_TEST.json +++ b/edge_orchestrator/tests/config/inventory_TEST.json @@ -6,7 +6,6 @@ "inception": { "category": "classification", "version": 1, - "pb_file_path": "modelforward/inception", "class_names": [ "OK", "KO" @@ -19,7 +18,6 @@ "marker_quality_control": { "category": "classification", "version": 1, - "pb_file_path": "modelforward/marker_quality_control", "class_names": [ "OK", "KO" @@ -32,7 +30,6 @@ "mask_classification_model": { "category": "classification", "version": 1, - "pb_file_path": "modelforward/mask_classification_model", "class_names": [ "OK", "KO" @@ -45,7 +42,6 @@ "mobilenet_v1_640x640": { "category": "object_detection", "version": 1, - "pb_file_path": "modelforward/mobilenet_v1_640x640", "class_names_path": "test_detection_labels", "output": { "boxes_coordinates": "detection_boxes", @@ -58,7 +54,6 @@ "mobilenet_v1_640x640_detect_classif": { "category": "object_detection_with_classification", "version": 1, - "pb_file_path": "modelforward/mobilenet_v1_640x640", "class_names_path": "test_detection_labels", "output": { "boxes_coordinates": "detection_boxes", @@ -77,4 +72,4 @@ "threshold_ratio_rule", "min_threshold_KO_rule" ] -} \ No newline at end of file +} diff --git a/edge_orchestrator/tests/conftest.py b/edge_orchestrator/tests/conftest.py index a5fd9256..503c83e9 100644 --- a/edge_orchestrator/tests/conftest.py +++ b/edge_orchestrator/tests/conftest.py @@ -9,7 +9,6 @@ TEST_STATION_CONFIG_2_PATH = ( TEST_STATION_CONFIGS_FOLDER_PATH / "station_config_TEST2.json" ) -TEST_INVENTORY_PATH = TEST_CONFIG_FOLDER_PATH / "inventory_TEST.json" ROOT_REPOSITORY_PATH = Path(__file__).parents[2] pytest_plugins = [ diff --git a/edge_orchestrator/tests/fixtures/items_config.py b/edge_orchestrator/tests/fixtures/items_config.py index 530bfe0e..7cfe29d1 100644 --- a/edge_orchestrator/tests/fixtures/items_config.py +++ b/edge_orchestrator/tests/fixtures/items_config.py @@ -13,7 +13,6 @@ def test_items_config(): "metadata": { "category": "classification", "name": "inception", - "pb_file_path": "modelforward/inception", "version": "1", }, } @@ -32,7 +31,6 @@ def test_items_config(): "metadata": { "category": "object_detection_with_classification", "name": "yolov3_harnais", - "pb_file_path": "modelforward/yolov3_harnais", "version": "1", }, } @@ -51,7 +49,6 @@ def test_items_config(): "metadata": { "category": "object_detection", "name": "yolov3_harnais", - "pb_file_path": "modelforward/yolov3_harnais", "version": "1", }, }, @@ -60,7 +57,6 @@ def test_items_config(): "metadata": { "category": "classification", "name": "inception", - "pb_file_path": "modelforward/inception", "version": "1", }, }, @@ -79,7 +75,6 @@ def test_items_config(): "metadata": { "category": "object_detection", "name": "yolov3_harnais", - "pb_file_path": "modelforward/yolov3_harnais", "version": "1", }, }, @@ -88,7 +83,6 @@ def test_items_config(): "metadata": { "category": "classification", "name": "inception", - "pb_file_path": "modelforward/inception", "version": "1", }, }, @@ -109,7 +103,6 @@ def test_items_config(): "metadata": { "category": "object_detection", "name": "yolov3_harnais", - "pb_file_path": "modelforward/yolov3_harnais", "version": "1", }, }, @@ -118,7 +111,6 @@ def test_items_config(): "metadata": { "category": "classification", "name": "inception", - "pb_file_path": "modelforward/inception", "version": "1", }, }, diff --git a/edge_orchestrator/tests/fixtures/supervisor_and_collaborators.py b/edge_orchestrator/tests/fixtures/supervisor_and_collaborators.py index 304b8646..f063f441 100644 --- a/edge_orchestrator/tests/fixtures/supervisor_and_collaborators.py +++ b/edge_orchestrator/tests/fixtures/supervisor_and_collaborators.py @@ -8,7 +8,6 @@ from edge_orchestrator.infrastructure.binary_storage.filesystem_binary_storage import ( FileSystemBinaryStorage, ) -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory from edge_orchestrator.infrastructure.metadata_storage.mongo_db_metadata_storage import ( MongoDbMetadataStorage, ) @@ -23,7 +22,6 @@ ) from tests.conftest import ( TEST_DATA_FOLDER_PATH, - TEST_INVENTORY_PATH, TEST_STATION_CONFIGS_FOLDER_PATH, ) @@ -47,16 +45,9 @@ def filesystem_binary_storage(): return FileSystemBinaryStorage(TEST_DATA_FOLDER_PATH / "storage") -@fixture(scope="function") -def json_inventory(): - return JsonInventory(TEST_INVENTORY_PATH) - - @fixture(scope="function") def json_station_config(json_inventory): - return JsonStationConfig( - TEST_STATION_CONFIGS_FOLDER_PATH, json_inventory, TEST_DATA_FOLDER_PATH - ) + return JsonStationConfig(TEST_STATION_CONFIGS_FOLDER_PATH, TEST_DATA_FOLDER_PATH) @fixture(scope="function") diff --git a/edge_orchestrator/tests/functional_tests/supervisor_inventory_route.feature b/edge_orchestrator/tests/functional_tests/supervisor_inventory_route.feature index 8920ca90..477e94fc 100644 --- a/edge_orchestrator/tests/functional_tests/supervisor_inventory_route.feature +++ b/edge_orchestrator/tests/functional_tests/supervisor_inventory_route.feature @@ -13,7 +13,6 @@ Feature: The client requests the inventory available on the station "inception": { "category": "classification", "version": 1, - "pb_file_path": "modelforward/inception", "class_names": [ "OK", "KO" @@ -26,7 +25,6 @@ Feature: The client requests the inventory available on the station "marker_quality_control": { "category": "classification", "version": 1, - "pb_file_path": "modelforward/marker_quality_control", "class_names": [ "OK", "KO" @@ -39,7 +37,6 @@ Feature: The client requests the inventory available on the station "mask_classification_model": { "category": "classification", "version": 1, - "pb_file_path": "modelforward/mask_classification_model", "class_names": [ "OK", "KO" @@ -52,7 +49,6 @@ Feature: The client requests the inventory available on the station "mobilenet_v1_640x640": { "category": "object_detection", "version": 1, - "pb_file_path": "modelforward/mobilenet_v1_640x640", "class_names_path": "test_detection_labels", "output": { "boxes_coordinates": "detection_boxes", @@ -65,7 +61,6 @@ Feature: The client requests the inventory available on the station "mobilenet_v1_640x640_detect_classif": { "category": "object_detection_with_classification", "version": 1, - "pb_file_path": "modelforward/mobilenet_v1_640x640", "class_names_path": "test_detection_labels", "output": { "boxes_coordinates": "detection_boxes", diff --git a/edge_orchestrator/tests/integration_tests/infrastructure/station_config/test_json_station_config.py b/edge_orchestrator/tests/integration_tests/infrastructure/station_config/test_json_station_config.py index b2e42e33..69bc1ac0 100644 --- a/edge_orchestrator/tests/integration_tests/infrastructure/station_config/test_json_station_config.py +++ b/edge_orchestrator/tests/integration_tests/infrastructure/station_config/test_json_station_config.py @@ -5,13 +5,11 @@ from freezegun import freeze_time from edge_orchestrator.domain.models.model_infos import ModelInfos -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory from edge_orchestrator.infrastructure.station_config.json_station_config import ( JsonStationConfig, ) from tests.conftest import ( TEST_DATA_FOLDER_PATH, - TEST_INVENTORY_PATH, TEST_STATION_CONFIGS_FOLDER_PATH, ) @@ -24,9 +22,8 @@ def test_get_models_for_camera_should_return_one_model_infos_when_camera_config_ self, ): # Given - inventory = JsonInventory(TEST_INVENTORY_PATH) json_station_config = JsonStationConfig( - TEST_STATION_CONFIGS_FOLDER_PATH, inventory, TEST_DATA_FOLDER_PATH + TEST_STATION_CONFIGS_FOLDER_PATH, TEST_DATA_FOLDER_PATH ) camera_id = "camera_id3" @@ -40,9 +37,8 @@ def test_get_models_for_camera_should_return_two_model_infos_when_camera_config_ self, ): # Given - inventory = JsonInventory(TEST_INVENTORY_PATH) json_station_config = JsonStationConfig( - TEST_STATION_CONFIGS_FOLDER_PATH, inventory, TEST_DATA_FOLDER_PATH + TEST_STATION_CONFIGS_FOLDER_PATH, TEST_DATA_FOLDER_PATH ) json_station_config.set_station_config("station_config_TEST2") camera_id = "camera_id3" diff --git a/edge_orchestrator/tests/unit_tests/domain/models/test_edge_station.py b/edge_orchestrator/tests/unit_tests/domain/models/test_edge_station.py index 5f1af5a7..bab7f085 100644 --- a/edge_orchestrator/tests/unit_tests/domain/models/test_edge_station.py +++ b/edge_orchestrator/tests/unit_tests/domain/models/test_edge_station.py @@ -2,11 +2,11 @@ from unittest.mock import patch import pytest -from edge_orchestrator.api_config import get_station_config from freezegun import freeze_time +from application.dto.station_config import StationConfig +from edge_orchestrator.api_config import get_station_config from edge_orchestrator.domain.models.edge_station import EdgeStation -from edge_orchestrator.domain.ports.station_config import StationConfig from edge_orchestrator.infrastructure.camera.fake_camera import FakeCamera diff --git a/edge_orchestrator/tests/unit_tests/domain/test_supervisor.py b/edge_orchestrator/tests/unit_tests/domain/test_supervisor.py index 1576d5af..44c1ac77 100644 --- a/edge_orchestrator/tests/unit_tests/domain/test_supervisor.py +++ b/edge_orchestrator/tests/unit_tests/domain/test_supervisor.py @@ -12,7 +12,6 @@ from edge_orchestrator.infrastructure.binary_storage.in_memory_binary_storage import ( InMemoryBinaryStorage, ) -from edge_orchestrator.infrastructure.inventory.json_inventory import JsonInventory from edge_orchestrator.infrastructure.metadata_storage.in_memory_metadata_storage import ( InMemoryMetadataStorage, ) @@ -30,7 +29,6 @@ ) from tests.conftest import ( TEST_DATA_FOLDER_PATH, - TEST_INVENTORY_PATH, TEST_STATION_CONFIGS_FOLDER_PATH, ) @@ -41,8 +39,6 @@ async def test_2_models_in_parallel(self, supervisor): random.seed(42) np.random.seed(42) - inventory = JsonInventory(TEST_INVENTORY_PATH) - models_graph = { "model_1": {"metadata": "inception", "depends_on": []}, "model_2": {"metadata": "inception", "depends_on": []}, @@ -50,7 +46,7 @@ async def test_2_models_in_parallel(self, supervisor): model_pipeline = [ ModelInfos.from_model_graph_node( - "camera_id", model_id, model, inventory, TEST_DATA_FOLDER_PATH + "camera_id", model_id, model, TEST_DATA_FOLDER_PATH ) for model_id, model in models_graph.items() ] @@ -76,8 +72,6 @@ async def test_2_models_in_serie(self, supervisor): random.seed(42) np.random.seed(42) - inventory = JsonInventory(TEST_INVENTORY_PATH) - models_graph = { "model_1": {"metadata": "inception", "depends_on": ["model_2"]}, "model_2": {"metadata": "mobilenet_v1_640x640", "depends_on": []}, @@ -85,7 +79,7 @@ async def test_2_models_in_serie(self, supervisor): model_pipeline = [ ModelInfos.from_model_graph_node( - "camera_id", model_id, model, inventory, TEST_DATA_FOLDER_PATH + "camera_id", model_id, model, TEST_DATA_FOLDER_PATH ) for model_id, model in models_graph.items() ] @@ -580,9 +574,8 @@ async def test_set_decision_should_send_final_decision_to_telemetry_sink( # Given item = Item(serial_number="", category="", cameras_metadata={}, binaries={}) item.id = "item_id" - inventory = JsonInventory(TEST_INVENTORY_PATH) station_config = JsonStationConfig( - TEST_STATION_CONFIGS_FOLDER_PATH, inventory, TEST_DATA_FOLDER_PATH + TEST_STATION_CONFIGS_FOLDER_PATH, TEST_DATA_FOLDER_PATH ) station_config.set_station_config("station_config_TEST") supervisor = Supervisor( @@ -625,9 +618,8 @@ async def test_inspect_should_log_information_about_item_processing( "Entering try Decision", "End of Decision", ] - inventory = JsonInventory(TEST_INVENTORY_PATH) station_config = JsonStationConfig( - TEST_STATION_CONFIGS_FOLDER_PATH, inventory, TEST_DATA_FOLDER_PATH + TEST_STATION_CONFIGS_FOLDER_PATH, TEST_DATA_FOLDER_PATH ) station_config.set_station_config("station_config_TEST") supervisor = Supervisor(