diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index b22a0d22a0..f8fe9954a5 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -31,7 +31,7 @@ jobs: pipenv run flake8 . - name: Run Type Checker run: | - pipenv run mypy . + pipenv run mypy src/manifests tests/tests_manifests - name: Run Tests with Coverage run: | pipenv run coverage run -m pytest --cov=./src --cov-report=xml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 361a4bbb23..f331ad9aaf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - id: mypy stages: [commit] name: mypy - entry: bash -c 'pipenv run mypy .' + entry: bash -c 'pipenv run mypy src/manifests tests/tests_manifests' language: system - id: pytest stages: [commit] diff --git a/pyproject.toml b/pyproject.toml index 796006c6af..fe5ac8c024 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,4 +4,13 @@ target-version = ['py37'] [tool.isort] profile = "black" -line_length = 160 \ No newline at end of file +line_length = 160 + +[tool.mypy] +python_version = 3.7 +disallow_untyped_defs = true +warn_return_any = true +warn_unused_configs = true +strict_optional = false +show_error_codes = true +ignore_missing_imports = true \ No newline at end of file diff --git a/src/build_workflow/builders.py b/src/build_workflow/builders.py index 1a2b7ca726..464fc79313 100644 --- a/src/build_workflow/builders.py +++ b/src/build_workflow/builders.py @@ -8,15 +8,15 @@ from build_workflow.builder_from_dist import BuilderFromDist from build_workflow.builder_from_source import BuilderFromSource -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponentFromDist, InputComponentFromSource class Builders(ABC): @classmethod def builder_from(self, component, target): - if type(component) is InputManifest.ComponentFromDist: + if type(component) is InputComponentFromDist: return BuilderFromDist(component, target) - elif type(component) is InputManifest.ComponentFromSource: + elif type(component) is InputComponentFromSource: return BuilderFromSource(component, target) else: raise ValueError(f"Invalid component type: {type(component)}") diff --git a/src/ci_workflow/ci_check_lists.py b/src/ci_workflow/ci_check_lists.py index 0a7bcb2189..2c3966c114 100644 --- a/src/ci_workflow/ci_check_lists.py +++ b/src/ci_workflow/ci_check_lists.py @@ -8,15 +8,15 @@ from ci_workflow.ci_check_list_dist import CiCheckListDist from ci_workflow.ci_check_list_source import CiCheckListSource -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponentFromDist, InputComponentFromSource class CiCheckLists(ABC): @classmethod def from_component(self, component, target): - if type(component) is InputManifest.ComponentFromDist: + if type(component) is InputComponentFromDist: return CiCheckListDist(component, target) - elif type(component) is InputManifest.ComponentFromSource: + elif type(component) is InputComponentFromSource: return CiCheckListSource(component, target) else: raise ValueError(f"Invalid component type: {type(component)}") diff --git a/src/git/git_repository.py b/src/git/git_repository.py index 4c07e5c299..e399932a95 100644 --- a/src/git/git_repository.py +++ b/src/git/git_repository.py @@ -8,18 +8,20 @@ import os import subprocess from pathlib import Path +from typing import Any, List from system.temporary_directory import TemporaryDirectory class GitRepository: + dir: str """ This class checks out a Git repository at a particular ref into an empty named directory (or temporary a directory if no named directory is given). Temporary directories will be automatically deleted when the GitRepository object goes out of scope; named directories will be left alone. Clients can obtain the actual commit ID by querying the "sha" attribute, and the temp directory name with "dir". """ - def __init__(self, url, ref, directory=None, working_subdirectory=None): + def __init__(self, url: str, ref: str, directory: str = None, working_subdirectory: str = None) -> None: self.url = url self.ref = ref if directory is None: @@ -32,15 +34,14 @@ def __init__(self, url, ref, directory=None, working_subdirectory=None): self.working_subdirectory = working_subdirectory self.__checkout__() - def __enter__(self): + def __enter__(self) -> 'GitRepository': return self - def __exit__(self, exc_type, exc_value, exc_traceback): + def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None: if self.temp_dir: self.temp_dir.__exit__(exc_type, exc_value, exc_traceback) - def __checkout__(self): - # Check out the repository + def __checkout__(self) -> None: self.execute_silent("git init", self.dir) self.execute_silent(f"git remote add origin {self.url}", self.dir) self.execute_silent(f"git fetch --depth 1 origin {self.ref}", self.dir) @@ -49,18 +50,18 @@ def __checkout__(self): logging.info(f"Checked out {self.url}@{self.ref} into {self.dir} at {self.sha}") @property - def working_directory(self): + def working_directory(self) -> str: if self.working_subdirectory: return os.path.join(self.dir, self.working_subdirectory) else: return self.dir @classmethod - def stable_ref(self, url, ref): + def stable_ref(self, url: str, ref: str) -> List[str]: results = subprocess.check_output(f"git ls-remote {url} {ref}", shell=True).decode().strip().split("\t") return results if len(results) > 1 else [ref, ref] - def execute_silent(self, command, cwd=None): + def execute_silent(self, command: str, cwd: str = None) -> None: cwd = cwd or self.working_directory logging.info(f'Executing "{command}" in {cwd}') subprocess.check_call( @@ -71,17 +72,17 @@ def execute_silent(self, command, cwd=None): stderr=subprocess.DEVNULL, ) - def output(self, command, cwd=None): + def output(self, command: str, cwd: str = None) -> str: cwd = cwd or self.working_directory logging.info(f'Executing "{command}" in {cwd}') return subprocess.check_output(command, cwd=cwd, shell=True).decode().strip() - def execute(self, command, cwd=None): + def execute(self, command: str, cwd: str = None) -> None: cwd = cwd or self.working_directory logging.info(f'Executing "{command}" in {cwd}') subprocess.check_call(command, cwd=cwd, shell=True) - def path(self, subdirname=None): + def path(self, subdirname: str = None) -> Path: dirname = self.dir if subdirname: dirname = os.path.join(self.dir, subdirname) diff --git a/src/manifests/build/build_manifest_1_0.py b/src/manifests/build/build_manifest_1_0.py index 2bd6f481b6..1a32e87d92 100644 --- a/src/manifests/build/build_manifest_1_0.py +++ b/src/manifests/build/build_manifest_1_0.py @@ -4,7 +4,8 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -from manifests.component_manifest import ComponentManifest +from typing import Any +from manifests.component_manifest import ComponentManifest, Components, Component """ A BuildManifest is an immutable view of the outputs from a build step @@ -35,7 +36,7 @@ """ -class BuildManifest_1_0(ComponentManifest): +class BuildManifest_1_0(ComponentManifest['BuildManifest_1_0', 'BuildComponents_1_0']): SCHEMA = { "build": { "required": True, @@ -73,29 +74,25 @@ class BuildManifest_1_0(ComponentManifest): }, } - def __init__(self, data): + def __init__(self, data: Any): super().__init__(data) self.build = self.Build(data["build"]) - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "schema-version": "1.0", "build": self.build.__to_dict__(), "components": self.components.__to_dict__() } - class Components(ComponentManifest.Components): - def __create__(self, data): - return BuildManifest_1_0.Component(data) - class Build: - def __init__(self, data): + def __init__(self, data: Any): self.name = data["name"] self.version = data["version"] self.architecture = data["architecture"] self.id = data["id"] - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "name": self.name, "version": self.version, @@ -103,21 +100,28 @@ def __to_dict__(self): "id": self.id } - class Component(ComponentManifest.Component): - def __init__(self, data): - super().__init__(data) - self.repository = data["repository"] - self.ref = data["ref"] - self.commit_id = data["commit_id"] - self.artifacts = data.get("artifacts", {}) - self.version = data["version"] - def __to_dict__(self): - return { - "name": self.name, - "repository": self.repository, - "ref": self.ref, - "commit_id": self.commit_id, - "artifacts": self.artifacts, - "version": self.version, - } +class BuildComponents_1_0(Components['BuildComponent_1_0']): + @classmethod + def __create__(self, data: Any) -> 'BuildComponent_1_0': + return BuildComponent_1_0(data) + + +class BuildComponent_1_0(Component): + def __init__(self, data: Any): + super().__init__(data) + self.repository = data["repository"] + self.ref = data["ref"] + self.commit_id = data["commit_id"] + self.artifacts = data.get("artifacts", {}) + self.version = data["version"] + + def __to_dict__(self) -> dict: + return { + "name": self.name, + "repository": self.repository, + "ref": self.ref, + "commit_id": self.commit_id, + "artifacts": self.artifacts, + "version": self.version, + } diff --git a/src/manifests/build/build_manifest_1_1.py b/src/manifests/build/build_manifest_1_1.py index 3da9a26d1d..5e2149bad0 100644 --- a/src/manifests/build/build_manifest_1_1.py +++ b/src/manifests/build/build_manifest_1_1.py @@ -4,7 +4,8 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -from manifests.component_manifest import ComponentManifest +from typing import Any +from manifests.component_manifest import ComponentManifest, Components, Component """ A BuildManifest is an immutable view of the outputs from a build step @@ -36,7 +37,7 @@ """ -class BuildManifest_1_1(ComponentManifest): +class BuildManifest_1_1(ComponentManifest['BuildManifest_1_1', 'BuildComponents_1_1']): SCHEMA = { "build": { "required": True, @@ -74,11 +75,11 @@ class BuildManifest_1_1(ComponentManifest): }, } - def __init__(self, data): + def __init__(self, data: Any): super().__init__(data) self.build = self.Build(data["build"]) - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "schema-version": "1.1", "build": self.build.__to_dict__(), @@ -86,13 +87,13 @@ def __to_dict__(self): } class Build: - def __init__(self, data): + def __init__(self, data: Any): self.name = data["name"] self.version = data["version"] self.architecture = data["architecture"] self.id = data["id"] - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "name": self.name, "version": self.version, @@ -100,25 +101,28 @@ def __to_dict__(self): "id": self.id } - class Components(ComponentManifest.Components): - def __create__(self, data): - return BuildManifest_1_1.Component(data) - class Component(ComponentManifest.Component): - def __init__(self, data): - super().__init__(data) - self.repository = data["repository"] - self.ref = data["ref"] - self.commit_id = data["commit_id"] - self.artifacts = data.get("artifacts", {}) - self.version = data["version"] +class BuildComponents_1_1(Components['BuildComponent_1_1']): + @classmethod + def __create__(self, data: Any) -> 'BuildComponent_1_1': + return BuildComponent_1_1(data) - def __to_dict__(self): - return { - "name": self.name, - "repository": self.repository, - "ref": self.ref, - "commit_id": self.commit_id, - "artifacts": self.artifacts, - "version": self.version, - } + +class BuildComponent_1_1(Component): + def __init__(self, data: Any): + super().__init__(data) + self.repository = data["repository"] + self.ref = data["ref"] + self.commit_id = data["commit_id"] + self.artifacts = data.get("artifacts", {}) + self.version = data["version"] + + def __to_dict__(self) -> dict: + return { + "name": self.name, + "repository": self.repository, + "ref": self.ref, + "commit_id": self.commit_id, + "artifacts": self.artifacts, + "version": self.version, + } diff --git a/src/manifests/build_manifest.py b/src/manifests/build_manifest.py index dac16d03ef..9eac84c1a8 100644 --- a/src/manifests/build_manifest.py +++ b/src/manifests/build_manifest.py @@ -4,9 +4,11 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. +from typing import Any + from manifests.build.build_manifest_1_0 import BuildManifest_1_0 from manifests.build.build_manifest_1_1 import BuildManifest_1_1 -from manifests.component_manifest import ComponentManifest +from manifests.component_manifest import Component, ComponentManifest, Components """ A BuildManifest is an immutable view of the outputs from a build step @@ -39,9 +41,7 @@ """ -class BuildManifest(ComponentManifest): - components: list - +class BuildManifest(ComponentManifest['BuildManifest', 'BuildComponents']): VERSIONS = { "1.0": BuildManifest_1_0, "1.1": BuildManifest_1_1, @@ -86,11 +86,12 @@ class BuildManifest(ComponentManifest): }, } - def __init__(self, data): + def __init__(self, data: Any): super().__init__(data) self.build = self.Build(data["build"]) + self.components = BuildComponents(data.get("components", [])) # type: ignore[assignment] - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "schema-version": "1.2", "build": self.build.__to_dict__(), @@ -98,14 +99,14 @@ def __to_dict__(self): } class Build: - def __init__(self, data): + def __init__(self, data: Any): self.name = data["name"] self.version = data["version"] self.platform = data["platform"] self.architecture = data["architecture"] self.id = data["id"] - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "name": self.name, "version": self.version, @@ -114,29 +115,31 @@ def __to_dict__(self): "id": self.id } - class Components(ComponentManifest.Components): - @classmethod - def __create__(self, data): - return BuildManifest.Component(data) - - class Component(ComponentManifest.Component): - def __init__(self, data): - super().__init__(data) - self.repository = data["repository"] - self.ref = data["ref"] - self.commit_id = data["commit_id"] - self.artifacts = data.get("artifacts", {}) - self.version = data["version"] - def __to_dict__(self): - return { - "name": self.name, - "repository": self.repository, - "ref": self.ref, - "commit_id": self.commit_id, - "artifacts": self.artifacts, - "version": self.version, - } +class BuildComponents(Components['BuildComponent']): + @classmethod + def __create__(self, data: Any) -> 'BuildComponent': + return BuildComponent(data) + + +class BuildComponent(Component): + def __init__(self, data: Any): + super().__init__(data) + self.repository = data["repository"] + self.ref = data["ref"] + self.commit_id = data["commit_id"] + self.artifacts = data.get("artifacts", {}) + self.version = data["version"] + + def __to_dict__(self) -> dict: + return { + "name": self.name, + "repository": self.repository, + "ref": self.ref, + "commit_id": self.commit_id, + "artifacts": self.artifacts, + "version": self.version, + } BuildManifest.VERSIONS = {"1.0": BuildManifest_1_0, "1.1": BuildManifest_1_1, "1.2": BuildManifest} diff --git a/src/manifests/bundle/bundle_manifest_1_0.py b/src/manifests/bundle/bundle_manifest_1_0.py index f4c6398111..2722cd1de6 100644 --- a/src/manifests/bundle/bundle_manifest_1_0.py +++ b/src/manifests/bundle/bundle_manifest_1_0.py @@ -4,10 +4,12 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -from manifests.component_manifest import ComponentManifest +from typing import Any +from manifests.component_manifest import Component, ComponentManifest, Components -class BundleManifest_1_0(ComponentManifest): + +class BundleManifest_1_0(ComponentManifest['BundleManifest_1_0', 'BundleComponents_1_0']): """ A BundleManifest is an immutable view of the outputs from a assemble step The manifest contains information about the bundle that was built (in the `assemble` section), @@ -57,11 +59,11 @@ class BundleManifest_1_0(ComponentManifest): }, } - def __init__(self, data): + def __init__(self, data: Any): super().__init__(data) self.build = self.Build(data["build"]) - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "schema-version": "1.0", "build": self.build.__to_dict__(), @@ -69,14 +71,14 @@ def __to_dict__(self): } class Build: - def __init__(self, data): + def __init__(self, data: Any): self.name = data["name"] self.version = data["version"] self.architecture = data["architecture"] self.location = data["location"] self.id = data["id"] - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "name": self.name, "version": self.version, @@ -85,23 +87,26 @@ def __to_dict__(self): "id": self.id } - class Components(ComponentManifest.Components): - def __create__(self, data): - return BundleManifest_1_0.Component(data) - class Component(ComponentManifest.Component): - def __init__(self, data): - super().__init__(data) - self.repository = data["repository"] - self.ref = data["ref"] - self.commit_id = data["commit_id"] - self.location = data["location"] +class BundleComponents_1_0(Components): + @classmethod + def __create__(self, data: Any) -> 'BundleComponent_1_0': + return BundleComponent_1_0(data) - def __to_dict__(self): - return { - "name": self.name, - "repository": self.repository, - "ref": self.ref, - "commit_id": self.commit_id, - "location": self.location - } + +class BundleComponent_1_0(Component): + def __init__(self, data: Any): + super().__init__(data) + self.repository = data["repository"] + self.ref = data["ref"] + self.commit_id = data["commit_id"] + self.location = data["location"] + + def __to_dict__(self) -> dict: + return { + "name": self.name, + "repository": self.repository, + "ref": self.ref, + "commit_id": self.commit_id, + "location": self.location + } diff --git a/src/manifests/bundle_manifest.py b/src/manifests/bundle_manifest.py index 193203bae1..5072e4c2b2 100644 --- a/src/manifests/bundle_manifest.py +++ b/src/manifests/bundle_manifest.py @@ -4,11 +4,13 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. +from typing import Any, Dict + from manifests.bundle.bundle_manifest_1_0 import BundleManifest_1_0 -from manifests.component_manifest import ComponentManifest +from manifests.component_manifest import Component, ComponentManifest, Components -class BundleManifest(ComponentManifest): +class BundleManifest(ComponentManifest['BundleManifest', 'BundleComponents']): """ A BundleManifest is an immutable view of the outputs from a assemble step The manifest contains information about the bundle that was built (in the `assemble` section), @@ -60,11 +62,12 @@ class BundleManifest(ComponentManifest): }, } - def __init__(self, data): + def __init__(self, data: Any) -> None: super().__init__(data) self.build = self.Build(data["build"]) + self.components = BundleComponents(data.get("components", [])) # type: ignore[assignment] - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "schema-version": "1.1", "build": self.build.__to_dict__(), @@ -72,7 +75,7 @@ def __to_dict__(self): } class Build: - def __init__(self, data): + def __init__(self, data: Dict[str, str]): self.name = data["name"] self.version = data["version"] self.platform = data["platform"] @@ -80,7 +83,7 @@ def __init__(self, data): self.location = data["location"] self.id = data["id"] - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "name": self.name, "version": self.version, @@ -90,27 +93,29 @@ def __to_dict__(self): "id": self.id, } - class Components(ComponentManifest.Components): - @classmethod - def __create__(self, data): - return BundleManifest.Component(data) - - class Component(ComponentManifest.Component): - def __init__(self, data): - super().__init__(data) - self.repository = data["repository"] - self.ref = data["ref"] - self.commit_id = data["commit_id"] - self.location = data["location"] - def __to_dict__(self): - return { - "name": self.name, - "repository": self.repository, - "ref": self.ref, - "commit_id": self.commit_id, - "location": self.location - } +class BundleComponents(Components['BundleComponent']): + @classmethod + def __create__(self, data: Any) -> 'BundleComponent': + return BundleComponent(data) + + +class BundleComponent(Component): + def __init__(self, data: Any): + super().__init__(data) + self.repository = data["repository"] + self.ref = data["ref"] + self.commit_id = data["commit_id"] + self.location = data["location"] + + def __to_dict__(self) -> dict: + return { + "name": self.name, + "repository": self.repository, + "ref": self.ref, + "commit_id": self.commit_id, + "location": self.location + } BundleManifest.VERSIONS = {"1.0": BundleManifest_1_0, "1.1": BundleManifest} diff --git a/src/manifests/component_manifest.py b/src/manifests/component_manifest.py index 5b23737236..6919da3ce2 100644 --- a/src/manifests/component_manifest.py +++ b/src/manifests/component_manifest.py @@ -6,11 +6,17 @@ import itertools import logging +from typing import Any, Callable, Dict, Generic, Iterator, List, Tuple, TypeVar from manifests.manifest import Manifest +ManifestType = TypeVar('ManifestType', bound='ComponentManifest') +ComponentsType = TypeVar('ComponentsType', bound='Components') +ComponentType = TypeVar('ComponentType', bound='Component') -class ComponentManifest(Manifest): + +class ComponentManifest(Manifest[ManifestType], Generic[ManifestType, ComponentsType]): + components: ComponentsType SCHEMA = { "schema-version": { "required": True, "type": "string", "allowed": ["1.0"] @@ -20,56 +26,63 @@ class ComponentManifest(Manifest): } } - def __init__(self, data): + def __init__(self, data: Any) -> None: super().__init__(data) - self.components = self.Components(data.get("components", [])) + self.components = Components(data.get("components", [])) # type: ignore[assignment] - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "schema-version": "1.0", "components": self.components.__to_dict__() } - class Components(dict): - def __init__(self, data): - super().__init__(map(lambda component: (component["name"], self.__create__(component)), data)) - @classmethod - def __create__(data): - ComponentManifest.Component(data) +class Components(Dict[str, ComponentType], Generic[ComponentType]): + def __init__(self, data: Dict[Any, Any]) -> None: + create_component: Callable[[Any], Tuple[str, ComponentType]] = lambda component: (component["name"], self.__create__(component)) + super().__init__(map(create_component, data)) + + @classmethod + def __create__(self, data: Any) -> ComponentType: + return Component(data) # type: ignore[return-value] + + def __to_dict__(self) -> List[Dict[Any, Any]]: + as_dict: Callable[[ComponentType], dict] = lambda component: component.__to_dict__() + return list(map(as_dict, self.values())) - def __to_dict__(self): - return list(map(lambda component: component.__to_dict__(), self.values())) + def select(self, focus: str = None) -> Iterator[ComponentType]: + """ + Select components. - def select(self, focus=None): - """ - Select components. + :param str focus: Choose one component. + :return: Collection of components. + :raises ValueError: Invalid platform or component name specified. + """ + is_focused: Callable[[ComponentType], bool] = lambda component: component.__matches__(focus) + selected, it = itertools.tee(filter(is_focused, self.values())) - :param str focus: Choose one component. - :return: Collection of components. - :raises ValueError: Invalid platform or component name specified. - """ - selected, it = itertools.tee(filter(lambda component: component.__matches__(focus), self.values())) + if not any(it): + raise ValueError(f"No components matched focus={focus}.") - if not any(it): - raise ValueError(f"No components matched focus={focus}.") + return selected - return selected - class Component: - def __init__(self, data): - self.name = data["name"] +class Component: + platforms: List[str] - def __to_dict__(self): - return { - "name": self.name - } + def __init__(self, data: dict) -> None: + self.name = data["name"] + + def __to_dict__(self) -> dict: + return { + "name": self.name + } - def __matches__(self, focus=None, platform=None): - matches = ((not focus) or (self.name == focus)) and ((not platform) or (not self.platforms) or (platform in self.platforms)) + def __matches__(self, focus: str = None, platform: str = None) -> bool: + matches = ((not focus) or (self.name == focus)) and ((not platform) or (not self.platforms) or (platform in self.platforms)) - if not matches: - logging.info(f"Skipping {self.name}") + if not matches: + logging.info(f"Skipping {self.name}") - return matches + return matches diff --git a/src/manifests/input_manifest.py b/src/manifests/input_manifest.py index 8ee85dc364..a044017bc8 100644 --- a/src/manifests/input_manifest.py +++ b/src/manifests/input_manifest.py @@ -39,12 +39,13 @@ import copy import itertools import logging +from typing import Any, Callable, Iterator, Optional from git.git_repository import GitRepository -from manifests.component_manifest import ComponentManifest +from manifests.component_manifest import Component, ComponentManifest, Components -class InputManifest(ComponentManifest): +class InputManifest(ComponentManifest['InputManifest', 'InputComponents']): SCHEMA = { "schema-version": {"required": True, "type": "string", "allowed": ["1.0"]}, "build": { @@ -99,14 +100,15 @@ class InputManifest(ComponentManifest): }, } - def __init__(self, data): + def __init__(self, data: Any): super().__init__(data) self.build = self.Build(data["build"]) self.ci = self.Ci(data.get("ci", None)) - self.components = self.Components(data.get("components", [])) - def __to_dict__(self): + self.components = InputComponents(data.get("components", [])) # type: ignore[assignment] + + def __to_dict__(self) -> dict: return { "schema-version": "1.0", "build": self.build.__to_dict__(), @@ -114,8 +116,8 @@ def __to_dict__(self): "components": self.components.__to_dict__(), } - def stable(self, architecture=None, platform=None, snapshot=False): - manifest = copy.deepcopy(self) + def stable(self, architecture: str = None, platform: str = None, snapshot: bool = False) -> 'InputManifest': + manifest: 'InputManifest' = copy.deepcopy(self) manifest.build.__stabilize__( platform=platform, architecture=architecture, @@ -125,25 +127,25 @@ def stable(self, architecture=None, platform=None, snapshot=False): return manifest class Ci: - def __init__(self, data): + def __init__(self, data: Any): self.image = None if data is None else self.Image(data.get("image", None)) - def __to_dict__(self): + def __to_dict__(self) -> Optional[dict]: return None if self.image is None else {"image": self.image.__to_dict__()} class Image: - def __init__(self, data): + def __init__(self, data: Any): self.name = data["name"] self.args = data.get("args", None) - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "name": self.name, "args": self.args } class Build: - def __init__(self, data): + def __init__(self, data: Any): self.name = data["name"] self.version = data["version"] self.platform = data.get("platform", None) @@ -151,12 +153,12 @@ def __init__(self, data): self.snapshot = data.get("snapshot", None) self.patches = data.get("patches", []) - def __stabilize__(self, platform, architecture, snapshot): + def __stabilize__(self, platform: str, architecture: str, snapshot: bool) -> None: self.architecture = architecture self.platform = platform self.snapshot = snapshot - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "name": self.name, "version": self.version, @@ -166,108 +168,113 @@ def __to_dict__(self): "snapshot": self.snapshot, } - class Components(ComponentManifest.Components): - @classmethod - def __create__(self, data): - return InputManifest.Component._from(data) - - def __stabilize__(self): - for component in self.values(): - component.__stabilize__() - - def select(self, focus=None, platform=None): - """ - Select components. - - :param str focus: Choose one component. - :param str platform: Only components targeting a given platform. - :return: Collection of components. - :raises ValueError: Invalid platform or component name specified. - """ - selected, it = itertools.tee(filter(lambda component: component.__matches__(focus, platform), self.values())) - - if not any(it): - raise ValueError(f"No components matched focus={focus}, platform={platform}.") - - return selected - - class Component(ComponentManifest.Component): - def __init__(self, data): - super().__init__(data) - self.platforms = data.get("platforms", None) - - @classmethod - def _from(self, data): - if "repository" in data: - return InputManifest.ComponentFromSource(data) - elif "dist" in data: - return InputManifest.ComponentFromDist(data) - else: - raise ValueError(f"Invalid component data: {data}") - - def __matches__(self, focus=None, platform=None): - matches = ((not focus) or (self.name == focus)) and ((not platform) or (not self.platforms) or (platform in self.platforms)) - - if not matches: - logging.info(f"Skipping {self.name}") - - return matches - - def __stabilize__(self): - pass - - class ComponentFromSource(Component): - def __init__(self, data): - super().__init__(data) - self.repository = data["repository"] - self.ref = data["ref"] - self.working_directory = data.get("working_directory", None) - self.checks = list(map(lambda entry: InputManifest.Check(entry), data.get("checks", []))) - - def __stabilize__(self): - ref, name = GitRepository.stable_ref(self.repository, self.ref) - logging.info(f"Updating ref for {self.repository} from {self.ref} to {ref} ({name})") - self.ref = ref - - def __to_dict__(self): - return { - "name": self.name, - "repository": self.repository, - "ref": self.ref, - "working_directory": self.working_directory, - "checks": list(map(lambda check: check.__to_dict__(), self.checks)), - "platforms": self.platforms, - } - class ComponentFromDist(Component): - def __init__(self, data): - super().__init__(data) - self.dist = data["dist"] - self.checks = list(map(lambda entry: InputManifest.Check(entry), data.get("checks", []))) +class InputComponents(Components['InputComponent']): + @classmethod + def __create__(self, data: Any) -> 'InputComponent': + return InputComponent._from(data) # type: ignore[no-any-return] + + def __stabilize__(self) -> None: + for component in self.values(): + component.__stabilize__() + + def select(self, focus: str = None, platform: str = None) -> Iterator['InputComponent']: + """ + Select components. + + :param str focus: Choose one component. + :param str platform: Only components targeting a given platform. + :return: Collection of components. + :raises ValueError: Invalid platform or component name specified. + """ + by_focus_and_platform: Callable[['InputComponent'], bool] = lambda component: component.__matches__(focus, platform) + selected, it = itertools.tee(filter(by_focus_and_platform, self.values())) + + if not any(it): + raise ValueError(f"No components matched focus={focus}, platform={platform}.") + + return selected + + +class InputComponent(Component): + def __init__(self, data: Any): + super().__init__(data) + self.platforms = data.get("platforms", None) + self.checks = list(map(lambda entry: Check(entry), data.get("checks", []))) + + @classmethod + def _from(self, data: Any) -> 'InputComponent': + if "repository" in data: + return InputComponentFromSource(data) + elif "dist" in data: + return InputComponentFromDist(data) + else: + raise ValueError(f"Invalid component data: {data}") + + def __matches__(self, focus: str = None, platform: str = None) -> bool: + matches = ((not focus) or (self.name == focus)) and ((not platform) or (not self.platforms) or (platform in self.platforms)) + + if not matches: + logging.info(f"Skipping {self.name}") + + return matches + + def __stabilize__(self) -> None: + pass + + +class InputComponentFromSource(InputComponent): + def __init__(self, data: Any) -> None: + super().__init__(data) + self.repository = data["repository"] + self.ref = data["ref"] + self.working_directory = data.get("working_directory", None) + + def __stabilize__(self) -> None: + ref, name = GitRepository.stable_ref(self.repository, self.ref) + logging.info(f"Updating ref for {self.repository} from {self.ref} to {ref} ({name})") + self.ref = ref + + def __to_dict__(self) -> dict: + return { + "name": self.name, + "repository": self.repository, + "ref": self.ref, + "working_directory": self.working_directory, + "checks": list(map(lambda check: check.__to_dict__(), self.checks)), + "platforms": self.platforms, + } + + +class InputComponentFromDist(InputComponent): + def __init__(self, data: Any): + super().__init__(data) + self.dist = data["dist"] + + def __to_dict__(self) -> dict: + return { + "name": self.name, + "dist": self.dist, + "platforms": self.platforms, + "checks": list(map(lambda check: check.__to_dict__(), self.checks)) + } - def __to_dict__(self): - return { - "name": self.name, - "dist": self.dist, - "platforms": self.platforms, - "checks": list(map(lambda check: check.__to_dict__(), self.checks)) - } - class Check: - def __init__(self, data): - if isinstance(data, dict): - if len(data) != 1: - raise ValueError(f"Invalid check format: {data}") - self.name, self.args = next(iter(data.items())) - else: - self.name = data - self.args = None - - def __to_dict__(self): - if self.args: - return {self.name: self.args} - else: - return self.name +class Check: + def __init__(self, data: Any) -> None: + if isinstance(data, dict): + if len(data) != 1: + raise ValueError(f"Invalid check format: {data}") + self.name, self.args = next(iter(data.items())) + else: + self.name = data + self.args = None + + def __to_dict__(self) -> dict: + if self.args: + return {self.name: self.args} + else: + return self.name # type: ignore[no-any-return] InputManifest.VERSIONS = {"1.0": InputManifest} diff --git a/src/manifests/input_manifests.py b/src/manifests/input_manifests.py index 03172d55e3..23654fb301 100644 --- a/src/manifests/input_manifests.py +++ b/src/manifests/input_manifests.py @@ -13,7 +13,7 @@ class InputManifests(Manifests): - def __init__(self): + def __init__(self) -> None: files = glob.glob(os.path.join(self.manifests_path, "**/opensearch-*.yml")) # there's an opensearch-1.0.0-maven.yml that we want to skip files = [f for f in files if re.search(r"[\\/]opensearch-([0-9.]*)\.yml$", f)] diff --git a/src/manifests/manifest.py b/src/manifests/manifest.py index a168b09ff7..3bf61938f2 100644 --- a/src/manifests/manifest.py +++ b/src/manifests/manifest.py @@ -8,31 +8,33 @@ import os import urllib.request from abc import ABC, abstractmethod -from typing import Dict, Optional +from typing import IO, Any, Dict, Generic, Optional, Type, TypeVar -import validators # type:ignore +import validators import yaml -from cerberus import Validator # type:ignore +from cerberus import Validator +T = TypeVar('T', bound='Manifest') -class Manifest(ABC): + +class Manifest(ABC, Generic[T]): SCHEMA = { "schema-version": { "required": True, "type": "string", "empty": False } } - VERSIONS: Optional[Dict] = None + VERSIONS: Optional[Dict[str, object]] = None @classmethod - def from_file(cls, file): + def from_file(cls, file: IO[Any]) -> T: yml = yaml.safe_load(file) version = yml["schema-version"] loader = cls.from_version(version) return loader(yml) @classmethod - def from_url(cls, url): + def from_url(cls, url: str) -> T: logging.info(f"Loading {url}") with urllib.request.urlopen(url) as f: yml = yaml.safe_load(f.read().decode("utf-8")) @@ -41,26 +43,25 @@ def from_url(cls, url): return loader(yml) @classmethod - def from_version(cls, version): + def from_version(cls, version: str) -> Type[T]: if cls.VERSIONS is None: - return cls - + return cls # type: ignore[return-value] if version in [None, ""]: raise ValueError(f"Missing manifest version, must be one of {', '.join(cls.VERSIONS.keys())}") try: - return cls.VERSIONS[version] + return cls.VERSIONS[version] # type: ignore[return-value] except KeyError: raise ValueError(f"Invalid manifest version: {version}, must be one of {', '.join(cls.VERSIONS.keys())}") @classmethod - def from_path(cls, path): + def from_path(cls, path: str) -> T: logging.info(f"Loading {path}") with open(path, "r") as f: return cls.from_file(f) @classmethod - def from_urlpath(cls, file_or_url): + def from_urlpath(cls, file_or_url: str) -> T: if validators.url(file_or_url): return cls.from_url(file_or_url) elif os.path.exists(file_or_url): @@ -69,9 +70,9 @@ def from_urlpath(cls, file_or_url): raise ValueError(f"Invalid manifest: {file_or_url}") @classmethod - def compact(cls, d): + def compact(cls, d: Any) -> Any: if isinstance(d, list): - return list(map(lambda i: cls.compact(i), d)) + return list(map(lambda i: cls.compact(i), d)) # type: ignore[return-value, no-any-return] elif isinstance(d, dict): result = {} for k, v in d.items(): @@ -82,31 +83,31 @@ def compact(cls, d): else: return d - def __to_dict(self): + def __to_dict(self) -> dict: return {} - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, type(self)): - return self.to_dict() == other.to_dict() + return self.to_dict() == other.to_dict() # type: ignore[no-any-return] return False - def to_dict(self): - return Manifest.compact(self.__to_dict__()) + def to_dict(self) -> Any: + return Manifest.compact(self.__to_dict__()) # type: ignore[attr-defined] - def to_file(self, path): + def to_file(self, path: str) -> None: with open(path, "w") as file: yaml.safe_dump(self.to_dict(), file) @abstractmethod - def __init__(self, data): + def __init__(self, data: Any) -> None: self.validate(data) self.version = str(data["schema-version"]) @property - def schema(self): + def schema(self) -> Any: return self.SCHEMA - def validate(self, data): + def validate(self, data: Any) -> None: v = Validator(self.schema) if not v.validate(data): raise ValueError(f"Invalid manifest schema: {v.errors}") diff --git a/src/manifests/manifests.py b/src/manifests/manifests.py index 74ea0ff642..3a811b5910 100644 --- a/src/manifests/manifests.py +++ b/src/manifests/manifests.py @@ -6,17 +6,22 @@ import os import re +from typing import Any, Generic, List, TypeVar -from sortedcontainers import SortedDict # type: ignore +from sortedcontainers import SortedDict +from manifests.manifest import Manifest -class Manifests(SortedDict): - def __init__(self, klass, files): +T = TypeVar('T', bound='Manifest') + + +class Manifests(SortedDict, Generic[T]): + def __init__(self, klass: Any, files: List[str]): super(Manifests, self).__init__() self.klass = klass self.__append__(files) - def __append__(self, files): + def __append__(self, files: List[str]) -> None: for filename in files: basename = os.path.basename(filename) match = re.search(r"-([0-9.]*).yml$", basename) @@ -28,16 +33,16 @@ def __append__(self, files): self.__setitem__(version, manifest) @property - def manifests_path(self): + def manifests_path(self) -> str: return os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "manifests")) @property - def versions(self): - return list(map(lambda manifest: manifest.build.version, self.values())) + def versions(self) -> List[T]: + return list(map(lambda manifest: manifest.build.version, self.values())) # type: ignore[no-any-return] @property - def latest(self): + def latest(self) -> T: if len(self) == 0: raise RuntimeError("No manifests found") - return self.values()[-1] + return self.values()[-1] # type: ignore[no-any-return] diff --git a/src/manifests/test_manifest.py b/src/manifests/test_manifest.py index 51a46e3377..8484acc078 100644 --- a/src/manifests/test_manifest.py +++ b/src/manifests/test_manifest.py @@ -4,10 +4,12 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -from manifests.component_manifest import ComponentManifest +from typing import Any +from manifests.component_manifest import Component, ComponentManifest, Components -class TestManifest(ComponentManifest): + +class TestManifest(ComponentManifest['TestManifest', 'TestComponents']): """ TestManifest contains the test support matrix for any component. @@ -60,38 +62,42 @@ class TestManifest(ComponentManifest): }, } - def __init__(self, data): + def __init__(self, data: Any) -> None: super().__init__(data) self.name = str(data["name"]) + self.components = TestComponents(data.get("components", [])) # type: ignore[assignment] - def __to_dict__(self): + def __to_dict__(self) -> dict: return { "schema-version": "1.0", "name": self.name, "components": self.components.__to_dict__() } - class Components(ComponentManifest.Components): - @classmethod - def __create__(self, data): - return TestManifest.Component(data) - class Component(ComponentManifest.Component): - def __init__(self, data): - super().__init__(data) - self.working_directory = data.get("working-directory", None) - self.integ_test = data.get("integ-test", None) - self.bwc_test = data.get("bwc-test", None) +class TestComponents(Components['TestComponent']): + @classmethod + def __create__(self, data: Any) -> 'TestComponent': + return TestComponent(data) + + +class TestComponent(Component): + def __init__(self, data: Any) -> None: + super().__init__(data) + self.working_directory = data.get("working-directory", None) + self.integ_test = data.get("integ-test", None) + self.bwc_test = data.get("bwc-test", None) + self.components = TestComponents(data.get("components", [])) # type: ignore[assignment] - def __to_dict__(self): - return { - "name": self.name, - "working-directory": self.working_directory, - "integ-test": self.integ_test, - "bwc-test": self.bwc_test - } + def __to_dict__(self) -> dict: + return { + "name": self.name, + "working-directory": self.working_directory, + "integ-test": self.integ_test, + "bwc-test": self.bwc_test + } TestManifest.VERSIONS = {"1.0": TestManifest} -TestManifest.__test__ = False # type:ignore +TestManifest.__test__ = False # type: ignore[attr-defined] diff --git a/src/system/process.py b/src/system/process.py index fede482115..c4e85f760d 100644 --- a/src/system/process.py +++ b/src/system/process.py @@ -8,7 +8,7 @@ import subprocess import tempfile -import psutil # type: ignore +import psutil class Process: diff --git a/src/system/properties_file.py b/src/system/properties_file.py index 4143abee03..6fd1648340 100644 --- a/src/system/properties_file.py +++ b/src/system/properties_file.py @@ -4,7 +4,7 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -from jproperties import Properties # type:ignore +from jproperties import Properties class PropertiesFile(Properties): diff --git a/src/system/temporary_directory.py b/src/system/temporary_directory.py index 5a259b2963..5931dc7c95 100644 --- a/src/system/temporary_directory.py +++ b/src/system/temporary_directory.py @@ -10,9 +10,11 @@ import shutil import stat import tempfile +from types import FunctionType +from typing import Any -def g__handleRemoveReadonly(func, path, exc): +def g__handleRemoveReadonly(func: FunctionType, path: str, exc: Any) -> Any: excvalue = exc[1] if func in (os.rmdir, os.remove, os.unlink) and excvalue.errno == errno.EACCES: os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 @@ -22,7 +24,7 @@ def g__handleRemoveReadonly(func, path, exc): class TemporaryDirectory: - def __init__(self, keep=False, chdir=False): + def __init__(self, keep: bool = False, chdir: bool = False): self.keep = keep self.name = tempfile.mkdtemp() if chdir: @@ -31,10 +33,10 @@ def __init__(self, keep=False, chdir=False): else: self.curdir = None - def __enter__(self): + def __enter__(self) -> 'TemporaryDirectory': return self - def __exit__(self, exc_type, exc_value, exc_traceback): + def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None: if self.curdir: os.chdir(self.curdir) diff --git a/tests/tests_assemble_workflow/test_bundle_recorder.py b/tests/tests_assemble_workflow/test_bundle_recorder.py index 7746478c70..a62066a3e5 100644 --- a/tests/tests_assemble_workflow/test_bundle_recorder.py +++ b/tests/tests_assemble_workflow/test_bundle_recorder.py @@ -10,7 +10,7 @@ import yaml from assemble_workflow.bundle_recorder import BundleRecorder -from manifests.build_manifest import BuildManifest +from manifests.build_manifest import BuildComponent, BuildManifest from manifests.bundle_manifest import BundleManifest from system.temporary_directory import TemporaryDirectory @@ -23,7 +23,7 @@ def setUp(self): self.bundle_recorder = BundleRecorder(manifest.build, "output_dir", "artifacts_dir", None) def test_record_component(self): - component = BuildManifest.Component( + component = BuildComponent( { "name": "job_scheduler", "repository": "https://github.com/opensearch-project/job_scheduler", @@ -88,7 +88,7 @@ def test_write_manifest(self): def test_record_component_public(self): self.bundle_recorder.base_url = "https://ci.opensearch.org/ci/ci-env-prod/job-name-opensearch/1.2.0/build-123/platform-mac/arch-amd64/" - component = BuildManifest.Component( + component = BuildComponent( { "name": "job_scheduler", "repository": "https://github.com/opensearch-project/job_scheduler", @@ -155,7 +155,7 @@ def setUp(self): self.bundle_recorder = BundleRecorder(manifest.build, "output_dir", "artifacts_dir", None) def test_record_component(self): - component = BuildManifest.Component( + component = BuildComponent( { "name": "alertingDashboards", "repository": "https://github.com/opensearch-project/alerting-dashboards-plugin", @@ -220,7 +220,7 @@ def test_write_manifest(self): def test_record_component_public(self): self.bundle_recorder.base_url = "https://ci.opensearch.org/ci/ci-env-prod/job-name-dashboards/1.2.0/build-123/platform-mac/arch-amd64/" - component = BuildManifest.Component( + component = BuildComponent( { "name": "alertingDashboards", "repository": "https://github.com/opensearch-project/alerting-dashboards-plugin", diff --git a/tests/tests_build_workflow/test_builder_from_dist.py b/tests/tests_build_workflow/test_builder_from_dist.py index dcd1338d91..48fef3636f 100644 --- a/tests/tests_build_workflow/test_builder_from_dist.py +++ b/tests/tests_build_workflow/test_builder_from_dist.py @@ -11,13 +11,13 @@ from build_workflow.build_target import BuildTarget from build_workflow.builder_from_dist import BuilderFromDist from manifests.build_manifest import BuildManifest -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponentFromDist class TestBuilderFromDist(unittest.TestCase): def setUp(self): self.builder = BuilderFromDist( - InputManifest.ComponentFromDist({"name": "common-utils", "dist": "url"}), + InputComponentFromDist({"name": "common-utils", "dist": "url"}), BuildTarget( name="OpenSearch", version="1.1.0", diff --git a/tests/tests_build_workflow/test_builder_from_source.py b/tests/tests_build_workflow/test_builder_from_source.py index 0ed191d341..421502d62e 100644 --- a/tests/tests_build_workflow/test_builder_from_source.py +++ b/tests/tests_build_workflow/test_builder_from_source.py @@ -10,14 +10,14 @@ from build_workflow.build_target import BuildTarget from build_workflow.builder_from_source import BuilderFromSource -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponentFromSource from paths.script_finder import ScriptFinder class TestBuilderFromSource(unittest.TestCase): def setUp(self): self.builder = BuilderFromSource( - InputManifest.ComponentFromSource({"name": "common-utils", "repository": "url", "ref": "ref"}), + InputComponentFromSource({"name": "common-utils", "repository": "url", "ref": "ref"}), BuildTarget( name="OpenSearch", version="1.1.0", diff --git a/tests/tests_ci_workflow/test_ci_check_gradle_properties_version.py b/tests/tests_ci_workflow/test_ci_check_gradle_properties_version.py index 12544ab4c1..dca2c09646 100644 --- a/tests/tests_ci_workflow/test_ci_check_gradle_properties_version.py +++ b/tests/tests_ci_workflow/test_ci_check_gradle_properties_version.py @@ -9,7 +9,7 @@ from ci_workflow.ci_check_gradle_properties_version import CiCheckGradlePropertiesVersion from ci_workflow.ci_target import CiTarget -from manifests.input_manifest import InputManifest +from manifests.input_manifest import Component from system.properties_file import PropertiesFile @@ -45,7 +45,7 @@ def test_invalid_version(self): def test_component_version_opensearch(self): check = self.__mock_check( props={"version": "1.1.0.0-SNAPSHOT"}, - component=InputManifest.Component({"name": "OpenSearch", "repository": "", "ref": ""}), + component=Component({"name": "OpenSearch", "repository": "", "ref": ""}), ) self.assertEqual(check.checked_version, "1.1.0-SNAPSHOT") @@ -61,7 +61,7 @@ def test_component_version_opensearch(self): def test_component_version(self): check = self.__mock_check( props={"version": "1.1.0-SNAPSHOT"}, - component=InputManifest.Component({"name": "Plugin", "repository": "", "ref": ""}), + component=Component({"name": "Plugin", "repository": "", "ref": ""}), ) self.assertEqual(check.checked_version, "1.1.0.0-SNAPSHOT") diff --git a/tests/tests_ci_workflow/test_ci_check_lists.py b/tests/tests_ci_workflow/test_ci_check_lists.py index 4ddc92f471..9b7bc51ce0 100644 --- a/tests/tests_ci_workflow/test_ci_check_lists.py +++ b/tests/tests_ci_workflow/test_ci_check_lists.py @@ -9,12 +9,12 @@ from ci_workflow.ci_check_list_dist import CiCheckListDist from ci_workflow.ci_check_list_source import CiCheckListSource from ci_workflow.ci_check_lists import CiCheckLists -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponentFromDist, InputComponentFromSource class TestCiCheckLists(unittest.TestCase): def test_from_component_source(self): - check_list = CiCheckLists.from_component(InputManifest.ComponentFromSource({ + check_list = CiCheckLists.from_component(InputComponentFromSource({ "name": "common-utils", "repository": "url", "ref": "ref" @@ -22,7 +22,7 @@ def test_from_component_source(self): self.assertIs(type(check_list), CiCheckListSource) def test_from_component_dist(self): - check_list = CiCheckLists.from_component(InputManifest.ComponentFromDist({ + check_list = CiCheckLists.from_component(InputComponentFromDist({ "name": "common-utils", "dist": "url" }), None) diff --git a/tests/tests_ci_workflow/test_ci_check_lists_dist.py b/tests/tests_ci_workflow/test_ci_check_lists_dist.py index 09657b8418..c0a6a9578e 100644 --- a/tests/tests_ci_workflow/test_ci_check_lists_dist.py +++ b/tests/tests_ci_workflow/test_ci_check_lists_dist.py @@ -10,7 +10,7 @@ from ci_workflow.ci_check_list_dist import CiCheckListDist from manifests.build_manifest import BuildManifest -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponentFromDist class TestCiCheckListsDist(unittest.TestCase): @@ -20,7 +20,7 @@ class TestCiCheckListsDist(unittest.TestCase): @patch("manifests.build_manifest.BuildManifest.from_url") def test_check(self, mock_manifest, *mocks): mock_manifest.return_value = BuildManifest.from_path(self.BUILD_MANIFEST) - component = InputManifest.ComponentFromDist({ + component = InputComponentFromDist({ "name": "common-utils", "dist": "url", "checks": ["manifest:component"] @@ -30,7 +30,7 @@ def test_check(self, mock_manifest, *mocks): mock_manifest.assert_called() def test_invalid_check(self, *mocks): - component = InputManifest.ComponentFromDist({ + component = InputComponentFromDist({ "name": "common-utils", "dist": "url", "checks": ["invalid:check"] diff --git a/tests/tests_ci_workflow/test_ci_check_lists_source.py b/tests/tests_ci_workflow/test_ci_check_lists_source.py index 94bf47fbe9..6029440c36 100644 --- a/tests/tests_ci_workflow/test_ci_check_lists_source.py +++ b/tests/tests_ci_workflow/test_ci_check_lists_source.py @@ -8,13 +8,13 @@ from unittest.mock import MagicMock, patch from ci_workflow.ci_check_list_source import CiCheckListSource -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponentFromSource class TestCiCheckListsSource(unittest.TestCase): @patch("ci_workflow.ci_check_list_source.GitRepository") def test_checkout(self, mock_git_repo): - component = InputManifest.ComponentFromSource({ + component = InputComponentFromSource({ "name": "common-utils", "repository": "url", "ref": "ref" @@ -26,7 +26,7 @@ def test_checkout(self, mock_git_repo): @patch("ci_workflow.ci_check_list_source.GitRepository") @patch("ci_workflow.ci_check_gradle_properties.PropertiesFile") def test_check(self, mock_properties_file, mock_check, *mocks): - component = InputManifest.ComponentFromSource({ + component = InputComponentFromSource({ "name": "common-utils", "repository": "url", "ref": "ref", @@ -41,7 +41,7 @@ def test_check(self, mock_properties_file, mock_check, *mocks): @patch("ci_workflow.ci_check_list_source.GitRepository") def test_invalid_check(self, *mocks): - component = InputManifest.ComponentFromSource({ + component = InputComponentFromSource({ "name": "common-utils", "repository": "url", "ref": "ref", diff --git a/tests/tests_ci_workflow/test_ci_check_manifest_component.py b/tests/tests_ci_workflow/test_ci_check_manifest_component.py index b54791a1d8..e1f4daf788 100644 --- a/tests/tests_ci_workflow/test_ci_check_manifest_component.py +++ b/tests/tests_ci_workflow/test_ci_check_manifest_component.py @@ -10,7 +10,7 @@ from ci_workflow.ci_check_manifest_component import CiCheckManifestComponent from manifests.build_manifest import BuildManifest -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponentFromDist class TestCiCheckManifestComponent(unittest.TestCase): @@ -19,7 +19,7 @@ class TestCiCheckManifestComponent(unittest.TestCase): @patch("ci_workflow.ci_check_manifest_component.BuildManifest") def test_retrieves_manifests(self, mock_manifest): - check = CiCheckManifestComponent(InputManifest.ComponentFromDist({ + check = CiCheckManifestComponent(InputComponentFromDist({ "name": "common-utils", "dist": "url" }), MagicMock()) @@ -34,7 +34,7 @@ def test_retrieves_manifests(self, mock_manifest): @patch("ci_workflow.ci_check_manifest_component.BuildManifest") def test_missing_component(self, mock_manifest): - check = CiCheckManifestComponent(InputManifest.ComponentFromDist({ + check = CiCheckManifestComponent(InputComponentFromDist({ "name": "does-not-exist", "dist": "url" }), MagicMock()) diff --git a/tests/tests_manifests/test_build_manifest.py b/tests/tests_manifests/test_build_manifest.py index b9973bfe57..47b97ee05a 100644 --- a/tests/tests_manifests/test_build_manifest.py +++ b/tests/tests_manifests/test_build_manifest.py @@ -9,22 +9,22 @@ import yaml -from manifests.build_manifest import BuildManifest +from manifests.build_manifest import BuildComponent, BuildManifest class TestBuildManifest(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.data_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "data")) self.manifest_filename = os.path.join(self.data_path, "opensearch-build-1.1.0.yml") self.manifest = BuildManifest.from_path(self.manifest_filename) - def test_build(self): + def test_build(self) -> None: self.assertEqual(self.manifest.version, "1.2") self.assertEqual(self.manifest.build.name, "OpenSearch") self.assertEqual(self.manifest.build.version, "1.1.0") self.assertEqual(len(self.manifest.components), 15) - def test_component(self): + def test_component(self) -> None: opensearch_component = self.manifest.components["OpenSearch"] self.assertEqual(opensearch_component.name, "OpenSearch") self.assertEqual( @@ -38,44 +38,44 @@ def test_component(self): ["core-plugins", "dist", "maven"], ) - def test_to_dict(self): + def test_to_dict(self) -> None: data = self.manifest.to_dict() with open(self.manifest_filename) as f: self.assertEqual(yaml.safe_load(f), data) - def test_components_index(self): + def test_components_index(self) -> None: component = self.manifest.components["index-management"] self.assertEqual(component.name, "index-management") - def test_components_index_error(self): + def test_components_index_error(self) -> None: with self.assertRaises(KeyError) as ctx: self.manifest.components["invalid-component"] self.assertEqual(str(ctx.exception), "'invalid-component'") - def test_versions(self): + def test_versions(self) -> None: self.assertTrue(len(BuildManifest.VERSIONS)) for version in BuildManifest.VERSIONS: manifest = BuildManifest.from_path(os.path.join(self.data_path, "build", f"opensearch-build-schema-version-{version}.yml")) self.assertEqual(version, manifest.version) self.assertIsNotNone(any(manifest.components)) - def test_select(self): + def test_select(self) -> None: path = os.path.join(self.data_path, "build", "opensearch-build-schema-version-1.2.yml") manifest = BuildManifest.from_path(path) self.assertEqual(len(list(manifest.components.select(focus="common-utils"))), 1) - def test_select_none(self): + def test_select_none(self) -> None: path = os.path.join(self.data_path, "build", "opensearch-build-schema-version-1.2.yml") manifest = BuildManifest.from_path(path) with self.assertRaises(ValueError) as ctx: self.assertEqual(len(list(manifest.components.select(focus="x"))), 0) self.assertEqual(str(ctx.exception), "No components matched focus=x.") - def test_component_matches(self): - self.assertTrue(BuildManifest.Component({"name": "x", "repository": "", "ref": "", "commit_id": "", "version": ""}).__matches__()) + def test_component_matches(self) -> None: + self.assertTrue(BuildComponent({"name": "x", "repository": "", "ref": "", "commit_id": "", "version": ""}).__matches__()) - def test_component_matches_focus(self): - component = BuildManifest.Component({"name": "x", "repository": "", "ref": "", "commit_id": "", "version": ""}) + def test_component_matches_focus(self) -> None: + component = BuildComponent({"name": "x", "repository": "", "ref": "", "commit_id": "", "version": ""}) self.assertTrue(component.__matches__(focus=None)) self.assertTrue(component.__matches__(focus="x")) self.assertFalse(component.__matches__(focus="y")) diff --git a/tests/tests_manifests/test_bundle_manifest.py b/tests/tests_manifests/test_bundle_manifest.py index 6fa7b008f1..f962d86e69 100644 --- a/tests/tests_manifests/test_bundle_manifest.py +++ b/tests/tests_manifests/test_bundle_manifest.py @@ -13,12 +13,12 @@ class TestBundleManifest(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.data_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "data")) self.manifest_filename = os.path.join(self.data_path, "opensearch-bundle-1.1.0.yml") self.manifest = BundleManifest.from_path(self.manifest_filename) - def test_build(self): + def test_build(self) -> None: self.assertEqual(self.manifest.version, "1.1") self.assertEqual(self.manifest.build.name, "OpenSearch") self.assertEqual(self.manifest.build.version, "1.1.0") @@ -27,7 +27,7 @@ def test_build(self): self.assertEqual(self.manifest.build.architecture, "x64") self.assertEqual(len(self.manifest.components), 13) - def test_component(self): + def test_component(self) -> None: opensearch_min_component = self.manifest.components["OpenSearch"] self.assertEqual(opensearch_min_component.name, "OpenSearch") self.assertEqual(opensearch_min_component.location, "artifacts/dist/opensearch-min-1.1.0-linux-x64.tar.gz") @@ -35,12 +35,12 @@ def test_component(self): self.assertEqual(opensearch_min_component.commit_id, "b7334f49d530ffd1a3f7bd0e5832b9b2a9caa583") self.assertEqual(opensearch_min_component.ref, "1.1") - def test_to_dict(self): + def test_to_dict(self) -> None: data = self.manifest.to_dict() with open(self.manifest_filename) as f: self.assertEqual(yaml.safe_load(f), data) - def test_versions(self): + def test_versions(self) -> None: self.assertTrue(len(BundleManifest.VERSIONS)) for version in BundleManifest.VERSIONS: manifest = BundleManifest.from_path(os.path.join(self.data_path, "bundle", f"opensearch-bundle-schema-version-{version}.yml")) diff --git a/tests/tests_manifests/test_component_manifest.py b/tests/tests_manifests/test_component_manifest.py index 6b27396b78..33fdd93f87 100644 --- a/tests/tests_manifests/test_component_manifest.py +++ b/tests/tests_manifests/test_component_manifest.py @@ -5,22 +5,24 @@ # compatible open source license. import unittest -from abc import abstractmethod +from typing import Any -from manifests.component_manifest import ComponentManifest +from manifests.component_manifest import Component, ComponentManifest, Components class TestComponentManifest(unittest.TestCase): - class UnitTestManifest(ComponentManifest): - class Components(ComponentManifest.Components): - @abstractmethod - def __create__(self, data): - return TestComponentManifest.UnitTestManifest.Component(data) + class UnitTestManifest(ComponentManifest['TestComponentManifest.UnitTestManifest', 'TestComponentManifest.TestComponents']): + pass - class Component(ComponentManifest.Component): - pass + class TestComponents(Components['TestComponentManifest.TestComponent']): + @classmethod + def __create__(self, data: Any) -> 'TestComponentManifest.TestComponent': + return TestComponentManifest.TestComponent(data) - def test_select(self): + class TestComponent(Component): + pass + + def test_select(self) -> None: manifest = TestComponentManifest.UnitTestManifest({ "schema-version": "1.0", "components": [ diff --git a/tests/tests_manifests/test_input_manifest.py b/tests/tests_manifests/test_input_manifest.py index dddc25b500..ffd263ffcc 100644 --- a/tests/tests_manifests/test_input_manifest.py +++ b/tests/tests_manifests/test_input_manifest.py @@ -7,26 +7,27 @@ import copy import os import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch import yaml -from manifests.input_manifest import InputManifest +from manifests.input_manifest import Check, InputComponent, InputComponentFromDist, InputComponentFromSource, InputManifest class TestInputManifest(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.maxDiff = None self.manifests_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "manifests")) - def test_1_0(self): + def test_1_0(self) -> None: path = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(manifest.version, "1.0") self.assertEqual(manifest.build.name, "OpenSearch") self.assertEqual(manifest.build.version, "1.0.0") self.assertEqual(len(list(manifest.components.select(focus="common-utils"))), 1) - opensearch_component = manifest.components["OpenSearch"] + opensearch_component: InputComponentFromSource = manifest.components["OpenSearch"] # type: ignore[assignment] + self.assertIsInstance(opensearch_component, InputComponentFromSource) self.assertEqual(opensearch_component.name, "OpenSearch") self.assertEqual( opensearch_component.repository, @@ -34,9 +35,9 @@ def test_1_0(self): ) self.assertEqual(opensearch_component.ref, "1.0") for component in manifest.components.values(): - self.assertIsInstance(component.ref, str) + self.assertIsInstance(component, InputComponentFromSource) - def test_1_1(self): + def test_1_1(self) -> None: path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(manifest.version, "1.0") @@ -44,7 +45,7 @@ def test_1_1(self): self.assertEqual(manifest.build.version, "1.1.0") self.assertEqual(len(list(manifest.components.select(focus="common-utils"))), 1) # opensearch component - opensearch_component = manifest.components["OpenSearch"] + opensearch_component: InputComponentFromSource = manifest.components["OpenSearch"] # type: ignore[assignment] self.assertEqual(opensearch_component.name, "OpenSearch") self.assertEqual( opensearch_component.repository, @@ -53,17 +54,17 @@ def test_1_1(self): self.assertEqual(opensearch_component.ref, "tags/1.1.0") # components for component in manifest.components.values(): - self.assertIsInstance(component.ref, str) + self.assertIsInstance(component, InputComponentFromSource) # alerting component checks - alerting_component = manifest.components["alerting"] + alerting_component: InputComponentFromSource = manifest.components["alerting"] # type: ignore[assignment] self.assertIsNotNone(alerting_component) self.assertEqual(len(alerting_component.checks), 2) for check in alerting_component.checks: - self.assertIsInstance(check, InputManifest.Check) + self.assertIsInstance(check, Check) self.assertIsNone(alerting_component.checks[0].args) self.assertEqual(alerting_component.checks[1].args, "alerting") - def test_1_2(self): + def test_1_2(self) -> None: path = os.path.join(self.manifests_path, "1.2.0/opensearch-1.2.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(manifest.version, "1.0") @@ -74,7 +75,7 @@ def test_1_2(self): self.assertNotEqual(len(manifest.components), 0) self.assertEqual(len(list(manifest.components.select(focus="common-utils"))), 1) # opensearch component - opensearch_component = manifest.components["OpenSearch"] + opensearch_component: InputComponentFromSource = manifest.components["OpenSearch"] # type: ignore[assignment] self.assertEqual(opensearch_component.name, "OpenSearch") self.assertEqual( opensearch_component.repository, @@ -83,24 +84,24 @@ def test_1_2(self): self.assertEqual(opensearch_component.ref, "tags/1.2.0") # components for component in manifest.components.values(): - self.assertIsInstance(component.ref, str) + self.assertIsInstance(component, InputComponentFromSource) # alerting component checks alerting_component = manifest.components["alerting"] self.assertIsNotNone(alerting_component) self.assertEqual(len(alerting_component.checks), 2) for check in alerting_component.checks: - self.assertIsInstance(check, InputManifest.Check) + self.assertIsInstance(check, Check) self.assertIsNone(alerting_component.checks[0].args) self.assertEqual(alerting_component.checks[1].args, "alerting") - def test_to_dict(self): + def test_to_dict(self) -> None: path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path) data = manifest.to_dict() with open(path) as f: self.assertEqual(yaml.safe_load(f), data) - def test_invalid_ref(self): + def test_invalid_ref(self) -> None: data_path = os.path.join(os.path.dirname(__file__), "data") manifest_path = os.path.join(data_path, "invalid-ref.yml") @@ -108,47 +109,47 @@ def test_invalid_ref(self): InputManifest.from_path(manifest_path) self.assertTrue(str(context.exception).startswith("Invalid manifest schema: {'components': ")) - def test_select(self): + def test_select(self) -> None: path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(len(list(manifest.components.select(focus="common-utils"))), 1) self.assertNotEqual(len(list(manifest.components.select(platform="windows"))), 0) self.assertEqual(len(list(manifest.components.select(focus="k-NN", platform="linux"))), 1) - def test_select_none(self): + def test_select_none(self) -> None: path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path) with self.assertRaises(ValueError) as ctx: self.assertEqual(len(list(manifest.components.select(focus="k-NN", platform="windows"))), 0) self.assertEqual(str(ctx.exception), "No components matched focus=k-NN, platform=windows.") - def test_component___matches__(self): - self.assertTrue(InputManifest.Component({"name": "x", "repository": "", "ref": ""}).__matches__()) + def test_component___matches__(self) -> None: + self.assertTrue(InputComponent({"name": "x", "repository": "", "ref": ""}).__matches__()) - def test_component___matches_platform__(self): + def test_component___matches_platform__(self) -> None: data = {"name": "x", "repository": "", "ref": ""} - self.assertTrue(InputManifest.Component(data).__matches__(platform=None)) - self.assertTrue(InputManifest.Component(data).__matches__(platform="x")) - self.assertTrue(InputManifest.Component({**data, "platforms": ["linux"]}).__matches__(platform="linux")) - self.assertTrue(InputManifest.Component({**data, "platforms": ["linux", "windows"]}).__matches__(platform="linux")) - self.assertFalse(InputManifest.Component({**data, "platforms": ["linux"]}).__matches__(platform="x")) - - def test_component___matches_focus__(self): - component = InputManifest.Component({"name": "x", "repository": "", "ref": ""}) + self.assertTrue(InputComponent(data).__matches__(platform=None)) + self.assertTrue(InputComponent(data).__matches__(platform="x")) + self.assertTrue(InputComponent({**data, "platforms": ["linux"]}).__matches__(platform="linux")) + self.assertTrue(InputComponent({**data, "platforms": ["linux", "windows"]}).__matches__(platform="linux")) + self.assertFalse(InputComponent({**data, "platforms": ["linux"]}).__matches__(platform="x")) + + def test_component___matches_focus__(self) -> None: + component = InputComponent({"name": "x", "repository": "", "ref": ""}) self.assertTrue(component.__matches__(focus=None)) self.assertTrue(component.__matches__(focus="x")) self.assertFalse(component.__matches__(focus="y")) @patch("subprocess.check_output") - def test_stable(self, mock_output): + def test_stable(self, mock_output: Mock) -> None: mock_output.return_value.decode.return_value = "updated\tHEAD" path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path).stable() - opensearch = manifest.components["OpenSearch"] + opensearch: InputComponentFromSource = manifest.components["OpenSearch"] # type: ignore[assignment] self.assertEqual(opensearch.ref, "updated") @patch("subprocess.check_output") - def test_stable_override_build(self, mock_output): + def test_stable_override_build(self, mock_output: Mock) -> None: mock_output.return_value.decode.return_value = "updated\tHEAD" path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path).stable(platform="windows", architecture="arm64", snapshot=True) @@ -156,26 +157,26 @@ def test_stable_override_build(self, mock_output): self.assertEqual(manifest.build.architecture, "arm64") self.assertTrue(manifest.build.snapshot) - def test_eq(self): + def test_eq(self) -> None: path = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") manifest1 = InputManifest.from_path(path) manifest2 = InputManifest.from_path(path) self.assertEqual(manifest1, manifest1) self.assertEqual(manifest1, manifest2) - def test_neq(self): + def test_neq(self) -> None: path1 = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") path2 = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest1 = InputManifest.from_path(path1) manifest2 = InputManifest.from_path(path2) self.assertNotEqual(manifest1, manifest2) - def test_neq_update(self): + def test_neq_update(self) -> None: path = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") manifest1 = InputManifest.from_path(path) manifest2 = copy.deepcopy(manifest1) self.assertEqual(manifest1, manifest2) - manifest2.components["name"] = InputManifest.ComponentFromDist({ + manifest2.components["name"] = InputComponentFromDist({ "name": "name", "dist": "dist" }) diff --git a/tests/tests_manifests/test_input_manifests.py b/tests/tests_manifests/test_input_manifests.py index eb1e79b14f..7d30d20530 100644 --- a/tests/tests_manifests/test_input_manifests.py +++ b/tests/tests_manifests/test_input_manifests.py @@ -10,19 +10,19 @@ class TestInputManifests(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.manifests = InputManifests() - def tests_configs(self): + def tests_configs(self) -> None: self.assertTrue(len(self.manifests)) - def test_1_1_0(self): + def test_1_1_0(self) -> None: manifest = self.manifests["1.1.0"] self.assertIsNotNone(manifest) self.assertEqual(manifest.version, "1.0") self.assertEqual(manifest.build.version, "1.1.0") - def test_latest(self): + def test_latest(self) -> None: manifest = self.manifests.latest self.assertIsNotNone(manifest) self.assertEqual(manifest.build.version, max(self.manifests.keys())) diff --git a/tests/tests_manifests/test_manifest.py b/tests/tests_manifests/test_manifest.py index 525aeb0c1f..ff7194d7f7 100644 --- a/tests/tests_manifests/test_manifest.py +++ b/tests/tests_manifests/test_manifest.py @@ -6,7 +6,8 @@ import os import unittest -from unittest.mock import MagicMock, patch +from typing import Any, Dict +from unittest.mock import MagicMock, Mock, patch import yaml @@ -15,20 +16,20 @@ class TestManifest(unittest.TestCase): - class SampleManifest(Manifest): - def __init__(self, data): + class SampleManifest(Manifest['TestManifest.SampleManifest']): + def __init__(self, data: Dict[Any, Any]) -> None: super().__init__(data) self.data = data - def __to_dict__(self): + def __to_dict__(self) -> dict: return self.data - class SampleManifestWithVersions(Manifest): - def __init__(self, data): + class SampleManifestWithVersions(Manifest['TestManifest.SampleManifest']): + def __init__(self, data: Dict[Any, Any]) -> None: super().__init__(data) self.data = data - def __to_dict__(self): + def __to_dict__(self) -> dict: return self.data SampleManifestWithVersions.VERSIONS = { @@ -36,18 +37,18 @@ def __to_dict__(self): "6.42": SampleManifestWithVersions } - def setUp(self): + def setUp(self) -> None: self.data_path = os.path.join(os.path.dirname(__file__), "data") - def test_manifest_is_abstract(self): + def test_manifest_is_abstract(self) -> None: with self.assertRaises(TypeError) as context: - Manifest(None) + Manifest(None) # type: ignore[abstract] self.assertEqual( "Can't instantiate abstract class Manifest with abstract methods __init__", context.exception.__str__(), ) - def test_invalid_version_empty(self): + def test_invalid_version_empty(self) -> None: manifest_path = os.path.join(self.data_path, "invalid-schema-version-empty.yml") with self.assertRaises(ValueError) as context: @@ -57,7 +58,7 @@ def test_invalid_version_empty(self): context.exception.__str__(), ) - def test_invalid_version_no_value(self): + def test_invalid_version_no_value(self) -> None: manifest_path = os.path.join(self.data_path, "invalid-schema-version-no-value.yml") with self.assertRaises(ValueError) as context: @@ -67,7 +68,7 @@ def test_invalid_version_no_value(self): context.exception.__str__(), ) - def test_compact(self): + def test_compact(self) -> None: self.assertEqual(Manifest.compact({}), {}) self.assertEqual(Manifest.compact({"x": "y"}), {"x": "y"}) self.assertEqual(Manifest.compact({"x": "y", "z": []}), {"x": "y"}) @@ -76,7 +77,7 @@ def test_compact(self): self.assertEqual(Manifest.compact({"x": True}), {"x": True}) self.assertEqual(Manifest.compact({"x": False}), {"x": False}) - def test_to_file(self): + def test_to_file(self) -> None: manifest_path = os.path.join(self.data_path, "min.yml") manifest = TestManifest.SampleManifest.from_path(manifest_path) @@ -87,7 +88,7 @@ def test_to_file(self): with open(output_path) as f: self.assertEqual(yaml.safe_load(f), manifest.to_dict()) - def test_invalid_version_no_value_3_14(self): + def test_invalid_version_no_value_3_14(self) -> None: manifest_path = os.path.join(self.data_path, "invalid-schema-version-no-value.yml") with self.assertRaises(ValueError) as context: @@ -97,7 +98,7 @@ def test_invalid_version_no_value_3_14(self): context.exception.__str__(), ) - def test_invalid_version_empty_3_14(self): + def test_invalid_version_empty_3_14(self) -> None: manifest_path = os.path.join(self.data_path, "invalid-schema-version-empty.yml") with self.assertRaises(ValueError) as context: @@ -107,7 +108,7 @@ def test_invalid_version_empty_3_14(self): context.exception.__str__(), ) - def test_invalid_version_3_14(self): + def test_invalid_version_3_14(self) -> None: manifest_path = os.path.join(self.data_path, "opensearch-build-1.1.0.yml") with self.assertRaises(ValueError) as context: @@ -118,7 +119,7 @@ def test_invalid_version_3_14(self): ) @patch("manifests.manifest.urllib.request.urlopen") - def test_from_url(self, mock_urlopen): + def test_from_url(self, mock_urlopen: Mock) -> None: cm = MagicMock() cm.read.return_value.decode.return_value = '{"schema-version":"3.14"}' cm.__enter__.return_value = cm @@ -127,7 +128,7 @@ def test_from_url(self, mock_urlopen): self.assertEqual(manifest.version, "3.14") mock_urlopen.assert_called_with("url") - def test_eq(self): + def test_eq(self) -> None: manifest_path = os.path.join(self.data_path, "min.yml") manifest1 = TestManifest.SampleManifest.from_path(manifest_path) manifest2 = TestManifest.SampleManifest.from_path(manifest_path) diff --git a/tests/tests_manifests/test_manifests.py b/tests/tests_manifests/test_manifests.py index 5516ce17af..da7fd253ea 100644 --- a/tests/tests_manifests/test_manifests.py +++ b/tests/tests_manifests/test_manifests.py @@ -7,6 +7,7 @@ import glob import os import unittest +from typing import Any from manifests.build_manifest import BuildManifest from manifests.manifests import Manifests @@ -14,32 +15,32 @@ class TestManifests(unittest.TestCase): class WildcardManifests(Manifests): - def __init__(self, klass, wildcard="*.yml"): + def __init__(self, klass: Any, wildcard: str = "*.yml") -> None: data_path = os.path.join(os.path.dirname(__file__), "data") manifest_filenames = os.path.join(data_path, wildcard) super().__init__(klass, glob.glob(manifest_filenames)) - def test_latest(self): + def test_latest(self) -> None: manifests = TestManifests.WildcardManifests(BuildManifest, "opensearch-build-*.yml") - latest_manifest = manifests.latest + latest_manifest: BuildManifest = manifests.latest self.assertEqual(latest_manifest.build.version, max(manifests.keys())) - def test_latest_no_manifests(self): + def test_latest_no_manifests(self) -> None: manifests = TestManifests.WildcardManifests(BuildManifest, "does-not-exist*.yml") with self.assertRaises(RuntimeError) as context: manifests.latest self.assertEqual("No manifests found", str(context.exception)) - def test_build_manifests(self): + def test_build_manifests(self) -> None: manifests = TestManifests.WildcardManifests(BuildManifest, "opensearch-build-*.yml") self.assertTrue(len(manifests) >= 2) - def test_invalid_filename(self): + def test_invalid_filename(self) -> None: with self.assertRaises(ValueError) as context: TestManifests.WildcardManifests(BuildManifest, "invalid-schema-version.yml") self.assertEqual("Invalid file: invalid-schema-version.yml", str(context.exception)) - def test_versions(self): + def test_versions(self) -> None: manifests = TestManifests.WildcardManifests(BuildManifest, "opensearch-build-*.yml") versions = manifests.versions self.assertTrue("1.1.0" in versions) diff --git a/tests/tests_manifests/test_test_manifest.py b/tests/tests_manifests/test_test_manifest.py index 8ad90720bf..cdc3d3cdcc 100644 --- a/tests/tests_manifests/test_test_manifest.py +++ b/tests/tests_manifests/test_test_manifest.py @@ -13,31 +13,31 @@ class TestTestManifest(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.maxDiff = None self.data_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "data")) self.manifest_filename = os.path.join(self.data_path, "opensearch-test-1.1.0.yml") self.manifest = TestManifest.from_path(self.manifest_filename) - def test_component(self): + def test_component(self) -> None: component = self.manifest.components["index-management"] self.assertEqual(component.name, "index-management") self.assertEqual(component.integ_test, {"test-configs": ["with-security", "without-security"]}) self.assertEqual(component.bwc_test, {"test-configs": ["with-security", "without-security"]}) - def test_component_with_working_directory(self): + def test_component_with_working_directory(self) -> None: component = self.manifest.components["dashboards-reports"] self.assertEqual(component.name, "dashboards-reports") self.assertEqual(component.working_directory, "reports-scheduler") self.assertEqual(component.integ_test, {"test-configs": ["without-security"]}) self.assertEqual(component.bwc_test, {"test-configs": ["without-security"]}) - def test_to_dict(self): + def test_to_dict(self) -> None: data = self.manifest.to_dict() with open(self.manifest_filename) as f: self.assertEqual(yaml.safe_load(f), data) - def test_versions(self): + def test_versions(self) -> None: self.assertTrue(len(TestManifest.VERSIONS)) for version in TestManifest.VERSIONS: manifest = TestManifest.from_path(os.path.join(self.data_path, "test", f"opensearch-test-schema-version-{version}.yml")) diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py index 04b2f6fe3e..5dadb70039 100644 --- a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py +++ b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py @@ -8,7 +8,7 @@ import unittest from unittest.mock import MagicMock, call, patch -from manifests.input_manifest import InputManifest +from manifests.input_manifest import Component from manifests_workflow.input_manifests_opensearch import InputManifestsOpenSearch @@ -41,7 +41,7 @@ def test_update(self, mock_input_manifest, mock_component_opensearch, mock_compo mock_component_opensearch.return_value = MagicMock(name="common-utils") mock_component_opensearch.checkout.return_value = MagicMock(version="0.10.0") mock_input_manifest_from_path.return_value.components = { - "common-utils": InputManifest.Component({"name": "common-utils", "repository": "git", "ref": "ref"}) + "common-utils": Component({"name": "common-utils", "repository": "git", "ref": "ref"}) } manifests = InputManifestsOpenSearch() manifests.update()