From a69d679b6ad6c29389b5ae9e4b092ab3f5a89f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Thu, 12 Sep 2024 15:51:07 +0200 Subject: [PATCH] refactor: tests/units/test_catalog.py --- pyproject.toml | 1 + tests/units/test_catalog.py | 305 +++++++++++++++--------------------- 2 files changed, 126 insertions(+), 180 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 11820caaf..026981512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -384,6 +384,7 @@ runtime-evaluated-base-classes = ["pydantic.BaseModel", "anta.models.AntaTest.In "SLF001", # Lots of private member accessed for test purposes ] "tests/units/*" = [ + "ARG002", # Sometimes we need to declare unused arguments when a parameter is not used but declared in @pytest.mark.parametrize "BLE001", # Do not catch blind exception: `Exception` - already disabling this in pylint "FBT001", # Boolean-typed positional argument in function definition "PLR0913", # Too many arguments to function call (8 > 5) diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index 13046f294..f8ce04ab1 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -7,7 +7,7 @@ from json import load as json_load from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any, Literal import pytest from pydantic import ValidationError @@ -28,33 +28,20 @@ VerifyReloadCause, VerifyUptime, ) -from tests.lib.utils import generate_test_ids_list from tests.units.test_models import FakeTestWithInput -# Test classes used as expected values +if TYPE_CHECKING: + from _pytest.mark.structures import ParameterSet DATA_DIR: Path = Path(__file__).parent.parent.resolve() / "data" -INIT_CATALOG_DATA: list[dict[str, Any]] = [ - { - "name": "test_catalog", - "filename": "test_catalog.yml", - "tests": [ - (VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.31.1F"])), - ], - }, - { - "name": "test_catalog", - "filename": "test_catalog.json", - "file_format": "json", - "tests": [ - (VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.31.1F"])), - ], - }, - { - "name": "test_catalog_with_tags", - "filename": "test_catalog_with_tags.yml", - "tests": [ +INIT_CATALOG_PARAMS: list[ParameterSet] = [ + pytest.param("test_catalog.yml", "yaml", [(VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.31.1F"]))], id="test_catalog_yaml"), + pytest.param("test_catalog.json", "json", [(VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.31.1F"]))], id="test_catalog_json"), + pytest.param( + "test_catalog_with_tags.yml", + "yaml", + [ ( VerifyUptime, VerifyUptime.Input( @@ -79,179 +66,132 @@ (VerifyMlagStatus, None), (VerifyL3MTU, {"mtu": 1500, "filters": {"tags": ["demo"]}}), ], - }, - { - "name": "test_empty_catalog", - "filename": "test_empty_catalog.yml", - "tests": [], - }, - { - "name": "test_empty_dict_catalog", - "filename": "test_empty_dict_catalog.yml", - "tests": [], - }, + id="test_catalog_with_tags", + ), + pytest.param("test_empty_catalog.yml", "yaml", [], id="test_empty_catalog"), + pytest.param("test_empty_dict_catalog.yml", "yaml", [], id="test_empty_dict_catalog"), ] -CATALOG_PARSE_FAIL_DATA: list[dict[str, Any]] = [ - { - "name": "undefined_tests", - "filename": "test_catalog_wrong_format.toto", - "file_format": "toto", - "error": "'toto' is not a valid format for an AntaCatalog file. Only 'yaml' and 'json' are supported.", - }, - { - "name": "invalid_json", - "filename": "test_catalog_invalid_json.json", - "file_format": "json", - "error": "JSONDecodeError", - }, - { - "name": "undefined_tests", - "filename": "test_catalog_with_undefined_tests.yml", - "error": "FakeTest is not defined in Python module anta.tests.software", - }, - { - "name": "undefined_module", - "filename": "test_catalog_with_undefined_module.yml", - "error": "Module named anta.tests.undefined cannot be imported", - }, - { - "name": "undefined_module", - "filename": "test_catalog_with_undefined_module.yml", - "error": "Module named anta.tests.undefined cannot be imported", - }, - { - "name": "syntax_error", - "filename": "test_catalog_with_syntax_error_module.yml", - "error": "Value error, Module named tests.data.syntax_error cannot be imported. Verify that the module exists and there is no Python syntax issues.", - }, - { - "name": "undefined_module_nested", - "filename": "test_catalog_with_undefined_module_nested.yml", - "error": "Module named undefined from package anta.tests cannot be imported", - }, - { - "name": "not_a_list", - "filename": "test_catalog_not_a_list.yml", - "error": "Value error, Syntax error when parsing: True\nIt must be a list of ANTA tests. Check the test catalog.", - }, - { - "name": "test_definition_not_a_dict", - "filename": "test_catalog_test_definition_not_a_dict.yml", - "error": "Value error, Syntax error when parsing: VerifyEOSVersion\nIt must be a dictionary. Check the test catalog.", - }, - { - "name": "test_definition_multiple_dicts", - "filename": "test_catalog_test_definition_multiple_dicts.yml", - "error": "Value error, Syntax error when parsing: {'VerifyEOSVersion': {'versions': ['4.25.4M', '4.26.1F']}, " - "'VerifyTerminAttrVersion': {'versions': ['4.25.4M']}}\nIt must be a dictionary with a single entry. Check the indentation in the test catalog.", - }, - {"name": "wrong_type_after_parsing", "filename": "test_catalog_wrong_type.yml", "error": "must be a dict, got str"}, +CATALOG_PARSE_FAIL_PARAMS: list[ParameterSet] = [ + pytest.param( + "test_catalog_wrong_format.toto", + "toto", + "'toto' is not a valid format for an AntaCatalog file. Only 'yaml' and 'json' are supported.", + id="undefined_tests", + ), + pytest.param("test_catalog_invalid_json.json", "json", "JSONDecodeError", id="invalid_json"), + pytest.param("test_catalog_with_undefined_tests.yml", "yaml", "FakeTest is not defined in Python module anta.tests.software", id="undefined_tests"), + pytest.param("test_catalog_with_undefined_module.yml", "yaml", "Module named anta.tests.undefined cannot be imported", id="undefined_module"), + pytest.param("test_catalog_with_undefined_module.yml", "yaml", "Module named anta.tests.undefined cannot be imported", id="undefined_module"), + pytest.param( + "test_catalog_with_syntax_error_module.yml", + "yaml", + "Value error, Module named tests.data.syntax_error cannot be imported. Verify that the module exists and there is no Python syntax issues.", + id="syntax_error", + ), + pytest.param( + "test_catalog_with_undefined_module_nested.yml", + "yaml", + "Module named undefined from package anta.tests cannot be imported", + id="undefined_module_nested", + ), + pytest.param( + "test_catalog_not_a_list.yml", + "yaml", + "Value error, Syntax error when parsing: True\nIt must be a list of ANTA tests. Check the test catalog.", + id="not_a_list", + ), + pytest.param( + "test_catalog_test_definition_not_a_dict.yml", + "yaml", + "Value error, Syntax error when parsing: VerifyEOSVersion\nIt must be a dictionary. Check the test catalog.", + id="test_definition_not_a_dict", + ), + pytest.param( + "test_catalog_test_definition_multiple_dicts.yml", + "yaml", + "Value error, Syntax error when parsing: {'VerifyEOSVersion': {'versions': ['4.25.4M', '4.26.1F']}, 'VerifyTerminAttrVersion': {'versions': ['4.25.4M']}}\n" + "It must be a dictionary with a single entry. Check the indentation in the test catalog.", + id="test_definition_multiple_dicts", + ), + pytest.param("test_catalog_wrong_type.yml", "yaml", "must be a dict, got str", id="wrong_type_after_parsing"), ] -CATALOG_FROM_DICT_FAIL_DATA: list[dict[str, Any]] = [ - { - "name": "undefined_tests", - "filename": "test_catalog_with_undefined_tests.yml", - "error": "FakeTest is not defined in Python module anta.tests.software", - }, - { - "name": "wrong_type", - "filename": "test_catalog_wrong_type.yml", - "error": "Wrong input type for catalog data, must be a dict, got str", - }, +CATALOG_FROM_DICT_FAIL_PARAMS: list[ParameterSet] = [ + pytest.param("test_catalog_with_undefined_tests.yml", "FakeTest is not defined in Python module anta.tests.software", id="undefined_tests"), + pytest.param("test_catalog_wrong_type.yml", "Wrong input type for catalog data, must be a dict, got str", id="wrong_type"), ] -CATALOG_FROM_LIST_FAIL_DATA: list[dict[str, Any]] = [ - { - "name": "wrong_inputs", - "tests": [ - ( - FakeTestWithInput, - AntaTest.Input(), - ), - ], - "error": "Test input has type AntaTest.Input but expected type FakeTestWithInput.Input", - }, - { - "name": "no_test", - "tests": [(None, None)], - "error": "Input should be a subclass of AntaTest", - }, - { - "name": "no_input_when_required", - "tests": [(FakeTestWithInput, None)], - "error": "FakeTestWithInput test inputs are not valid: 1 validation error for Input\n\tstring\n\t Field required", - }, - { - "name": "wrong_input_type", - "tests": [(FakeTestWithInput, {"string": True})], - "error": "FakeTestWithInput test inputs are not valid: 1 validation error for Input\n\tstring\n\t Input should be a valid string", - }, +CATALOG_FROM_LIST_FAIL_PARAMS: list[ParameterSet] = [ + pytest.param([(FakeTestWithInput, AntaTest.Input())], "Test input has type AntaTest.Input but expected type FakeTestWithInput.Input", id="wrong_inputs"), + pytest.param([(None, None)], "Input should be a subclass of AntaTest", id="no_test"), + pytest.param( + [(FakeTestWithInput, None)], + "FakeTestWithInput test inputs are not valid: 1 validation error for Input\n\tstring\n\t Field required", + id="no_input_when_required", + ), + pytest.param( + [(FakeTestWithInput, {"string": True})], + "FakeTestWithInput test inputs are not valid: 1 validation error for Input\n\tstring\n\t Input should be a valid string", + id="wrong_input_type", + ), ] -TESTS_SETTER_FAIL_DATA: list[dict[str, Any]] = [ - { - "name": "not_a_list", - "tests": "not_a_list", - "error": "The catalog must contain a list of tests", - }, - { - "name": "not_a_list_of_test_definitions", - "tests": [42, 43], - "error": "A test in the catalog must be an AntaTestDefinition instance", - }, +TESTS_SETTER_FAIL_PARAMS: list[ParameterSet] = [ + pytest.param("not_a_list", "The catalog must contain a list of tests", id="not_a_list"), + pytest.param([42, 43], "A test in the catalog must be an AntaTestDefinition instance", id="not_a_list_of_test_definitions"), ] class TestAntaCatalog: """Test for anta.catalog.AntaCatalog.""" - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test_parse(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "tests"), INIT_CATALOG_PARAMS) + def test_parse(self, filename: str, file_format: Literal["yaml", "json"], tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]]) -> None: """Instantiate AntaCatalog from a file.""" - catalog: AntaCatalog = AntaCatalog.parse(DATA_DIR / catalog_data["filename"], file_format=catalog_data.get("file_format", "yaml")) + catalog: AntaCatalog = AntaCatalog.parse(DATA_DIR / filename, file_format=file_format) - assert len(catalog.tests) == len(catalog_data["tests"]) - for test_id, (test, inputs_data) in enumerate(catalog_data["tests"]): + assert len(catalog.tests) == len(tests) + for test_id, (test, inputs_data) in enumerate(tests): assert catalog.tests[test_id].test == test if inputs_data is not None: inputs = test.Input(**inputs_data) if isinstance(inputs_data, dict) else inputs_data assert inputs == catalog.tests[test_id].inputs - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test_from_list(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "tests"), INIT_CATALOG_PARAMS) + def test_from_list(self, tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]]) -> None: """Instantiate AntaCatalog from a list.""" - catalog: AntaCatalog = AntaCatalog.from_list(catalog_data["tests"]) + catalog: AntaCatalog = AntaCatalog.from_list(tests) - assert len(catalog.tests) == len(catalog_data["tests"]) - for test_id, (test, inputs_data) in enumerate(catalog_data["tests"]): + assert len(catalog.tests) == len(tests) + for test_id, (test, inputs_data) in enumerate(tests): assert catalog.tests[test_id].test == test if inputs_data is not None: inputs = test.Input(**inputs_data) if isinstance(inputs_data, dict) else inputs_data assert inputs == catalog.tests[test_id].inputs - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test_from_dict(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "tests"), INIT_CATALOG_PARAMS) + def test_from_dict( + self, filename: str, file_format: Literal["yaml", "json"], tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]] + ) -> None: """Instantiate AntaCatalog from a dict.""" - file = DATA_DIR / catalog_data["filename"] - with file.open(encoding="UTF-8") as file: - file_format = catalog_data.get("file_format", "yaml") - data = safe_load(file) if file_format == "yaml" else json_load(file) + file = DATA_DIR / filename + with file.open(encoding="UTF-8") as f: + data = safe_load(f) if file_format == "yaml" else json_load(f) catalog: AntaCatalog = AntaCatalog.from_dict(data) - assert len(catalog.tests) == len(catalog_data["tests"]) - for test_id, (test, inputs_data) in enumerate(catalog_data["tests"]): + assert len(catalog.tests) == len(tests) + for test_id, (test, inputs_data) in enumerate(tests): assert catalog.tests[test_id].test == test if inputs_data is not None: inputs = test.Input(**inputs_data) if isinstance(inputs_data, dict) else inputs_data assert inputs == catalog.tests[test_id].inputs - @pytest.mark.parametrize("catalog_data", CATALOG_PARSE_FAIL_DATA, ids=generate_test_ids_list(CATALOG_PARSE_FAIL_DATA)) - def test_parse_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "error"), CATALOG_PARSE_FAIL_PARAMS) + def test_parse_fail(self, filename: str, file_format: Literal["yaml", "json"], error: str) -> None: """Errors when instantiating AntaCatalog from a file.""" with pytest.raises((ValidationError, TypeError, ValueError, OSError)) as exec_info: - AntaCatalog.parse(DATA_DIR / catalog_data["filename"], file_format=catalog_data.get("file_format", "yaml")) + AntaCatalog.parse(DATA_DIR / filename, file_format=file_format) if isinstance(exec_info.value, ValidationError): - assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + assert error in exec_info.value.errors()[0]["msg"] else: - assert catalog_data["error"] in str(exec_info) + assert error in str(exec_info) def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: """Errors when instantiating AntaCatalog from a file.""" @@ -263,25 +203,25 @@ def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: assert "Unable to parse ANTA Test Catalog file" in message assert "FileNotFoundError: [Errno 2] No such file or directory" in message - @pytest.mark.parametrize("catalog_data", CATALOG_FROM_LIST_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_LIST_FAIL_DATA)) - def test_from_list_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("tests", "error"), CATALOG_FROM_LIST_FAIL_PARAMS) + def test_from_list_fail(self, tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]], error: str) -> None: """Errors when instantiating AntaCatalog from a list of tuples.""" with pytest.raises(ValidationError) as exec_info: - AntaCatalog.from_list(catalog_data["tests"]) - assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + AntaCatalog.from_list(tests) + assert error in exec_info.value.errors()[0]["msg"] - @pytest.mark.parametrize("catalog_data", CATALOG_FROM_DICT_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_DICT_FAIL_DATA)) - def test_from_dict_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "error"), CATALOG_FROM_DICT_FAIL_PARAMS) + def test_from_dict_fail(self, filename: str, error: str) -> None: """Errors when instantiating AntaCatalog from a list of tuples.""" - file = DATA_DIR / catalog_data["filename"] - with file.open(encoding="UTF-8") as file: - data = safe_load(file) + file = DATA_DIR / filename + with file.open(encoding="UTF-8") as f: + data = safe_load(f) with pytest.raises((ValidationError, TypeError)) as exec_info: AntaCatalog.from_dict(data) if isinstance(exec_info.value, ValidationError): - assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + assert error in exec_info.value.errors()[0]["msg"] else: - assert catalog_data["error"] in str(exec_info) + assert error in str(exec_info) def test_filename(self) -> None: """Test filename.""" @@ -290,25 +230,30 @@ def test_filename(self) -> None: catalog = AntaCatalog(filename=Path("test")) assert catalog.filename == Path("test") - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test__tests_setter_success(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "tests"), INIT_CATALOG_PARAMS) + def test__tests_setter_success( # pylint: disable=unused-argument + self, + filename: str, + file_format: Literal["yaml", "json"], + tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]], + ) -> None: """Success when setting AntaCatalog.tests from a list of tuples.""" catalog = AntaCatalog() - catalog.tests = [AntaTestDefinition(test=test, inputs=inputs) for test, inputs in catalog_data["tests"]] - assert len(catalog.tests) == len(catalog_data["tests"]) - for test_id, (test, inputs_data) in enumerate(catalog_data["tests"]): + catalog.tests = [AntaTestDefinition(test=test, inputs=inputs) for test, inputs in tests] + assert len(catalog.tests) == len(tests) + for test_id, (test, inputs_data) in enumerate(tests): assert catalog.tests[test_id].test == test if inputs_data is not None: inputs = test.Input(**inputs_data) if isinstance(inputs_data, dict) else inputs_data assert inputs == catalog.tests[test_id].inputs - @pytest.mark.parametrize("catalog_data", TESTS_SETTER_FAIL_DATA, ids=generate_test_ids_list(TESTS_SETTER_FAIL_DATA)) - def test__tests_setter_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("tests", "error"), TESTS_SETTER_FAIL_PARAMS) + def test__tests_setter_fail(self, tests: list[Any], error: str) -> None: """Errors when setting AntaCatalog.tests from a list of tuples.""" catalog = AntaCatalog() with pytest.raises(TypeError) as exec_info: - catalog.tests = catalog_data["tests"] - assert catalog_data["error"] in str(exec_info) + catalog.tests = tests + assert error in str(exec_info) def test_build_indexes_all(self) -> None: """Test AntaCatalog.build_indexes()."""