From 23d706a5d6098c071165ab50f41eb7959621220a Mon Sep 17 00:00:00 2001 From: Guillaume Mulocher Date: Mon, 25 Mar 2024 16:48:50 +0100 Subject: [PATCH] test: Add tests (#606) --- tests/units/anta_tests/test_field_notices.py | 92 ++++++++++++--- tests/units/cli/exec/test_utils.py | 11 +- tests/units/cli/get/test_commands.py | 23 ++-- tests/units/cli/test__init__.py | 17 ++- tests/units/test_models.py | 117 +++++++++++++++---- 5 files changed, 212 insertions(+), 48 deletions(-) diff --git a/tests/units/anta_tests/test_field_notices.py b/tests/units/anta_tests/test_field_notices.py index 12c62be84..9edd42ac1 100644 --- a/tests/units/anta_tests/test_field_notices.py +++ b/tests/units/anta_tests/test_field_notices.py @@ -2,7 +2,6 @@ # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. """Test inputs for anta.tests.field_notices.""" - from __future__ import annotations from typing import Any @@ -21,7 +20,9 @@ "modelName": "DCS-7280QRA-C36S", "details": { "deviations": [], - "components": [{"name": "Aboot", "version": "Aboot-veos-8.0.0-3255441"}], + "components": [ + {"name": "Aboot", "version": "Aboot-veos-8.0.0-3255441"} + ], }, }, ], @@ -38,12 +39,17 @@ "modelName": "DCS-7280QRA-C36S", "details": { "deviations": [], - "components": [{"name": "Aboot", "version": "Aboot-veos-4.0.1-3255441"}], + "components": [ + {"name": "Aboot", "version": "Aboot-veos-4.0.1-3255441"} + ], }, }, ], "inputs": None, - "expected": {"result": "failure", "messages": ["device is running incorrect version of aboot (4.0.1)"]}, + "expected": { + "result": "failure", + "messages": ["device is running incorrect version of aboot (4.0.1)"], + }, }, { "name": "failure-4.1", @@ -55,12 +61,17 @@ "modelName": "DCS-7280QRA-C36S", "details": { "deviations": [], - "components": [{"name": "Aboot", "version": "Aboot-veos-4.1.0-3255441"}], + "components": [ + {"name": "Aboot", "version": "Aboot-veos-4.1.0-3255441"} + ], }, }, ], "inputs": None, - "expected": {"result": "failure", "messages": ["device is running incorrect version of aboot (4.1.0)"]}, + "expected": { + "result": "failure", + "messages": ["device is running incorrect version of aboot (4.1.0)"], + }, }, { "name": "failure-6.0", @@ -72,12 +83,17 @@ "modelName": "DCS-7280QRA-C36S", "details": { "deviations": [], - "components": [{"name": "Aboot", "version": "Aboot-veos-6.0.1-3255441"}], + "components": [ + {"name": "Aboot", "version": "Aboot-veos-6.0.1-3255441"} + ], }, }, ], "inputs": None, - "expected": {"result": "failure", "messages": ["device is running incorrect version of aboot (6.0.1)"]}, + "expected": { + "result": "failure", + "messages": ["device is running incorrect version of aboot (6.0.1)"], + }, }, { "name": "failure-6.1", @@ -89,12 +105,17 @@ "modelName": "DCS-7280QRA-C36S", "details": { "deviations": [], - "components": [{"name": "Aboot", "version": "Aboot-veos-6.1.1-3255441"}], + "components": [ + {"name": "Aboot", "version": "Aboot-veos-6.1.1-3255441"} + ], }, }, ], "inputs": None, - "expected": {"result": "failure", "messages": ["device is running incorrect version of aboot (6.1.1)"]}, + "expected": { + "result": "failure", + "messages": ["device is running incorrect version of aboot (6.1.1)"], + }, }, { "name": "skipped-model", @@ -106,12 +127,17 @@ "modelName": "vEOS-lab", "details": { "deviations": [], - "components": [{"name": "Aboot", "version": "Aboot-veos-8.0.0-3255441"}], + "components": [ + {"name": "Aboot", "version": "Aboot-veos-8.0.0-3255441"} + ], }, }, ], "inputs": None, - "expected": {"result": "skipped", "messages": ["device is not impacted by FN044"]}, + "expected": { + "result": "skipped", + "messages": ["device is not impacted by FN044"], + }, }, { "name": "success-JPE", @@ -207,7 +233,10 @@ }, ], "inputs": None, - "expected": {"result": "skipped", "messages": ["Platform is not impacted by FN072"]}, + "expected": { + "result": "skipped", + "messages": ["Platform is not impacted by FN072"], + }, }, { "name": "skipped-range-JPE", @@ -225,6 +254,38 @@ "inputs": None, "expected": {"result": "skipped", "messages": ["Device not exposed"]}, }, + { + "name": "skipped-range-K-JPE", + "test": VerifyFieldNotice72Resolution, + "eos_data": [ + { + "modelName": "DCS-7280SR3K-48YC8", + "serialNumber": "JPE2134000", + "details": { + "deviations": [], + "components": [{"name": "FixedSystemvrm1", "version": "5"}], + }, + }, + ], + "inputs": None, + "expected": {"result": "skipped", "messages": ["Device not exposed"]}, + }, + { + "name": "skipped-range-JAS", + "test": VerifyFieldNotice72Resolution, + "eos_data": [ + { + "modelName": "DCS-7280SR3-48YC8", + "serialNumber": "JAS2041000", + "details": { + "deviations": [], + "components": [{"name": "FixedSystemvrm1", "version": "5"}], + }, + }, + ], + "inputs": None, + "expected": {"result": "skipped", "messages": ["Device not exposed"]}, + }, { "name": "skipped-range-K-JAS", "test": VerifyFieldNotice72Resolution, @@ -287,6 +348,9 @@ }, ], "inputs": None, - "expected": {"result": "error", "messages": ["Error in running test - FixedSystemvrm1 not found"]}, + "expected": { + "result": "error", + "messages": ["Error in running test - FixedSystemvrm1 not found"], + }, }, ] diff --git a/tests/units/cli/exec/test_utils.py b/tests/units/cli/exec/test_utils.py index adf1c7407..ca3519486 100644 --- a/tests/units/cli/exec/test_utils.py +++ b/tests/units/cli/exec/test_utils.py @@ -90,10 +90,13 @@ async def dummy_collect(self: AntaDevice, command: AntaCommand) -> None: command.output = per_device_command_output.get(self.name, "") # Need to patch the child device class - with patch("anta.device.AsyncEOSDevice.collect", side_effect=dummy_collect, autospec=True) as mocked_collect, patch( - "anta.inventory.AntaInventory.connect_inventory", - side_effect=mock_connect_inventory, - ) as mocked_connect_inventory: + with ( + patch("anta.device.AsyncEOSDevice.collect", side_effect=dummy_collect, autospec=True) as mocked_collect, + patch( + "anta.inventory.AntaInventory.connect_inventory", + side_effect=mock_connect_inventory, + ) as mocked_connect_inventory, + ): mocked_collect.side_effect = dummy_collect await clear_counters_utils(test_inventory, tags=tags) diff --git a/tests/units/cli/get/test_commands.py b/tests/units/cli/get/test_commands.py index 67d49bc16..bc564ccb0 100644 --- a/tests/units/cli/get/test_commands.py +++ b/tests/units/cli/get/test_commands.py @@ -63,15 +63,20 @@ def mock_cvp_connect(_self: CvpClient, *_args: str, **_kwargs: str) -> None: raise CvpApiError(msg="mocked CvpApiError") # always get a token - with patch("anta.cli.get.commands.get_cv_token", return_value="dummy_token"), patch( - "cvprac.cvp_client.CvpClient.connect", - autospec=True, - side_effect=mock_cvp_connect, - ) as mocked_cvp_connect, patch("cvprac.cvp_client.CvpApi.get_inventory", autospec=True, return_value=[]) as mocked_get_inventory, patch( - "cvprac.cvp_client.CvpApi.get_devices_in_container", - autospec=True, - return_value=[], - ) as mocked_get_devices_in_container: + with ( + patch("anta.cli.get.commands.get_cv_token", return_value="dummy_token"), + patch( + "cvprac.cvp_client.CvpClient.connect", + autospec=True, + side_effect=mock_cvp_connect, + ) as mocked_cvp_connect, + patch("cvprac.cvp_client.CvpApi.get_inventory", autospec=True, return_value=[]) as mocked_get_inventory, + patch( + "cvprac.cvp_client.CvpApi.get_devices_in_container", + autospec=True, + return_value=[], + ) as mocked_get_devices_in_container, + ): result = click_runner.invoke(anta, cli_args) if not cvp_connect_failure: diff --git a/tests/units/cli/test__init__.py b/tests/units/cli/test__init__.py index 3679e0dfd..070108350 100644 --- a/tests/units/cli/test__init__.py +++ b/tests/units/cli/test__init__.py @@ -6,8 +6,11 @@ from __future__ import annotations from typing import TYPE_CHECKING +from unittest.mock import patch -from anta.cli import anta +import pytest + +from anta.cli import anta, cli from anta.cli.utils import ExitCode if TYPE_CHECKING: @@ -47,3 +50,15 @@ def test_anta_get_help(click_runner: CliRunner) -> None: result = click_runner.invoke(anta, ["get", "--help"]) assert result.exit_code == ExitCode.OK assert "Usage: anta get" in result.output + + +def test_uncaught_failure_anta(caplog: pytest.LogCaptureFixture) -> None: + """Test uncaught failure when running ANTA cli.""" + with ( + pytest.raises(SystemExit) as e_info, + patch("anta.cli.anta", side_effect=ZeroDivisionError()), + ): + cli() + assert "CRITICAL" in caplog.text + assert "Uncaught Exception when running ANTA CLI" in caplog.text + assert e_info.value.code == 1 diff --git a/tests/units/test_models.py b/tests/units/test_models.py index 17c6eaba1..11775e60b 100644 --- a/tests/units/test_models.py +++ b/tests/units/test_models.py @@ -2,7 +2,6 @@ # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. """test anta.models.py.""" - # Mypy does not understand AntaTest.Input typing # mypy: disable-error-code=attr-defined from __future__ import annotations @@ -41,7 +40,9 @@ class FakeTestWithFailedCommand(AntaTest): name = "FakeTestWithFailedCommand" description = "ANTA test with a command that failed" categories: ClassVar[list[str]] = [] - commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show version", errors=["failed command"])] + commands: ClassVar[list[AntaCommand | AntaTemplate]] = [ + AntaCommand(command="show version", errors=["failed command"]) + ] @AntaTest.anta_test def test(self) -> None: @@ -58,7 +59,9 @@ class FakeTestWithUnsupportedCommand(AntaTest): commands: ClassVar[list[AntaCommand | AntaTemplate]] = [ AntaCommand( command="show hardware counter drop", - errors=["Unavailable command (not supported on this hardware platform) (at token 2: 'counter')"], + errors=[ + "Unavailable command (not supported on this hardware platform) (at token 2: 'counter')" + ], ) ] @@ -93,7 +96,9 @@ class FakeTestWithTemplate(AntaTest): name = "FakeTestWithTemplate" description = "ANTA test with template that always succeed" categories: ClassVar[list[str]] = [] - commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaTemplate(template="show interface {interface}")] + commands: ClassVar[list[AntaCommand | AntaTemplate]] = [ + AntaTemplate(template="show interface {interface}") + ] class Input(AntaTest.Input): """Inputs for FakeTestWithTemplate test.""" @@ -116,7 +121,9 @@ class FakeTestWithTemplateNoRender(AntaTest): name = "FakeTestWithTemplateNoRender" description = "ANTA test with template that miss the render() method" categories: ClassVar[list[str]] = [] - commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaTemplate(template="show interface {interface}")] + commands: ClassVar[list[AntaCommand | AntaTemplate]] = [ + AntaTemplate(template="show interface {interface}") + ] class Input(AntaTest.Input): """Inputs for FakeTestWithTemplateNoRender test.""" @@ -133,9 +140,13 @@ class FakeTestWithTemplateBadRender1(AntaTest): """ANTA test with template that raises a AntaTemplateRenderError exception.""" name = "FakeTestWithTemplateBadRender" - description = "ANTA test with template that raises a AntaTemplateRenderError exception" + description = ( + "ANTA test with template that raises a AntaTemplateRenderError exception" + ) categories: ClassVar[list[str]] = [] - commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaTemplate(template="show interface {interface}")] + commands: ClassVar[list[AntaCommand | AntaTemplate]] = [ + AntaTemplate(template="show interface {interface}") + ] class Input(AntaTest.Input): """Inputs for FakeTestWithTemplateBadRender1 test.""" @@ -158,7 +169,9 @@ class FakeTestWithTemplateBadRender2(AntaTest): name = "FakeTestWithTemplateBadRender2" description = "ANTA test with template that raises an arbitrary exception" categories: ClassVar[list[str]] = [] - commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaTemplate(template="show interface {interface}")] + commands: ClassVar[list[AntaCommand | AntaTemplate]] = [ + AntaTemplate(template="show interface {interface}") + ] class Input(AntaTest.Input): """Inputs for FakeTestWithTemplateBadRender2 test.""" @@ -335,7 +348,9 @@ def test(self) -> None: "expected": { "__init__": { "result": "error", - "messages": ["AntaTemplate are provided but render() method has not been implemented for tests.units.test_models.FakeTestWithTemplateNoRender"], + "messages": [ + "AntaTemplate are provided but render() method has not been implemented for tests.units.test_models.FakeTestWithTemplateNoRender" + ], }, "test": {"result": "error"}, }, @@ -347,7 +362,9 @@ def test(self) -> None: "expected": { "__init__": { "result": "error", - "messages": ["Cannot render template {template='show interface {interface}' version='latest' revision=None ofmt='json' use_cache=True}"], + "messages": [ + "Cannot render template {template='show interface {interface}' version='latest' revision=None ofmt='json' use_cache=True}" + ], }, "test": {"result": "error"}, }, @@ -359,7 +376,9 @@ def test(self) -> None: "expected": { "__init__": { "result": "error", - "messages": ["Exception in tests.units.test_models.FakeTestWithTemplateBadRender2.render(): RuntimeError"], + "messages": [ + "Exception in tests.units.test_models.FakeTestWithTemplateBadRender2.render(): RuntimeError" + ], }, "test": {"result": "error"}, }, @@ -429,7 +448,9 @@ def test(self) -> None: "__init__": {"result": "unset"}, "test": { "result": "skipped", - "messages": ["Skipped because show hardware counter drop is not supported on pytest"], + "messages": [ + "Skipped because show hardware counter drop is not supported on pytest" + ], }, }, }, @@ -456,7 +477,10 @@ class WrongTestNoName(AntaTest): def test(self) -> None: self.result.is_success() - assert exec_info.value.args[0] == "Class tests.units.test_models.WrongTestNoName is missing required class attribute name" + assert ( + exec_info.value.args[0] + == "Class tests.units.test_models.WrongTestNoName is missing required class attribute name" + ) with pytest.raises(NotImplementedError) as exec_info: @@ -471,7 +495,10 @@ class WrongTestNoDescription(AntaTest): def test(self) -> None: self.result.is_success() - assert exec_info.value.args[0] == "Class tests.units.test_models.WrongTestNoDescription is missing required class attribute description" + assert ( + exec_info.value.args[0] + == "Class tests.units.test_models.WrongTestNoDescription is missing required class attribute description" + ) with pytest.raises(NotImplementedError) as exec_info: @@ -486,7 +513,10 @@ class WrongTestNoCategories(AntaTest): def test(self) -> None: self.result.is_success() - assert exec_info.value.args[0] == "Class tests.units.test_models.WrongTestNoCategories is missing required class attribute categories" + assert ( + exec_info.value.args[0] + == "Class tests.units.test_models.WrongTestNoCategories is missing required class attribute categories" + ) with pytest.raises(NotImplementedError) as exec_info: @@ -501,22 +531,31 @@ class WrongTestNoCommands(AntaTest): def test(self) -> None: self.result.is_success() - assert exec_info.value.args[0] == "Class tests.units.test_models.WrongTestNoCommands is missing required class attribute commands" + assert ( + exec_info.value.args[0] + == "Class tests.units.test_models.WrongTestNoCommands is missing required class attribute commands" + ) def _assert_test(self, test: AntaTest, expected: dict[str, Any]) -> None: assert test.result.result == expected["result"] if "messages" in expected: - for result_msg, expected_msg in zip(test.result.messages, expected["messages"]): # NOTE: zip(strict=True) has been added in Python 3.10 + for result_msg, expected_msg in zip( + test.result.messages, expected["messages"] + ): # NOTE: zip(strict=True) has been added in Python 3.10 assert expected_msg in result_msg - @pytest.mark.parametrize("data", ANTATEST_DATA, ids=generate_test_ids(ANTATEST_DATA)) + @pytest.mark.parametrize( + "data", ANTATEST_DATA, ids=generate_test_ids(ANTATEST_DATA) + ) def test__init__(self, device: AntaDevice, data: dict[str, Any]) -> None: """Test the AntaTest constructor.""" expected = data["expected"]["__init__"] test = data["test"](device, inputs=data["inputs"]) self._assert_test(test, expected) - @pytest.mark.parametrize("data", ANTATEST_DATA, ids=generate_test_ids(ANTATEST_DATA)) + @pytest.mark.parametrize( + "data", ANTATEST_DATA, ids=generate_test_ids(ANTATEST_DATA) + ) def test_test(self, device: AntaDevice, data: dict[str, Any]) -> None: """Test the AntaTest.test method.""" expected = data["expected"]["test"] @@ -538,7 +577,9 @@ class FakeTestWithBlacklist(AntaTest): name = "FakeTestWithBlacklist" description = "ANTA test that has blacklisted command" categories: ClassVar[list[str]] = [] - commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command=data)] + commands: ClassVar[list[AntaCommand | AntaTemplate]] = [ + AntaCommand(command=data) + ] @AntaTest.anta_test def test(self) -> None: @@ -549,3 +590,39 @@ def test(self) -> None: # Run the test() method asyncio.run(test_instance.test()) assert test_instance.result.result == "error" + + +class TestAntaComamnd: + """Test for anta.models.AntaCommand.""" + + # ruff: noqa: B018 + # pylint: disable=pointless-statement + + def test_empty_output_access(self) -> None: + """Test for both json and text ofmt.""" + json_cmd = AntaCommand(command="show dummy") + text_cmd = AntaCommand(command="show dummy", ofmt="text") + msg = "There is no output for command show dummy" + with pytest.raises(RuntimeError, match=msg): + json_cmd.json_output + with pytest.raises(RuntimeError, match=msg): + text_cmd.text_output + + def test_wrong_format_output_access(self) -> None: + """Test for both json and text ofmt.""" + json_cmd = AntaCommand(command="show dummy", output={}) + json_cmd_2 = AntaCommand(command="show dummy", output="not_json") + text_cmd = AntaCommand(command="show dummy", ofmt="text", output="blah") + text_cmd_2 = AntaCommand( + command="show dummy", ofmt="text", output={"not_a": "string"} + ) + msg = "Output of command show dummy is invalid" + msg = "Output of command show dummy is invalid" + with pytest.raises(RuntimeError, match=msg): + json_cmd.text_output + with pytest.raises(RuntimeError, match=msg): + text_cmd.json_output + with pytest.raises(RuntimeError, match=msg): + json_cmd_2.text_output + with pytest.raises(RuntimeError, match=msg): + text_cmd_2.json_output