diff --git a/anta/cli/nrfu/commands.py b/anta/cli/nrfu/commands.py index 7c22c90ee..4182e952e 100644 --- a/anta/cli/nrfu/commands.py +++ b/anta/cli/nrfu/commands.py @@ -116,15 +116,8 @@ def tpl_report(ctx: click.Context, template: pathlib.Path, output: pathlib.Path required=True, help="Path to save the report as a Markdown file", ) -@click.option( - "--only-failed-tests", - is_flag=True, - default=False, - show_envvar=True, - help="Only include failed tests in the report.", -) -def md_report(ctx: click.Context, md_output: pathlib.Path, *, only_failed_tests: bool = False) -> None: +def md_report(ctx: click.Context, md_output: pathlib.Path) -> None: """ANTA command to check network state with Markdown report.""" run_tests(ctx) - save_markdown_report(ctx, md_output=md_output, only_failed_tests=only_failed_tests) + save_markdown_report(ctx, md_output=md_output) exit_with_code(ctx) diff --git a/anta/cli/nrfu/utils.py b/anta/cli/nrfu/utils.py index 88015b9e4..48cd07c5b 100644 --- a/anta/cli/nrfu/utils.py +++ b/anta/cli/nrfu/utils.py @@ -135,17 +135,16 @@ def save_to_csv(ctx: click.Context, csv_file: pathlib.Path) -> None: ctx.exit(ExitCode.USAGE_ERROR) -def save_markdown_report(ctx: click.Context, md_output: pathlib.Path, *, only_failed_tests: bool = False) -> None: +def save_markdown_report(ctx: click.Context, md_output: pathlib.Path) -> None: """Save the markdown report to a file. Parameters ---------- ctx: Click context containing the result manager. md_output: Path to save the markdown report. - only_failed_tests: If True, only failed tests will be included in the report. Default is False. """ try: - MDReportGenerator.generate(results=_get_result_manager(ctx), md_filename=md_output, only_failed_tests=only_failed_tests) + MDReportGenerator.generate(results=_get_result_manager(ctx), md_filename=md_output) console.print(f"Markdown report saved to {md_output} ✅", style="cyan") except OSError: console.print(f"Failed to save Markdown report to {md_output} ❌", style="cyan") diff --git a/anta/constants.py b/anta/constants.py index 7a211a75f..175a4adcc 100644 --- a/anta/constants.py +++ b/anta/constants.py @@ -15,5 +15,5 @@ - [Summary Totals](#summary-totals) - [Summary Totals Device Under Test](#summary-totals-device-under-test) - [Summary Totals Per Category](#summary-totals-per-category) - - [Failed Test Results Summary](#failed-test-results-summary)""" + - [Test Results](#test-results)""" """Table of Contents for the Markdown report.""" diff --git a/anta/reporter/md_reporter.py b/anta/reporter/md_reporter.py index 82a978ce9..0cc5b03e2 100644 --- a/anta/reporter/md_reporter.py +++ b/anta/reporter/md_reporter.py @@ -32,24 +32,17 @@ class MDReportGenerator: The `generate` class method will loop over all the section subclasses and call their `generate_section` method. The final report will be generated in the same order as the `sections` list of the method. - - The class method also accepts an optional `only_failed_tests` flag to generate a report with only failed tests. - - By default, the report will include all test results. """ @classmethod - def generate(cls, results: ResultManager, md_filename: Path, *, only_failed_tests: bool = False) -> None: + def generate(cls, results: ResultManager, md_filename: Path) -> None: """Generate and write the various sections of the markdown report. Parameters ---------- results: The ResultsManager instance containing all test results. md_filename: The path to the markdown file to write the report into. - only_failed_tests: Flag to generate a report with only failed tests. Defaults to False. """ - MDReportBase.ONLY_FAILED_TESTS = only_failed_tests - try: with md_filename.open("w", encoding="utf-8") as mdfile: sections: list[MDReportBase] = [ @@ -58,8 +51,7 @@ def generate(cls, results: ResultManager, md_filename: Path, *, only_failed_test SummaryTotals(mdfile, results), SummaryTotalsDeviceUnderTest(mdfile, results), SummaryTotalsPerCategory(mdfile, results), - FailedTestResultsSummary(mdfile, results), - AllTestResults(mdfile, results), + TestResults(mdfile, results), ] for section in sections: section.generate_section() @@ -76,8 +68,6 @@ class MDReportBase(ABC): to generate and write content to the provided markdown file. """ - ONLY_FAILED_TESTS: ClassVar[bool] = False - def __init__(self, mdfile: TextIOWrapper, results: ResultManager) -> None: """Initialize the MDReportBase with an open markdown file object to write to and a ResultManager instance. @@ -113,7 +103,7 @@ def generate_heading_name(self) -> str: ------- str: Formatted header name. - Example: + Example ------- - `ANTAReport` will become ANTA Report. - `TestResultsSummary` will become Test Results Summary. @@ -151,7 +141,7 @@ def write_heading(self, heading_level: int) -> None: ---------- heading_level: The level of the heading (1-6). - Example: + Example ------- ## Test Results Summary """ @@ -190,10 +180,6 @@ def generate_section(self) -> None: """Generate the `# ANTA Report` section of the markdown report.""" self.write_heading(heading_level=1) toc = MD_REPORT_TOC - - if not self.ONLY_FAILED_TESTS: - toc += "\n - [All Test Results](#all-test-results)" - self.mdfile.write(toc + "\n\n") @@ -217,10 +203,10 @@ def generate_rows(self) -> Generator[str, None, None]: """Generate the rows of the summary totals table.""" yield ( f"| {self.results.get_total_results()} " - f"| {self.results.get_total_results('success')} " - f"| {self.results.get_total_results('skipped')} " - f"| {self.results.get_total_results('failure')} " - f"| {self.results.get_total_results('error')} |\n" + f"| {self.results.get_total_results({'success'})} " + f"| {self.results.get_total_results({'skipped'})} " + f"| {self.results.get_total_results({'failure'})} " + f"| {self.results.get_total_results({'error'})} |\n" ) def generate_section(self) -> None: @@ -277,35 +263,8 @@ def generate_section(self) -> None: self.write_table(table_heading=self.TABLE_HEADING) -class FailedTestResultsSummary(MDReportBase): - """Generate the `## Failed Test Results Summary` section of the markdown report.""" - - TABLE_HEADING: ClassVar[list[str]] = [ - "| Device Under Test | Categories | Test | Description | Custom Field | Result | Messages |", - "| ----------------- | ---------- | ---- | ----------- | ------------ | ------ | -------- |", - ] - - def generate_rows(self) -> Generator[str, None, None]: - """Generate the rows of the failed test results table.""" - for result in self.results.get_results(status={"failure", "error"}, sort_by=["name", "test"]): - messages = self.safe_markdown(", ".join(result.messages)) - categories = ", ".join(result.categories) - yield ( - f"| {result.name or '-'} | {categories or '-'} | {result.test or '-'} " - f"| {result.description or '-'} | {self.safe_markdown(result.custom_field) or '-'} | {result.result or '-'} | {messages or '-'} |\n" - ) - - def generate_section(self) -> None: - """Generate the `## Failed Test Results Summary` section of the markdown report.""" - self.write_heading(heading_level=2) - self.write_table(table_heading=self.TABLE_HEADING, last_table=self.ONLY_FAILED_TESTS) - - -class AllTestResults(MDReportBase): - """Generates the `## All Test Results` section of the markdown report. - - This section is generated only if the report includes all results. - """ +class TestResults(MDReportBase): + """Generates the `## Test Results` section of the markdown report.""" TABLE_HEADING: ClassVar[list[str]] = [ "| Device Under Test | Categories | Test | Description | Custom Field | Result | Messages |", @@ -323,10 +282,6 @@ def generate_rows(self) -> Generator[str, None, None]: ) def generate_section(self) -> None: - """Generate the `## All Test Results` section of the markdown report. - - This section is generated only if the report includes all results. - """ - if not self.ONLY_FAILED_TESTS: - self.write_heading(heading_level=2) - self.write_table(table_heading=self.TABLE_HEADING, last_table=True) + """Generate the `## Test Results` section of the markdown report.""" + self.write_heading(heading_level=2) + self.write_table(table_heading=self.TABLE_HEADING, last_table=True) diff --git a/anta/result_manager/__init__.py b/anta/result_manager/__init__.py index 72e5b1979..1900a28b1 100644 --- a/anta/result_manager/__init__.py +++ b/anta/result_manager/__init__.py @@ -209,29 +209,22 @@ def add(self, result: TestResult) -> None: # Every time a new result is added, we need to clear the cached property self.__dict__.pop("results_by_status", None) - def get_results(self, status: set[TestStatus] | TestStatus | None = None, sort_by: list[str] | None = None) -> list[TestResult]: + def get_results(self, status: set[TestStatus] | None = None, sort_by: list[str] | None = None) -> list[TestResult]: """Get the results, optionally filtered by status and sorted by TestResult fields. If no status is provided, all results are returned. Parameters ---------- - status: Optional TestStatus or set of TestStatus literals to filter the results. + status: Optional set of TestStatus literals to filter the results. sort_by: Optional list of TestResult fields to sort the results. Returns ------- List of TestResult. """ - if status is None: - # Return all results - results = self._result_entries - elif isinstance(status, set): - # Return results for multiple statuses - results = list(chain.from_iterable(self.results_by_status.get(status, []) for status in status)) - else: - # Return results for a single status - results = self.results_by_status.get(status, []) + # Return all results if no status is provided, otherwise return results for multiple statuses + results = self._result_entries if status is None else list(chain.from_iterable(self.results_by_status.get(status, []) for status in status)) if sort_by: accepted_fields = TestResult.model_fields.keys() @@ -242,14 +235,14 @@ def get_results(self, status: set[TestStatus] | TestStatus | None = None, sort_b return results - def get_total_results(self, status: set[TestStatus] | TestStatus | None = None) -> int: + def get_total_results(self, status: set[TestStatus] | None = None) -> int: """Get the total number of results, optionally filtered by status. If no status is provided, the total number of results is returned. Parameters ---------- - status: TestStatus or set of TestStatus literals to filter the results. + status: Optional set of TestStatus literals to filter the results. Returns ------- @@ -259,12 +252,8 @@ def get_total_results(self, status: set[TestStatus] | TestStatus | None = None) # Return the total number of results return sum(len(results) for results in self.results_by_status.values()) - if isinstance(status, set): - # Return the total number of results for multiple statuses - return sum(len(self.results_by_status.get(status, [])) for status in status) - - # Return the total number of results for a single status - return len(self.results_by_status.get(status, [])) + # Return the total number of results for multiple statuses + return sum(len(self.results_by_status.get(status, [])) for status in status) def get_status(self, *, ignore_error: bool = False) -> str: """Return the current status including error_status if ignore_error is False.""" diff --git a/docs/cli/nrfu.md b/docs/cli/nrfu.md index e21a1bcb7..e1f61e4df 100644 --- a/docs/cli/nrfu.md +++ b/docs/cli/nrfu.md @@ -45,7 +45,7 @@ Options `--device` and `--test` can be used to target one or multiple devices an ### Hide results -Option `--hide` can be used to hide test results in the output based on their status. The option can be repeated. Example: `anta nrfu --hide error --hide skipped`. +Option `--hide` can be used to hide test results in the output or report file based on their status. The option can be repeated. Example: `anta nrfu --hide error --hide skipped`. ## Performing NRFU with text rendering @@ -169,9 +169,7 @@ Options: ## Performing NRFU and saving results in a Markdown file -The `md-report` command in NRFU testing generates a comprehensive Markdown report containing various sections, including detailed statistics for devices and test categories. By default, this command saves all test results from the current run in the Markdown file. However, you can choose to include only failed tests by using the `--only-failed-tests` option in the command line interface. - -This feature is particularly useful for comparing reports from multiple runs, allowing you to quickly identify and track issues across your network over time. By focusing on failed tests, you can efficiently pinpoint areas that require attention or have shown improvement between different test executions. +The `md-report` command in NRFU testing generates a comprehensive Markdown report containing various sections, including detailed statistics for devices and test categories. ### Command overview @@ -183,11 +181,9 @@ Usage: anta nrfu md-report [OPTIONS] ANTA command to check network state with Markdown report. Options: - --md-output FILE Path to save the report as a Markdown file [env var: - ANTA_NRFU_MD_REPORT_MD_OUTPUT; required] - --only-failed-tests Only include failed tests in the report. [env var: - ANTA_NRFU_MD_REPORT_ONLY_FAILED_TESTS] - --help Show this message and exit. + --md-output FILE Path to save the report as a Markdown file [env var: + ANTA_NRFU_MD_REPORT_MD_OUTPUT; required] + --help Show this message and exit. ``` ### Example diff --git a/tests/data/test_md_report_only_failed_tests.md b/tests/data/test_md_report.md similarity index 84% rename from tests/data/test_md_report_only_failed_tests.md rename to tests/data/test_md_report.md index 34cac395a..9360dbc74 100644 --- a/tests/data/test_md_report_only_failed_tests.md +++ b/tests/data/test_md_report.md @@ -7,7 +7,7 @@ - [Summary Totals](#summary-totals) - [Summary Totals Device Under Test](#summary-totals-device-under-test) - [Summary Totals Per Category](#summary-totals-per-category) - - [Failed Test Results Summary](#failed-test-results-summary) + - [Test Results](#test-results) ## Test Results Summary @@ -43,7 +43,7 @@ | System | 2 | 0 | 0 | 2 | 0 | | VXLAN | 2 | 1 | 1 | 0 | 0 | -## Failed Test Results Summary +## Test Results | Device Under Test | Categories | Test | Description | Custom Field | Result | Messages | | ----------------- | ---------- | ---- | ----------- | ------------ | ------ | -------- | @@ -51,20 +51,29 @@ | DC1-LEAF1A | BGP | VerifyBGPPeerCount | Verifies the count of BGP peers. | - | failure | Failures: [{'afi': 'ipv4', 'safi': 'unicast', 'vrfs': {'PROD': 'Expected: 2, Actual: 1'}}, {'afi': 'ipv4', 'safi': 'multicast', 'vrfs': {'DEV': 'Expected: 3, Actual: 0'}}] | | DC1-LEAF1A | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] | | DC1-LEAF1A | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-LEAF1A' instead. | +| DC1-LEAF1A | Interfaces | VerifyInterfaceUtilization | Verifies that the utilization of interfaces is below a certain threshold. | - | success | - | | DC1-LEAF1A | Connectivity | VerifyLLDPNeighbors | Verifies that the provided LLDP neighbors are connected properly. | - | failure | Wrong LLDP neighbor(s) on port(s): Ethernet1 DC1-SPINE1_Ethernet1 Ethernet2 DC1-SPINE2_Ethernet1 Port(s) not configured: Ethernet7 | +| DC1-LEAF1A | MLAG | VerifyMlagStatus | Verifies the health status of the MLAG configuration. | - | success | - | | DC1-LEAF1A | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' | | DC1-LEAF1A | Connectivity | VerifyReachability | Test the network reachability to one or many destination IP(s). | - | error | ping vrf MGMT 1.1.1.1 source Management1 repeat 2 has failed: No source interface Management1 | +| DC1-LEAF1A | Routing | VerifyRoutingTableEntry | Verifies that the provided routes are present in the routing table of a specified VRF. | - | success | - | | DC1-LEAF1A | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | Wrong STP mode configured for the following VLAN(s): [10, 20] | | DC1-LEAF1A | SNMP | VerifySnmpStatus | Verifies if the SNMP agent is enabled. | - | failure | SNMP agent disabled in vrf default | | DC1-LEAF1A | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default | +| DC1-LEAF1A | Security | VerifyTelnetStatus | Verifies if Telnet is disabled in the default VRF. | - | success | - | +| DC1-LEAF1A | VXLAN | VerifyVxlan1Interface | Verifies the Vxlan1 interface status. | - | success | - | | DC1-SPINE1 | BFD | VerifyBFDSpecificPeers | Verifies the IPv4 BFD peer's sessions and remote disc in the specified VRF. | - | failure | Following BFD peers are not configured, status is not up or remote disc is zero: {'192.0.255.8': {'default': 'Not Configured'}, '192.0.255.7': {'default': 'Not Configured'}} | | DC1-SPINE1 | BGP | VerifyBGPPeerCount | Verifies the count of BGP peers. | - | failure | Failures: [{'afi': 'ipv4', 'safi': 'unicast', 'vrfs': {'PROD': 'Not Configured', 'default': 'Expected: 3, Actual: 4'}}, {'afi': 'ipv4', 'safi': 'multicast', 'vrfs': {'DEV': 'Not Configured'}}, {'afi': 'evpn', 'vrfs': {'default': 'Expected: 2, Actual: 4'}}] | | DC1-SPINE1 | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] | | DC1-SPINE1 | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-SPINE1' instead. | +| DC1-SPINE1 | Interfaces | VerifyInterfaceUtilization | Verifies that the utilization of interfaces is below a certain threshold. | - | success | - | | DC1-SPINE1 | Connectivity | VerifyLLDPNeighbors | Verifies that the provided LLDP neighbors are connected properly. | - | failure | Wrong LLDP neighbor(s) on port(s): Ethernet1 DC1-LEAF1A_Ethernet1 Ethernet2 DC1-LEAF1B_Ethernet1 Port(s) not configured: Ethernet7 | +| DC1-SPINE1 | MLAG | VerifyMlagStatus | Verifies the health status of the MLAG configuration. | - | skipped | MLAG is disabled | | DC1-SPINE1 | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' | | DC1-SPINE1 | Connectivity | VerifyReachability | Test the network reachability to one or many destination IP(s). | - | error | ping vrf MGMT 1.1.1.1 source Management1 repeat 2 has failed: No source interface Management1 | | DC1-SPINE1 | Routing | VerifyRoutingTableEntry | Verifies that the provided routes are present in the routing table of a specified VRF. | - | failure | The following route(s) are missing from the routing table of VRF default: ['10.1.0.2'] | | DC1-SPINE1 | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | STP mode 'rapidPvst' not configured for the following VLAN(s): [10, 20] | | DC1-SPINE1 | SNMP | VerifySnmpStatus | Verifies if the SNMP agent is enabled. | - | failure | SNMP agent disabled in vrf default | | DC1-SPINE1 | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default | +| DC1-SPINE1 | Security | VerifyTelnetStatus | Verifies if Telnet is disabled in the default VRF. | - | success | - | +| DC1-SPINE1 | VXLAN | VerifyVxlan1Interface | Verifies the Vxlan1 interface status. | - | skipped | Vxlan1 interface is not configured | diff --git a/tests/data/test_md_report_all_tests.md b/tests/data/test_md_report_all_tests.md deleted file mode 100644 index 8eacb5bdf..000000000 --- a/tests/data/test_md_report_all_tests.md +++ /dev/null @@ -1,106 +0,0 @@ -# ANTA Report - -**Table of Contents:** - -- [ANTA Report](#anta-report) - - [Test Results Summary](#test-results-summary) - - [Summary Totals](#summary-totals) - - [Summary Totals Device Under Test](#summary-totals-device-under-test) - - [Summary Totals Per Category](#summary-totals-per-category) - - [Failed Test Results Summary](#failed-test-results-summary) - - [All Test Results](#all-test-results) - -## Test Results Summary - -### Summary Totals - -| Total Tests | Total Tests Success | Total Tests Skipped | Total Tests Failure | Total Tests Error | -| ----------- | ------------------- | ------------------- | ------------------- | ------------------| -| 30 | 7 | 2 | 19 | 2 | - -### Summary Totals Device Under Test - -| Device Under Test | Total Tests | Tests Success | Tests Skipped | Tests Failure | Tests Error | Categories Skipped | Categories Failed | -| ------------------| ----------- | ------------- | ------------- | ------------- | ----------- | -------------------| ------------------| -| DC1-SPINE1 | 15 | 2 | 2 | 10 | 1 | MLAG, VXLAN | AAA, BFD, BGP, Connectivity, Routing, SNMP, STP, Services, Software, System | -| DC1-LEAF1A | 15 | 5 | 0 | 9 | 1 | - | AAA, BFD, BGP, Connectivity, SNMP, STP, Services, Software, System | - -### Summary Totals Per Category - -| Test Category | Total Tests | Tests Success | Tests Skipped | Tests Failure | Tests Error | -| ------------- | ----------- | ------------- | ------------- | ------------- | ----------- | -| AAA | 2 | 0 | 0 | 2 | 0 | -| BFD | 2 | 0 | 0 | 2 | 0 | -| BGP | 2 | 0 | 0 | 2 | 0 | -| Connectivity | 4 | 0 | 0 | 2 | 2 | -| Interfaces | 2 | 2 | 0 | 0 | 0 | -| MLAG | 2 | 1 | 1 | 0 | 0 | -| Routing | 2 | 1 | 0 | 1 | 0 | -| SNMP | 2 | 0 | 0 | 2 | 0 | -| STP | 2 | 0 | 0 | 2 | 0 | -| Security | 2 | 2 | 0 | 0 | 0 | -| Services | 2 | 0 | 0 | 2 | 0 | -| Software | 2 | 0 | 0 | 2 | 0 | -| System | 2 | 0 | 0 | 2 | 0 | -| VXLAN | 2 | 1 | 1 | 0 | 0 | - -## Failed Test Results Summary - -| Device Under Test | Categories | Test | Description | Custom Field | Result | Messages | -| ----------------- | ---------- | ---- | ----------- | ------------ | ------ | -------- | -| DC1-LEAF1A | BFD | VerifyBFDSpecificPeers | Verifies the IPv4 BFD peer's sessions and remote disc in the specified VRF. | - | failure | Following BFD peers are not configured, status is not up or remote disc is zero: {'192.0.255.8': {'default': 'Not Configured'}, '192.0.255.7': {'default': 'Not Configured'}} | -| DC1-LEAF1A | BGP | VerifyBGPPeerCount | Verifies the count of BGP peers. | - | failure | Failures: [{'afi': 'ipv4', 'safi': 'unicast', 'vrfs': {'PROD': 'Expected: 2, Actual: 1'}}, {'afi': 'ipv4', 'safi': 'multicast', 'vrfs': {'DEV': 'Expected: 3, Actual: 0'}}] | -| DC1-LEAF1A | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] | -| DC1-LEAF1A | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-LEAF1A' instead. | -| DC1-LEAF1A | Connectivity | VerifyLLDPNeighbors | Verifies that the provided LLDP neighbors are connected properly. | - | failure | Wrong LLDP neighbor(s) on port(s): Ethernet1 DC1-SPINE1_Ethernet1 Ethernet2 DC1-SPINE2_Ethernet1 Port(s) not configured: Ethernet7 | -| DC1-LEAF1A | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' | -| DC1-LEAF1A | Connectivity | VerifyReachability | Test the network reachability to one or many destination IP(s). | - | error | ping vrf MGMT 1.1.1.1 source Management1 repeat 2 has failed: No source interface Management1 | -| DC1-LEAF1A | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | Wrong STP mode configured for the following VLAN(s): [10, 20] | -| DC1-LEAF1A | SNMP | VerifySnmpStatus | Verifies if the SNMP agent is enabled. | - | failure | SNMP agent disabled in vrf default | -| DC1-LEAF1A | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default | -| DC1-SPINE1 | BFD | VerifyBFDSpecificPeers | Verifies the IPv4 BFD peer's sessions and remote disc in the specified VRF. | - | failure | Following BFD peers are not configured, status is not up or remote disc is zero: {'192.0.255.8': {'default': 'Not Configured'}, '192.0.255.7': {'default': 'Not Configured'}} | -| DC1-SPINE1 | BGP | VerifyBGPPeerCount | Verifies the count of BGP peers. | - | failure | Failures: [{'afi': 'ipv4', 'safi': 'unicast', 'vrfs': {'PROD': 'Not Configured', 'default': 'Expected: 3, Actual: 4'}}, {'afi': 'ipv4', 'safi': 'multicast', 'vrfs': {'DEV': 'Not Configured'}}, {'afi': 'evpn', 'vrfs': {'default': 'Expected: 2, Actual: 4'}}] | -| DC1-SPINE1 | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] | -| DC1-SPINE1 | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-SPINE1' instead. | -| DC1-SPINE1 | Connectivity | VerifyLLDPNeighbors | Verifies that the provided LLDP neighbors are connected properly. | - | failure | Wrong LLDP neighbor(s) on port(s): Ethernet1 DC1-LEAF1A_Ethernet1 Ethernet2 DC1-LEAF1B_Ethernet1 Port(s) not configured: Ethernet7 | -| DC1-SPINE1 | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' | -| DC1-SPINE1 | Connectivity | VerifyReachability | Test the network reachability to one or many destination IP(s). | - | error | ping vrf MGMT 1.1.1.1 source Management1 repeat 2 has failed: No source interface Management1 | -| DC1-SPINE1 | Routing | VerifyRoutingTableEntry | Verifies that the provided routes are present in the routing table of a specified VRF. | - | failure | The following route(s) are missing from the routing table of VRF default: ['10.1.0.2'] | -| DC1-SPINE1 | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | STP mode 'rapidPvst' not configured for the following VLAN(s): [10, 20] | -| DC1-SPINE1 | SNMP | VerifySnmpStatus | Verifies if the SNMP agent is enabled. | - | failure | SNMP agent disabled in vrf default | -| DC1-SPINE1 | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default | - -## All Test Results - -| Device Under Test | Categories | Test | Description | Custom Field | Result | Messages | -| ----------------- | ---------- | ---- | ----------- | ------------ | ------ | -------- | -| DC1-LEAF1A | BFD | VerifyBFDSpecificPeers | Verifies the IPv4 BFD peer's sessions and remote disc in the specified VRF. | - | failure | Following BFD peers are not configured, status is not up or remote disc is zero: {'192.0.255.8': {'default': 'Not Configured'}, '192.0.255.7': {'default': 'Not Configured'}} | -| DC1-LEAF1A | BGP | VerifyBGPPeerCount | Verifies the count of BGP peers. | - | failure | Failures: [{'afi': 'ipv4', 'safi': 'unicast', 'vrfs': {'PROD': 'Expected: 2, Actual: 1'}}, {'afi': 'ipv4', 'safi': 'multicast', 'vrfs': {'DEV': 'Expected: 3, Actual: 0'}}] | -| DC1-LEAF1A | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] | -| DC1-LEAF1A | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-LEAF1A' instead. | -| DC1-LEAF1A | Interfaces | VerifyInterfaceUtilization | Verifies that the utilization of interfaces is below a certain threshold. | - | success | - | -| DC1-LEAF1A | Connectivity | VerifyLLDPNeighbors | Verifies that the provided LLDP neighbors are connected properly. | - | failure | Wrong LLDP neighbor(s) on port(s): Ethernet1 DC1-SPINE1_Ethernet1 Ethernet2 DC1-SPINE2_Ethernet1 Port(s) not configured: Ethernet7 | -| DC1-LEAF1A | MLAG | VerifyMlagStatus | Verifies the health status of the MLAG configuration. | - | success | - | -| DC1-LEAF1A | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' | -| DC1-LEAF1A | Connectivity | VerifyReachability | Test the network reachability to one or many destination IP(s). | - | error | ping vrf MGMT 1.1.1.1 source Management1 repeat 2 has failed: No source interface Management1 | -| DC1-LEAF1A | Routing | VerifyRoutingTableEntry | Verifies that the provided routes are present in the routing table of a specified VRF. | - | success | - | -| DC1-LEAF1A | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | Wrong STP mode configured for the following VLAN(s): [10, 20] | -| DC1-LEAF1A | SNMP | VerifySnmpStatus | Verifies if the SNMP agent is enabled. | - | failure | SNMP agent disabled in vrf default | -| DC1-LEAF1A | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default | -| DC1-LEAF1A | Security | VerifyTelnetStatus | Verifies if Telnet is disabled in the default VRF. | - | success | - | -| DC1-LEAF1A | VXLAN | VerifyVxlan1Interface | Verifies the Vxlan1 interface status. | - | success | - | -| DC1-SPINE1 | BFD | VerifyBFDSpecificPeers | Verifies the IPv4 BFD peer's sessions and remote disc in the specified VRF. | - | failure | Following BFD peers are not configured, status is not up or remote disc is zero: {'192.0.255.8': {'default': 'Not Configured'}, '192.0.255.7': {'default': 'Not Configured'}} | -| DC1-SPINE1 | BGP | VerifyBGPPeerCount | Verifies the count of BGP peers. | - | failure | Failures: [{'afi': 'ipv4', 'safi': 'unicast', 'vrfs': {'PROD': 'Not Configured', 'default': 'Expected: 3, Actual: 4'}}, {'afi': 'ipv4', 'safi': 'multicast', 'vrfs': {'DEV': 'Not Configured'}}, {'afi': 'evpn', 'vrfs': {'default': 'Expected: 2, Actual: 4'}}] | -| DC1-SPINE1 | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] | -| DC1-SPINE1 | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-SPINE1' instead. | -| DC1-SPINE1 | Interfaces | VerifyInterfaceUtilization | Verifies that the utilization of interfaces is below a certain threshold. | - | success | - | -| DC1-SPINE1 | Connectivity | VerifyLLDPNeighbors | Verifies that the provided LLDP neighbors are connected properly. | - | failure | Wrong LLDP neighbor(s) on port(s): Ethernet1 DC1-LEAF1A_Ethernet1 Ethernet2 DC1-LEAF1B_Ethernet1 Port(s) not configured: Ethernet7 | -| DC1-SPINE1 | MLAG | VerifyMlagStatus | Verifies the health status of the MLAG configuration. | - | skipped | MLAG is disabled | -| DC1-SPINE1 | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' | -| DC1-SPINE1 | Connectivity | VerifyReachability | Test the network reachability to one or many destination IP(s). | - | error | ping vrf MGMT 1.1.1.1 source Management1 repeat 2 has failed: No source interface Management1 | -| DC1-SPINE1 | Routing | VerifyRoutingTableEntry | Verifies that the provided routes are present in the routing table of a specified VRF. | - | failure | The following route(s) are missing from the routing table of VRF default: ['10.1.0.2'] | -| DC1-SPINE1 | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | STP mode 'rapidPvst' not configured for the following VLAN(s): [10, 20] | -| DC1-SPINE1 | SNMP | VerifySnmpStatus | Verifies if the SNMP agent is enabled. | - | failure | SNMP agent disabled in vrf default | -| DC1-SPINE1 | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default | -| DC1-SPINE1 | Security | VerifyTelnetStatus | Verifies if Telnet is disabled in the default VRF. | - | success | - | -| DC1-SPINE1 | VXLAN | VerifyVxlan1Interface | Verifies the Vxlan1 interface status. | - | skipped | Vxlan1 interface is not configured | diff --git a/tests/units/cli/nrfu/test_commands.py b/tests/units/cli/nrfu/test_commands.py index 53f419842..a0b97947f 100644 --- a/tests/units/cli/nrfu/test_commands.py +++ b/tests/units/cli/nrfu/test_commands.py @@ -116,7 +116,7 @@ def test_anta_nrfu_csv_failure(click_runner: CliRunner, tmp_path: Path) -> None: assert not csv_output.exists() -def test_anta_nrfu_md_report_all_tests(click_runner: CliRunner, tmp_path: Path) -> None: +def test_anta_nrfu_md_report(click_runner: CliRunner, tmp_path: Path) -> None: """Test anta nrfu md-report.""" md_output = tmp_path / "test.md" result = click_runner.invoke(anta, ["nrfu", "md-report", "--md-output", str(md_output)]) @@ -125,15 +125,6 @@ def test_anta_nrfu_md_report_all_tests(click_runner: CliRunner, tmp_path: Path) assert md_output.exists() -def test_anta_nrfu_md_report_only_failed_tests(click_runner: CliRunner, tmp_path: Path) -> None: - """Test anta nrfu md-report --only-failed-tests.""" - md_output = tmp_path / "test.md" - result = click_runner.invoke(anta, ["nrfu", "md-report", "--md-output", str(md_output), "--only-failed-tests"]) - assert result.exit_code == ExitCode.OK - assert "Markdown report saved to" in result.output - assert md_output.exists() - - def test_anta_nrfu_md_report_failure(click_runner: CliRunner, tmp_path: Path) -> None: """Test anta nrfu md-report failure.""" md_output = tmp_path / "test.md" diff --git a/tests/units/reporter/test_md_reporter.py b/tests/units/reporter/test_md_reporter.py index d3e18e0de..a60773374 100644 --- a/tests/units/reporter/test_md_reporter.py +++ b/tests/units/reporter/test_md_reporter.py @@ -16,23 +16,17 @@ DATA_DIR: Path = Path(__file__).parent.parent.parent.resolve() / "data" -@pytest.mark.parametrize( - ("only_failed_tests", "expected_report_name"), - [ - pytest.param(True, "test_md_report_only_failed_tests.md", id="only_failed_tests"), - pytest.param(False, "test_md_report_all_tests.md", id="all_tests"), - ], -) -def test_md_report_generate(tmp_path: Path, result_manager: ResultManager, expected_report_name: str, *, only_failed_tests: bool) -> None: +def test_md_report_generate(tmp_path: Path, result_manager: ResultManager) -> None: """Test the MDReportGenerator class.""" md_filename = tmp_path / "test.md" + expected_report = "test_md_report.md" # Generate the Markdown report - MDReportGenerator.generate(result_manager, md_filename, only_failed_tests=only_failed_tests) + MDReportGenerator.generate(result_manager, md_filename) assert md_filename.exists() # Load the existing Markdown report to compare with the generated one - with (DATA_DIR / expected_report_name).open("r", encoding="utf-8") as f: + with (DATA_DIR / expected_report).open("r", encoding="utf-8") as f: expected_content = f.read() # Check the content of the Markdown file diff --git a/tests/units/result_manager/test__init__.py b/tests/units/result_manager/test__init__.py index f73ff231c..66a6cfb1d 100644 --- a/tests/units/result_manager/test__init__.py +++ b/tests/units/result_manager/test__init__.py @@ -199,7 +199,7 @@ def test_add_clear_cache(self, result_manager: ResultManager, test_result_factor def test_get_results(self, result_manager: ResultManager) -> None: """Test ResultManager.get_results.""" # Check for single status - success_results = result_manager.get_results(status="success") + success_results = result_manager.get_results(status={"success"}) assert len(success_results) == 7 assert all(r.result == "success" for r in success_results) @@ -246,10 +246,10 @@ def test_get_total_results(self, result_manager: ResultManager) -> None: assert result_manager.get_total_results() == 30 # Test single status - assert result_manager.get_total_results(status="success") == 7 - assert result_manager.get_total_results(status="failure") == 19 - assert result_manager.get_total_results(status="error") == 2 - assert result_manager.get_total_results(status="skipped") == 2 + assert result_manager.get_total_results(status={"success"}) == 7 + assert result_manager.get_total_results(status={"failure"}) == 19 + assert result_manager.get_total_results(status={"error"}) == 2 + assert result_manager.get_total_results(status={"skipped"}) == 2 # Test multiple statuses assert result_manager.get_total_results(status={"success", "failure"}) == 26