From 78dd0eaf2115368f7f3bd445ef60ead5df63309c Mon Sep 17 00:00:00 2001 From: David Liu Date: Wed, 1 May 2024 23:34:42 -0400 Subject: [PATCH] Ensure Python tester omits skipped tests --- Changelog.md | 1 + .../autotest_server/testers/py/py_tester.py | 2 +- server/autotest_server/testers/tester.py | 2 +- .../autotest_server/tests/testers/__init__.py | 0 .../tests/testers/py/__init__.py | 0 .../testers/py/fixtures/sample_tests_skip.py | 10 +++ .../py/fixtures/sample_tests_success.py | 6 ++ .../tests/testers/py/test_py_tester.py | 63 +++++++++++++++++++ 8 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 server/autotest_server/tests/testers/__init__.py create mode 100644 server/autotest_server/tests/testers/py/__init__.py create mode 100644 server/autotest_server/tests/testers/py/fixtures/sample_tests_skip.py create mode 100644 server/autotest_server/tests/testers/py/fixtures/sample_tests_success.py create mode 100644 server/autotest_server/tests/testers/py/test_py_tester.py diff --git a/Changelog.md b/Changelog.md index d746537e..8bf8e3ab 100644 --- a/Changelog.md +++ b/Changelog.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented here. ## [unreleased] - Add tidyverse as a default R tester package (#512) +- Omit skipped test cases in Python tester (#522) ## [v2.4.2] - Ensure _env_status is updated to "setup" earlier when a request to update test settings is made (#499) diff --git a/server/autotest_server/testers/py/py_tester.py b/server/autotest_server/testers/py/py_tester.py index d6777577..88c57129 100644 --- a/server/autotest_server/testers/py/py_tester.py +++ b/server/autotest_server/testers/py/py_tester.py @@ -89,7 +89,7 @@ def pytest_runtest_makereport(self, item, call): """ outcome = yield rep = outcome.get_result() - if rep.failed or item.nodeid not in self.results: + if rep.failed or (item.nodeid not in self.results and not rep.skipped and rep.when != "teardown"): self.results[item.nodeid] = { "status": "failure" if rep.failed else "success", "name": item.nodeid, diff --git a/server/autotest_server/testers/tester.py b/server/autotest_server/testers/tester.py index 6658d68f..abe555b2 100644 --- a/server/autotest_server/testers/tester.py +++ b/server/autotest_server/testers/tester.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from functools import wraps from typing import Optional, Callable, Any, Type, Dict, List -from testers.specs import TestSpecs +from .specs import TestSpecs import traceback diff --git a/server/autotest_server/tests/testers/__init__.py b/server/autotest_server/tests/testers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/autotest_server/tests/testers/py/__init__.py b/server/autotest_server/tests/testers/py/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/autotest_server/tests/testers/py/fixtures/sample_tests_skip.py b/server/autotest_server/tests/testers/py/fixtures/sample_tests_skip.py new file mode 100644 index 00000000..2f18e2d2 --- /dev/null +++ b/server/autotest_server/tests/testers/py/fixtures/sample_tests_skip.py @@ -0,0 +1,10 @@ +import pytest + + +def add_one(x): + return x + 1 + + +@pytest.mark.skip +def test_add_one(): + assert add_one(1) == 2 diff --git a/server/autotest_server/tests/testers/py/fixtures/sample_tests_success.py b/server/autotest_server/tests/testers/py/fixtures/sample_tests_success.py new file mode 100644 index 00000000..eb62e43a --- /dev/null +++ b/server/autotest_server/tests/testers/py/fixtures/sample_tests_success.py @@ -0,0 +1,6 @@ +def add_one(x): + return x + 1 + + +def test_add_one(): + assert add_one(1) == 2 diff --git a/server/autotest_server/tests/testers/py/test_py_tester.py b/server/autotest_server/tests/testers/py/test_py_tester.py new file mode 100644 index 00000000..a111378f --- /dev/null +++ b/server/autotest_server/tests/testers/py/test_py_tester.py @@ -0,0 +1,63 @@ +from ....testers.specs import TestSpecs +from ....testers.py.py_tester import PyTester + + +def test_success(request, monkeypatch) -> None: + """Test that when a test succeeds, it is added to the results.""" + monkeypatch.chdir(request.fspath.dirname) + tester = PyTester( + specs=TestSpecs.from_json( + """ + { + "test_data": { + "script_files": ["fixtures/sample_tests_success.py"], + "category": ["instructor"], + "timeout": 30, + "tester": "pytest", + "output_verbosity": "short", + "extra_info": { + "criterion": "", + "name": "Python Test Group 1" + } + } + } + """ + ) + ) + results = tester.run_python_tests() + assert len(results) == 1 + assert "fixtures/sample_tests_success.py" in results + assert len(results["fixtures/sample_tests_success.py"]) == 1 + + result = results["fixtures/sample_tests_success.py"][0] + assert result["status"] == "success" + # nodeid is inexact in CI test + assert result["name"].endswith("fixtures/sample_tests_success.py::test_add_one") + assert result["errors"] == "" + assert result["description"] is None + + +def test_skip(request, monkeypatch) -> None: + """Test that when a test is skipped, it is omitted from the results.""" + monkeypatch.chdir(request.fspath.dirname) + tester = PyTester( + specs=TestSpecs.from_json( + """ + { + "test_data": { + "script_files": ["fixtures/sample_tests_skip.py"], + "category": ["instructor"], + "timeout": 30, + "tester": "pytest", + "output_verbosity": "short", + "extra_info": { + "criterion": "", + "name": "Python Test Group 1" + } + } + } + """ + ) + ) + results = tester.run_python_tests() + assert results == {"fixtures/sample_tests_skip.py": []}