Ensure Python tester omits skipped tests

Changelog.md

@@ -3,6 +3,7 @@ All notable changes to this project will be documented here.
 
 ## [unreleased]
 - Add tidyverse as a default R tester package (#512)
+- Omit skipped test cases in Python tester (#522)
 
 ## [v2.4.2]
 - Ensure _env_status is updated to "setup" earlier when a request to update test settings is made (#499)

server/autotest_server/testers/py/py_tester.py

@@ -89,7 +89,7 @@ def pytest_runtest_makereport(self, item, call):
         """
         outcome = yield
         rep = outcome.get_result()
-        if rep.failed or item.nodeid not in self.results:
+        if rep.failed or (item.nodeid not in self.results and not rep.skipped and rep.when != "teardown"):
             self.results[item.nodeid] = {
                 "status": "failure" if rep.failed else "success",
                 "name": item.nodeid,

server/autotest_server/testers/tester.py

@@ -2,7 +2,7 @@
 from abc import ABC, abstractmethod
 from functools import wraps
 from typing import Optional, Callable, Any, Type, Dict, List
-from testers.specs import TestSpecs
+from .specs import TestSpecs
 import traceback
 
 

server/autotest_server/tests/testers/py/fixtures/sample_tests_skip.py

@@ -0,0 +1,10 @@
+import pytest
+
+
+def add_one(x):
+    return x + 1
+
+
+@pytest.mark.skip
+def test_add_one():
+    assert add_one(1) == 2

server/autotest_server/tests/testers/py/fixtures/sample_tests_success.py

@@ -0,0 +1,6 @@
+def add_one(x):
+    return x + 1
+
+
+def test_add_one():
+    assert add_one(1) == 2

server/autotest_server/tests/testers/py/test_py_tester.py

@@ -0,0 +1,63 @@
+from ....testers.specs import TestSpecs
+from ....testers.py.py_tester import PyTester
+
+
+def test_success(request, monkeypatch) -> None:
+    """Test that when a test succeeds, it is added to the results."""
+    monkeypatch.chdir(request.fspath.dirname)
+    tester = PyTester(
+        specs=TestSpecs.from_json(
+            """
+        {
+          "test_data": {
+            "script_files": ["fixtures/sample_tests_success.py"],
+            "category": ["instructor"],
+            "timeout": 30,
+            "tester": "pytest",
+            "output_verbosity": "short",
+            "extra_info": {
+              "criterion": "",
+              "name": "Python Test Group 1"
+            }
+          }
+        }
+    """
+        )
+    )
+    results = tester.run_python_tests()
+    assert len(results) == 1
+    assert "fixtures/sample_tests_success.py" in results
+    assert len(results["fixtures/sample_tests_success.py"]) == 1
+
+    result = results["fixtures/sample_tests_success.py"][0]
+    assert result["status"] == "success"
+    # nodeid is inexact in CI test
+    assert result["name"].endswith("fixtures/sample_tests_success.py::test_add_one")
+    assert result["errors"] == ""
+    assert result["description"] is None
+
+
+def test_skip(request, monkeypatch) -> None:
+    """Test that when a test is skipped, it is omitted from the results."""
+    monkeypatch.chdir(request.fspath.dirname)
+    tester = PyTester(
+        specs=TestSpecs.from_json(
+            """
+        {
+          "test_data": {
+            "script_files": ["fixtures/sample_tests_skip.py"],
+            "category": ["instructor"],
+            "timeout": 30,
+            "tester": "pytest",
+            "output_verbosity": "short",
+            "extra_info": {
+              "criterion": "",
+              "name": "Python Test Group 1"
+            }
+          }
+        }
+    """
+        )
+    )
+    results = tester.run_python_tests()
+    assert results == {"fixtures/sample_tests_skip.py": []}