From 03fcdd473ed689d83890ef022f6980926dc30bbe Mon Sep 17 00:00:00 2001 From: donny-wong <141858744+donny-wong@users.noreply.github.com> Date: Tue, 9 Apr 2024 16:49:17 -0400 Subject: [PATCH 1/3] Sticky bit unit test (#501) --------- Co-authored-by: Donny Wong Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../tests/test_autotest_server.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/server/autotest_server/tests/test_autotest_server.py b/server/autotest_server/tests/test_autotest_server.py index bc3be969..f2a61b61 100644 --- a/server/autotest_server/tests/test_autotest_server.py +++ b/server/autotest_server/tests/test_autotest_server.py @@ -1,7 +1,10 @@ +import subprocess + import pytest import fakeredis import rq import autotest_server +import os @pytest.fixture @@ -26,3 +29,22 @@ def fake_redis_db(monkeypatch, fake_job): def test_redis_connection(fake_redis_conn): assert autotest_server.redis_connection() == fake_redis_conn + + +def test_sticky(): + workers = autotest_server.config["workers"] + autotest_worker = workers[0]["user"] + autotest_worker_working_dir = f"/home/docker/.autotesting/workers/{autotest_worker}" + path = f"{autotest_worker_working_dir}/test_sticky" + + if not os.path.exists(path): + mkdir_cmd = f"sudo -u {autotest_worker} mkdir {path}" + chmod_cmd = f"sudo -u {autotest_worker} chmod 000 {path}" + chmod_sticky_cmd = f"sudo -u {autotest_worker} chmod +t {path}" + subprocess.run(mkdir_cmd, shell=True) + subprocess.run(chmod_cmd, shell=True) + subprocess.run(chmod_sticky_cmd, shell=True) + + autotest_server._clear_working_directory(autotest_worker_working_dir, autotest_worker) + + assert os.path.exists(path) is False From 39ca6da9bf94b58aca422f84bfa9cd602d792a60 Mon Sep 17 00:00:00 2001 From: Samuel Maldonado Date: Wed, 24 Apr 2024 15:41:23 -0400 Subject: [PATCH 2/3] Clean local directory before setting up test files (#510) --- server/autotest_server/__init__.py | 1 + .../tests/test_autotest_server.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/server/autotest_server/__init__.py b/server/autotest_server/__init__.py index 88524c26..217421e3 100644 --- a/server/autotest_server/__init__.py +++ b/server/autotest_server/__init__.py @@ -343,6 +343,7 @@ def run_test(settings_id, test_id, files_url, categories, user, test_env_vars): redis_connection().hset("autotest:settings", key=settings_id, value=json.dumps(settings)) test_username, tests_path = tester_user() try: + _clear_working_directory(tests_path, test_username) _setup_files(settings_id, user, files_url, tests_path, test_username) cmd = run_test_command(test_username=test_username) results = _run_test_specs(cmd, settings, categories, tests_path, test_username, test_id, test_env_vars) diff --git a/server/autotest_server/tests/test_autotest_server.py b/server/autotest_server/tests/test_autotest_server.py index f2a61b61..0cf74b38 100644 --- a/server/autotest_server/tests/test_autotest_server.py +++ b/server/autotest_server/tests/test_autotest_server.py @@ -48,3 +48,20 @@ def test_sticky(): autotest_server._clear_working_directory(autotest_worker_working_dir, autotest_worker) assert os.path.exists(path) is False + + +def test_pre_remove(): + workers = autotest_server.config["workers"] + autotest_worker = workers[0]["user"] + autotest_worker_working_dir = f"/home/docker/.autotesting/workers/{autotest_worker}" + path = f"{autotest_worker_working_dir}/__pycache__" + + if not os.path.exists(path): + mkdir_cmd = f"sudo -u {autotest_worker} mkdir {path}" + chmod_cmd = f"sudo -u {autotest_worker} chmod 000 {path}" + subprocess.run(mkdir_cmd, shell=True) + subprocess.run(chmod_cmd, shell=True) + + autotest_server._clear_working_directory(autotest_worker_working_dir, autotest_worker) + + assert os.path.exists(path) is False From b89381339e0808506ea96ffaf97e68c0776cd5d2 Mon Sep 17 00:00:00 2001 From: David Liu Date: Wed, 8 May 2024 18:01:53 +0000 Subject: [PATCH 3/3] Ensure Python tester omits skipped tests (#522) --- Changelog.md | 3 + .../autotest_server/testers/py/py_tester.py | 2 +- server/autotest_server/testers/tester.py | 2 +- .../autotest_server/tests/testers/__init__.py | 0 .../tests/testers/py/__init__.py | 0 .../testers/py/fixtures/sample_tests_skip.py | 10 +++ .../py/fixtures/sample_tests_success.py | 6 ++ .../tests/testers/py/test_py_tester.py | 63 +++++++++++++++++++ 8 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 server/autotest_server/tests/testers/__init__.py create mode 100644 server/autotest_server/tests/testers/py/__init__.py create mode 100644 server/autotest_server/tests/testers/py/fixtures/sample_tests_skip.py create mode 100644 server/autotest_server/tests/testers/py/fixtures/sample_tests_success.py create mode 100644 server/autotest_server/tests/testers/py/test_py_tester.py diff --git a/Changelog.md b/Changelog.md index de8bbbf4..95426230 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,9 @@ # CHANGELOG All notable changes to this project will be documented here. +## [v2.4.3] +- Omit skipped test cases in Python tester (#522) + ## [v2.4.2] - Ensure _env_status is updated to "setup" earlier when a request to update test settings is made (#499) diff --git a/server/autotest_server/testers/py/py_tester.py b/server/autotest_server/testers/py/py_tester.py index d6777577..88c57129 100644 --- a/server/autotest_server/testers/py/py_tester.py +++ b/server/autotest_server/testers/py/py_tester.py @@ -89,7 +89,7 @@ def pytest_runtest_makereport(self, item, call): """ outcome = yield rep = outcome.get_result() - if rep.failed or item.nodeid not in self.results: + if rep.failed or (item.nodeid not in self.results and not rep.skipped and rep.when != "teardown"): self.results[item.nodeid] = { "status": "failure" if rep.failed else "success", "name": item.nodeid, diff --git a/server/autotest_server/testers/tester.py b/server/autotest_server/testers/tester.py index 6658d68f..abe555b2 100644 --- a/server/autotest_server/testers/tester.py +++ b/server/autotest_server/testers/tester.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from functools import wraps from typing import Optional, Callable, Any, Type, Dict, List -from testers.specs import TestSpecs +from .specs import TestSpecs import traceback diff --git a/server/autotest_server/tests/testers/__init__.py b/server/autotest_server/tests/testers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/autotest_server/tests/testers/py/__init__.py b/server/autotest_server/tests/testers/py/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/autotest_server/tests/testers/py/fixtures/sample_tests_skip.py b/server/autotest_server/tests/testers/py/fixtures/sample_tests_skip.py new file mode 100644 index 00000000..2f18e2d2 --- /dev/null +++ b/server/autotest_server/tests/testers/py/fixtures/sample_tests_skip.py @@ -0,0 +1,10 @@ +import pytest + + +def add_one(x): + return x + 1 + + +@pytest.mark.skip +def test_add_one(): + assert add_one(1) == 2 diff --git a/server/autotest_server/tests/testers/py/fixtures/sample_tests_success.py b/server/autotest_server/tests/testers/py/fixtures/sample_tests_success.py new file mode 100644 index 00000000..eb62e43a --- /dev/null +++ b/server/autotest_server/tests/testers/py/fixtures/sample_tests_success.py @@ -0,0 +1,6 @@ +def add_one(x): + return x + 1 + + +def test_add_one(): + assert add_one(1) == 2 diff --git a/server/autotest_server/tests/testers/py/test_py_tester.py b/server/autotest_server/tests/testers/py/test_py_tester.py new file mode 100644 index 00000000..a111378f --- /dev/null +++ b/server/autotest_server/tests/testers/py/test_py_tester.py @@ -0,0 +1,63 @@ +from ....testers.specs import TestSpecs +from ....testers.py.py_tester import PyTester + + +def test_success(request, monkeypatch) -> None: + """Test that when a test succeeds, it is added to the results.""" + monkeypatch.chdir(request.fspath.dirname) + tester = PyTester( + specs=TestSpecs.from_json( + """ + { + "test_data": { + "script_files": ["fixtures/sample_tests_success.py"], + "category": ["instructor"], + "timeout": 30, + "tester": "pytest", + "output_verbosity": "short", + "extra_info": { + "criterion": "", + "name": "Python Test Group 1" + } + } + } + """ + ) + ) + results = tester.run_python_tests() + assert len(results) == 1 + assert "fixtures/sample_tests_success.py" in results + assert len(results["fixtures/sample_tests_success.py"]) == 1 + + result = results["fixtures/sample_tests_success.py"][0] + assert result["status"] == "success" + # nodeid is inexact in CI test + assert result["name"].endswith("fixtures/sample_tests_success.py::test_add_one") + assert result["errors"] == "" + assert result["description"] is None + + +def test_skip(request, monkeypatch) -> None: + """Test that when a test is skipped, it is omitted from the results.""" + monkeypatch.chdir(request.fspath.dirname) + tester = PyTester( + specs=TestSpecs.from_json( + """ + { + "test_data": { + "script_files": ["fixtures/sample_tests_skip.py"], + "category": ["instructor"], + "timeout": 30, + "tester": "pytest", + "output_verbosity": "short", + "extra_info": { + "criterion": "", + "name": "Python Test Group 1" + } + } + } + """ + ) + ) + results = tester.run_python_tests() + assert results == {"fixtures/sample_tests_skip.py": []}