From 174f2fd0a532d36e3f73727a763eaeecd57e32e5 Mon Sep 17 00:00:00 2001 From: Samuel Maldonado Date: Fri, 31 May 2024 18:48:46 -0400 Subject: [PATCH 1/7] clear files created in tmp after test runs --- server/autotest_server/__init__.py | 5 +++ .../tests/test_autotest_server.py | 38 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/server/autotest_server/__init__.py b/server/autotest_server/__init__.py index 217421e3..9efb8db7 100644 --- a/server/autotest_server/__init__.py +++ b/server/autotest_server/__init__.py @@ -257,9 +257,13 @@ def _clear_working_directory(tests_path: str, test_username: str) -> None: if test_username != getpass.getuser(): sticky_cmd = f"sudo -u {test_username} -- bash -c 'chmod -Rf -t {tests_path}'" chmod_cmd = f"sudo -u {test_username} -- bash -c 'chmod -Rf ugo+rwX {tests_path}'" + clean_tmp_cmd = ( + f"sudo -u {test_username} -- bash -c 'find /tmp -maxdepth 1 -user {test_username} -exec rm " f"-rf {{}} +' " + ) else: sticky_cmd = f"chmod -Rf -t {tests_path}" chmod_cmd = f"chmod -Rf ugo+rwX {tests_path}" + clean_tmp_cmd = f"find /tmp -maxdepth 1 -user {test_username} -exec rm -rf {{}} +" subprocess.run(sticky_cmd, shell=True) subprocess.run(chmod_cmd, shell=True) @@ -268,6 +272,7 @@ def _clear_working_directory(tests_path: str, test_username: str) -> None: # set the group ownership with sudo (and that is only done in ../install.sh) clean_cmd = f"rm -rf {tests_path}/.[!.]* {tests_path}/*" subprocess.run(clean_cmd, shell=True) + subprocess.run(clean_tmp_cmd, shell=True) def _stop_tester_processes(test_username: str) -> None: diff --git a/server/autotest_server/tests/test_autotest_server.py b/server/autotest_server/tests/test_autotest_server.py index 0cf74b38..260767dd 100644 --- a/server/autotest_server/tests/test_autotest_server.py +++ b/server/autotest_server/tests/test_autotest_server.py @@ -65,3 +65,41 @@ def test_pre_remove(): autotest_server._clear_working_directory(autotest_worker_working_dir, autotest_worker) assert os.path.exists(path) is False + + +def test_tmp_remove_file(): + workers = autotest_server.config["workers"] + autotest_worker = workers[0]["user"] + autotest_worker_working_dir = f"/home/docker/.autotesting/workers/{autotest_worker}" + path = "/tmp/test.txt" + touch_cmd = f"sudo -u {autotest_worker} touch {path}" + subprocess.run(touch_cmd, shell=True) + autotest_server._clear_working_directory(autotest_worker_working_dir, autotest_worker) + assert os.path.exists(path) is False + + +def test_tmp_remove_dir(): + workers = autotest_server.config["workers"] + autotest_worker = workers[0]["user"] + autotest_worker_working_dir = f"/home/docker/.autotesting/workers/{autotest_worker}" + path = "/tmp/folder" + mkdir_cmd = f"sudo -u {autotest_worker} mkdir {path}" + subprocess.run(mkdir_cmd, shell=True) + touch_cmd = f"sudo -u {autotest_worker} touch {path}/test.txt" + subprocess.run(touch_cmd, shell=True) + autotest_server._clear_working_directory(autotest_worker_working_dir, autotest_worker) + assert os.path.exists(path) is False + + +def test_tmp_remove_other_user(): + workers = autotest_server.config["workers"] + autotest_worker_creator = workers[0]["user"] + autotest_worker_remover = workers[1]["user"] + autotest_worker_working_dir = f"/home/docker/.autotesting/workers/{autotest_worker_remover}" + path = "/tmp/folder" + mkdir_cmd = f"sudo -u {autotest_worker_creator} mkdir {path}" + subprocess.run(mkdir_cmd, shell=True) + touch_cmd = f"sudo -u {autotest_worker_creator} touch {path}/test.txt" + subprocess.run(touch_cmd, shell=True) + autotest_server._clear_working_directory(autotest_worker_working_dir, autotest_worker_remover) + assert os.path.exists(path) is True From 2eab2818e746399f8eb0d8836c4522b1e6069cdc Mon Sep 17 00:00:00 2001 From: Samuel Maldonado Date: Fri, 31 May 2024 18:54:54 -0400 Subject: [PATCH 2/7] Changelog: update changelog --- Changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog.md b/Changelog.md index 7248b0be..b3223191 100644 --- a/Changelog.md +++ b/Changelog.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented here. ## [unreleased] - Add tidyverse as a default R tester package (#512) +- Clean up tmp directory after test runs (#528) ## [v2.4.3] - Omit skipped test cases in Python tester (#522) From 863f64eddc26d2182bda389ceb49ad9f4d75c1fb Mon Sep 17 00:00:00 2001 From: Samuel Maldonado Date: Fri, 31 May 2024 18:59:34 -0400 Subject: [PATCH 3/7] update test settings to add another user --- server/autotest_server/tests/fixtures/test_config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/autotest_server/tests/fixtures/test_config.yml b/server/autotest_server/tests/fixtures/test_config.yml index 3b651714..1b731323 100644 --- a/server/autotest_server/tests/fixtures/test_config.yml +++ b/server/autotest_server/tests/fixtures/test_config.yml @@ -3,7 +3,9 @@ redis_url: fake_url supervisor_url: fake_url workers: - user: fake_user - queues: + queues: &queues - high - low - batch + - user: fake_user_2 + queues: *queues From 85ee49d43bdbb11ee35e50b551de696c39794ddc Mon Sep 17 00:00:00 2001 From: Samuel Maldonado Date: Fri, 31 May 2024 19:29:00 -0400 Subject: [PATCH 4/7] add users in test config --- .github/workflows/test_ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 42e12de1..ade16ff9 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -36,5 +36,9 @@ jobs: ${{ runner.os }}-pip- - name: Install python packages run: python -m pip install pytest fakeredis typing-extensions -r ${{ matrix.test-dir }}/requirements.txt + - name: Create users + run: | + adduser --disabled-login --no-create-home fake_user && usermod -aG fake_user docker + adduser --disabled-login --no-create-home fake_user_2 && usermod -aG fake_user_2 docker - name: run tests run: pytest ${{ matrix.test-dir }} From 20646262d31242039d959d7198625d7b309bfe9f Mon Sep 17 00:00:00 2001 From: Samuel Maldonado Date: Tue, 4 Jun 2024 12:00:03 -0400 Subject: [PATCH 5/7] use sudo to add users --- .github/workflows/test_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index ade16ff9..257dd47c 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -38,7 +38,7 @@ jobs: run: python -m pip install pytest fakeredis typing-extensions -r ${{ matrix.test-dir }}/requirements.txt - name: Create users run: | - adduser --disabled-login --no-create-home fake_user && usermod -aG fake_user docker - adduser --disabled-login --no-create-home fake_user_2 && usermod -aG fake_user_2 docker + sudo adduser --disabled-login --no-create-home fake_user && usermod -aG fake_user docker + sudo adduser --disabled-login --no-create-home fake_user_2 && usermod -aG fake_user_2 docker - name: run tests run: pytest ${{ matrix.test-dir }} From dfc77a9e2c6eaad603c0475096242d7cd07231a8 Mon Sep 17 00:00:00 2001 From: Samuel Maldonado Date: Tue, 4 Jun 2024 13:25:27 -0400 Subject: [PATCH 6/7] do not attempt to add users to nonexistent group --- .github/workflows/test_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 257dd47c..45a09496 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -38,7 +38,7 @@ jobs: run: python -m pip install pytest fakeredis typing-extensions -r ${{ matrix.test-dir }}/requirements.txt - name: Create users run: | - sudo adduser --disabled-login --no-create-home fake_user && usermod -aG fake_user docker - sudo adduser --disabled-login --no-create-home fake_user_2 && usermod -aG fake_user_2 docker + sudo adduser --disabled-login --no-create-home fake_user + sudo adduser --disabled-login --no-create-home fake_user_2 - name: run tests run: pytest ${{ matrix.test-dir }} From c606c9d5f0b76d08d68d0a11ac7a7af93668fc7d Mon Sep 17 00:00:00 2001 From: Samuel Maldonado Date: Mon, 10 Jun 2024 12:27:20 -0400 Subject: [PATCH 7/7] update docstring based on PR feedback --- server/autotest_server/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/autotest_server/__init__.py b/server/autotest_server/__init__.py index 9efb8db7..a44d8d98 100644 --- a/server/autotest_server/__init__.py +++ b/server/autotest_server/__init__.py @@ -252,7 +252,8 @@ def _run_test_specs( def _clear_working_directory(tests_path: str, test_username: str) -> None: """ - Run commands that clear the tests_path working directory + Run commands that clear the tests_path working directory, as well + as clearing any files or directories owned by test_username in the /tmp directory """ if test_username != getpass.getuser(): sticky_cmd = f"sudo -u {test_username} -- bash -c 'chmod -Rf -t {tests_path}'"