From 13139ec2b2db235090f67ca0d9057e6bc9075f55 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Mon, 30 Dec 2024 06:52:50 +0530 Subject: [PATCH 01/34] Improve error message when forced fail test fails --- mutmut/__main__.py | 8 +++--- tests/test_mutation.py | 60 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index f9097da6..a8f43e3d 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -885,14 +885,14 @@ def print_stats(source_file_mutation_data_by_path, force_output=False): print_status(f'{(s.total - s.not_checked)}/{s.total} 🎉 {s.killed} 🫥 {s.no_tests} ⏰ {s.timeout} 🤔 {s.suspicious} 🙁 {s.survived} 🔇 {s.skipped}', force_output=force_output) -def run_forced_fail(runner): +def run_forced_fail_test(runner): os.environ['MUTANT_UNDER_TEST'] = 'fail' with CatchOutput(spinner_title='Running forced fail test') as catcher: try: if runner.run_forced_fail() == 0: catcher.dump_output() - print("FAILED") - os._exit(1) + print("Error: Unable to force a test failure during the forced fail test") + raise SystemExit(1) except MutmutProgrammaticFailException: pass os.environ['MUTANT_UNDER_TEST'] = '' @@ -1254,7 +1254,7 @@ def run(mutant_names, *, max_children): print(' done') # this can't be the first thing, because it can fail deep inside pytest/django setup and then everything is destroyed - run_forced_fail(runner) + run_forced_fail_test(runner) runner.prepare_main_test_run() diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 3abc6fb5..4d7ca2cd 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -1,8 +1,11 @@ +import os from io import StringIO +from unittest.mock import Mock import pytest from parso import parse +import mutmut from mutmut.__main__ import ( CLASS_NAME_SEPARATOR, FuncContext, @@ -13,7 +16,7 @@ pragma_no_mutate_lines, write_all_mutants_to_file, yield_mutants_for_module, - yield_mutants_for_node, + yield_mutants_for_node, run_forced_fail_test, Config, MutmutProgrammaticFailException, ) @@ -407,3 +410,58 @@ def bar(): # # mutants = mutants_for_source(source) # assert len(mutants) == 1 + + +def test_run_forced_fail_test_with_failing_test(capfd): + mutmut.config = _default_mutmut_config() + runner = _mocked_runner_run_forced_failed(return_value=1) + + run_forced_fail_test(runner) + + out, err = capfd.readouterr() + assert 'Running forced fail test' in out + assert 'done' in out + assert os.environ['MUTANT_UNDER_TEST'] is '' + + +def test_run_forced_fail_test_with_mutmut_programmatic_fail_exception(capfd): + mutmut.config = _default_mutmut_config() + runner = _mocked_runner_run_forced_failed(side_effect=MutmutProgrammaticFailException()) + + run_forced_fail_test(runner) + + out, err = capfd.readouterr() + assert 'Running forced fail test' in out + assert 'done' in out + assert os.environ['MUTANT_UNDER_TEST'] is '' + + +def test_run_forced_fail_test_with_all_tests_passing(capfd): + mutmut.config = _default_mutmut_config() + runner = _mocked_runner_run_forced_failed(return_value=0) + + with pytest.raises(SystemExit) as error: + run_forced_fail_test(runner) + + assert error.value.code is 1 + out, err = capfd.readouterr() + assert 'Running forced fail test' in out + assert 'Error: Unable to force a test failure during the forced fail test' in out + + +def _default_mutmut_config(): + return Config( + do_not_mutate=[], + also_copy=[], + max_stack_depth=-1, + debug=False, + paths_to_mutate=[] + ) + +def _mocked_runner_run_forced_failed(return_value=None, side_effect=None): + runner = Mock() + runner.run_forced_fail = Mock( + return_value=return_value, + side_effect=side_effect + ) + return runner From 587f7d3e27bfd4756cea23c508ed3572c09ed5e7 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Mon, 30 Dec 2024 13:31:06 +0530 Subject: [PATCH 02/34] Update error message with suggestion --- mutmut/__main__.py | 2 +- tests/test_mutation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index a8f43e3d..e7ccab3e 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -891,7 +891,7 @@ def run_forced_fail_test(runner): try: if runner.run_forced_fail() == 0: catcher.dump_output() - print("Error: Unable to force a test failure during the forced fail test") + print("FAILED: Unable to force test failures") raise SystemExit(1) except MutmutProgrammaticFailException: pass diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 4d7ca2cd..ef327b03 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -446,7 +446,7 @@ def test_run_forced_fail_test_with_all_tests_passing(capfd): assert error.value.code is 1 out, err = capfd.readouterr() assert 'Running forced fail test' in out - assert 'Error: Unable to force a test failure during the forced fail test' in out + assert 'FAILED: Unable to force test failures' in out def _default_mutmut_config(): From 70cbf8025d308d60268449e5fb7240c394bfee65 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:37:58 +0530 Subject: [PATCH 03/34] Add some sleeps and debug output to rule out race condition for GitHub Actions test failure --- mutmut/__main__.py | 3 ++- tests/test_mutation.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index e7ccab3e..920d762c 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -891,7 +891,8 @@ def run_forced_fail_test(runner): try: if runner.run_forced_fail() == 0: catcher.dump_output() - print("FAILED: Unable to force test failures") + print("\nFAILED: Unable to force test failures") + sleep(2) raise SystemExit(1) except MutmutProgrammaticFailException: pass diff --git a/tests/test_mutation.py b/tests/test_mutation.py index ef327b03..681fd04a 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -1,5 +1,6 @@ import os from io import StringIO +from time import sleep from unittest.mock import Mock import pytest @@ -419,6 +420,11 @@ def test_run_forced_fail_test_with_failing_test(capfd): run_forced_fail_test(runner) out, err = capfd.readouterr() + print(f"out: {out}") + print(f"err: {err}") + sleep(2) + print(f"out: {out}") + print(f"err: {err}") assert 'Running forced fail test' in out assert 'done' in out assert os.environ['MUTANT_UNDER_TEST'] is '' @@ -431,6 +437,11 @@ def test_run_forced_fail_test_with_mutmut_programmatic_fail_exception(capfd): run_forced_fail_test(runner) out, err = capfd.readouterr() + print(f"out: {out}") + print(f"err: {err}") + sleep(2) + print(f"out: {out}") + print(f"err: {err}") assert 'Running forced fail test' in out assert 'done' in out assert os.environ['MUTANT_UNDER_TEST'] is '' @@ -445,6 +456,11 @@ def test_run_forced_fail_test_with_all_tests_passing(capfd): assert error.value.code is 1 out, err = capfd.readouterr() + print(f"out: {out}") + print(f"err: {err}") + sleep(2) + print(f"out: {out}") + print(f"err: {err}") assert 'Running forced fail test' in out assert 'FAILED: Unable to force test failures' in out From c08280d1b979648c0924f7d85c73d722c5bd60a8 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:49:54 +0530 Subject: [PATCH 04/34] Try running on MacOS in case that's the difference --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 02b3def5..0172bd97 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: tests: name: Tests - runs-on: ubuntu-latest + runs-on: macos-14 steps: - name: Checkout uses: actions/checkout@v3 From 615f592afecf1986d13e6a90fc64e50661022d4e Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:59:13 +0530 Subject: [PATCH 05/34] Revert "Add some sleeps and debug output to rule out race condition for GitHub Actions test failure" This reverts commit 70cbf8025d308d60268449e5fb7240c394bfee65. --- mutmut/__main__.py | 3 +-- tests/test_mutation.py | 16 ---------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 920d762c..e7ccab3e 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -891,8 +891,7 @@ def run_forced_fail_test(runner): try: if runner.run_forced_fail() == 0: catcher.dump_output() - print("\nFAILED: Unable to force test failures") - sleep(2) + print("FAILED: Unable to force test failures") raise SystemExit(1) except MutmutProgrammaticFailException: pass diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 681fd04a..ef327b03 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -1,6 +1,5 @@ import os from io import StringIO -from time import sleep from unittest.mock import Mock import pytest @@ -420,11 +419,6 @@ def test_run_forced_fail_test_with_failing_test(capfd): run_forced_fail_test(runner) out, err = capfd.readouterr() - print(f"out: {out}") - print(f"err: {err}") - sleep(2) - print(f"out: {out}") - print(f"err: {err}") assert 'Running forced fail test' in out assert 'done' in out assert os.environ['MUTANT_UNDER_TEST'] is '' @@ -437,11 +431,6 @@ def test_run_forced_fail_test_with_mutmut_programmatic_fail_exception(capfd): run_forced_fail_test(runner) out, err = capfd.readouterr() - print(f"out: {out}") - print(f"err: {err}") - sleep(2) - print(f"out: {out}") - print(f"err: {err}") assert 'Running forced fail test' in out assert 'done' in out assert os.environ['MUTANT_UNDER_TEST'] is '' @@ -456,11 +445,6 @@ def test_run_forced_fail_test_with_all_tests_passing(capfd): assert error.value.code is 1 out, err = capfd.readouterr() - print(f"out: {out}") - print(f"err: {err}") - sleep(2) - print(f"out: {out}") - print(f"err: {err}") assert 'Running forced fail test' in out assert 'FAILED: Unable to force test failures' in out From 2c3113d33e9e77f65b7317a290e9d1bb31972416 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:59:19 +0530 Subject: [PATCH 06/34] Revert "Try running on MacOS in case that's the difference" This reverts commit c08280d1b979648c0924f7d85c73d722c5bd60a8. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0172bd97..02b3def5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: tests: name: Tests - runs-on: macos-14 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 From 5c60640e5866eaa670324f29ea9b13e7e11cc65b Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Mon, 13 Jan 2025 07:43:01 +0530 Subject: [PATCH 07/34] Attempt to see actual output in GitHub Actions --- .github/workflows/tests.yml | 3 ++- tests/test_mutation.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 41637cf5..190eeb10 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,5 +27,6 @@ jobs: pip install -r test_requirements.txt - name: Run tests - run: python -m pytest + # run: python -m pytest + run: python -m pytest -vvv --capture=no tests/test_mutation.py::test_run_forced_fail_test_with_failing_test_TEMP continue-on-error: false diff --git a/tests/test_mutation.py b/tests/test_mutation.py index ef327b03..837cc0f0 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -412,6 +412,20 @@ def bar(): # assert len(mutants) == 1 +def test_run_forced_fail_test_with_failing_test_TEMP(): + print("temp_test_run_forced_fail_test_with_failing_test - start") + mutmut.config = _default_mutmut_config() + runner = _mocked_runner_run_forced_failed(return_value=1) + + run_forced_fail_test(runner) + + # out, err = capfd.readouterr() + # assert 'Running forced fail test' in out + # assert 'done' in out + assert os.environ['MUTANT_UNDER_TEST'] is '' + print("temp_test_run_forced_fail_test_with_failing_test - end") + + def test_run_forced_fail_test_with_failing_test(capfd): mutmut.config = _default_mutmut_config() runner = _mocked_runner_run_forced_failed(return_value=1) From 7d7c8b9e0ae0ec85148813abb6a94e0ae38262b0 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Mon, 13 Jan 2025 07:45:16 +0530 Subject: [PATCH 08/34] More GitHub Actions debug output --- .github/workflows/tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 190eeb10..ee117ba7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,12 @@ jobs: pip install -r requirements.txt pip install -r test_requirements.txt - - name: Run tests + - name: Run tests TEMP # run: python -m pytest run: python -m pytest -vvv --capture=no tests/test_mutation.py::test_run_forced_fail_test_with_failing_test_TEMP continue-on-error: false + + - name: Run tests + # run: python -m pytest + run: python -m pytest -vvv --capture=no tests/test_mutation.py::test_run_forced_fail_test_with_failing_test + continue-on-error: false From 4d3050a5dd6b347ed167cb5f99d0d3857a1e30bc Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Mon, 13 Jan 2025 07:51:37 +0530 Subject: [PATCH 09/34] Clutching at straws --- tests/test_mutation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 837cc0f0..5f175e22 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -432,9 +432,9 @@ def test_run_forced_fail_test_with_failing_test(capfd): run_forced_fail_test(runner) - out, err = capfd.readouterr() - assert 'Running forced fail test' in out - assert 'done' in out + captured_output = capfd.readouterr() + assert 'Running forced fail test' in captured_output.out + assert 'done' in captured_output.out assert os.environ['MUTANT_UNDER_TEST'] is '' From 66b37057f98f8c0b200e73de4edd8a35b7a3e0ea Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Mon, 13 Jan 2025 07:53:30 +0530 Subject: [PATCH 10/34] Revert "Clutching at straws" This reverts commit 4d3050a5dd6b347ed167cb5f99d0d3857a1e30bc. --- tests/test_mutation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 5f175e22..837cc0f0 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -432,9 +432,9 @@ def test_run_forced_fail_test_with_failing_test(capfd): run_forced_fail_test(runner) - captured_output = capfd.readouterr() - assert 'Running forced fail test' in captured_output.out - assert 'done' in captured_output.out + out, err = capfd.readouterr() + assert 'Running forced fail test' in out + assert 'done' in out assert os.environ['MUTANT_UNDER_TEST'] is '' From 030faa88307263c6db63dfa42c32a5c8eaf15e5f Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Mon, 13 Jan 2025 07:54:49 +0530 Subject: [PATCH 11/34] try a sleep in case it is a race condition --- tests/test_mutation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 837cc0f0..a70d6d95 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -1,5 +1,6 @@ import os from io import StringIO +from time import sleep from unittest.mock import Mock import pytest @@ -432,6 +433,8 @@ def test_run_forced_fail_test_with_failing_test(capfd): run_forced_fail_test(runner) + sleep(5) + out, err = capfd.readouterr() assert 'Running forced fail test' in out assert 'done' in out From ab66e9e668b1de147a75db84b9003205d72fbb27 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Mon, 13 Jan 2025 08:10:06 +0530 Subject: [PATCH 12/34] Revert "try a sleep in case it is a race condition" This reverts commit 030faa88307263c6db63dfa42c32a5c8eaf15e5f. --- tests/test_mutation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_mutation.py b/tests/test_mutation.py index a70d6d95..837cc0f0 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -1,6 +1,5 @@ import os from io import StringIO -from time import sleep from unittest.mock import Mock import pytest @@ -433,8 +432,6 @@ def test_run_forced_fail_test_with_failing_test(capfd): run_forced_fail_test(runner) - sleep(5) - out, err = capfd.readouterr() assert 'Running forced fail test' in out assert 'done' in out From 6fd2e3ba45002ca155b3c27d52561f09734631a1 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:24:19 +0530 Subject: [PATCH 13/34] Debug output for GitHub Actions --- mutmut/__main__.py | 8 ++++++++ tests/test_mutation.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index ba298f96..334ea1e0 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -906,6 +906,7 @@ def __init__(self, callback=lambda s: None, spinner_title=None): class StdOutRedirect(TextIOBase): def __init__(self, catcher): + print("In StdOutRedirect.__init__") self.catcher = catcher def write(self, s): @@ -918,27 +919,34 @@ def write(self, s): # noinspection PyMethodMayBeStatic def stop(self): + print("In CatchOutput.stop") sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ def start(self): + print("In CatchOutput.start") if self.spinner_title: print_status(self.spinner_title) sys.stdout = self.redirect sys.stderr = self.redirect if mutmut.config.debug: + print("Stopping because debug") self.stop() def dump_output(self): + print("In CatchOutput.dump_output") + print(f"self.strings: {self.strings}") self.stop() for line in self.strings: print(line, end='') def __enter__(self): + print("In CatchOutput.__enter__") self.start() return self def __exit__(self, exc_type, exc_val, exc_tb): + print("In CatchOutput.__exit__") self.stop() if self.spinner_title: print() diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 837cc0f0..4cf34743 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -433,6 +433,9 @@ def test_run_forced_fail_test_with_failing_test(capfd): run_forced_fail_test(runner) out, err = capfd.readouterr() + with capfd.disabled(): + print(out) + print(err) assert 'Running forced fail test' in out assert 'done' in out assert os.environ['MUTANT_UNDER_TEST'] is '' From f94b6495a3dc447126f20b961f4a6894039aa003 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:26:52 +0530 Subject: [PATCH 14/34] Debug output for GitHub Actions --- tests/test_mutation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 4cf34743..574d7de3 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -434,8 +434,8 @@ def test_run_forced_fail_test_with_failing_test(capfd): out, err = capfd.readouterr() with capfd.disabled(): - print(out) - print(err) + print(f"out: {out}") + print(f"err: {err}") assert 'Running forced fail test' in out assert 'done' in out assert os.environ['MUTANT_UNDER_TEST'] is '' From 5777f876e9de8adee39cd39e3c997c70dcfd3b94 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:35:42 +0530 Subject: [PATCH 15/34] Try explicit catcher.stop() --- mutmut/__main__.py | 1 + tests/test_mutation.py | 1 + 2 files changed, 2 insertions(+) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 334ea1e0..90194165 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -895,6 +895,7 @@ def run_forced_fail_test(runner): raise SystemExit(1) except MutmutProgrammaticFailException: pass + catcher.stop() os.environ['MUTANT_UNDER_TEST'] = '' print(' done') diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 574d7de3..8c7240e9 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -434,6 +434,7 @@ def test_run_forced_fail_test_with_failing_test(capfd): out, err = capfd.readouterr() with capfd.disabled(): + print() print(f"out: {out}") print(f"err: {err}") assert 'Running forced fail test' in out From 009f55b428f1a3beee4dfecb74f177d0f0e098c3 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:37:46 +0530 Subject: [PATCH 16/34] Try with os._exit(1) --- mutmut/__main__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 90194165..327608a2 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -892,10 +892,11 @@ def run_forced_fail_test(runner): if runner.run_forced_fail() == 0: catcher.dump_output() print("FAILED: Unable to force test failures") - raise SystemExit(1) + # raise SystemExit(1) + os._exit(1) except MutmutProgrammaticFailException: pass - catcher.stop() + # catcher.stop() os.environ['MUTANT_UNDER_TEST'] = '' print(' done') From 712d279fd8b1730dfb5f2d69e04f9db11b9e910c Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:43:00 +0530 Subject: [PATCH 17/34] Move the done into the with CatchOutput block --- mutmut/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 327608a2..0b6f2d19 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -897,8 +897,10 @@ def run_forced_fail_test(runner): except MutmutProgrammaticFailException: pass # catcher.stop() + print(' done - a') + os.environ['MUTANT_UNDER_TEST'] = '' - print(' done') + print(' done - b') class CatchOutput: From c2f57abc2f1bebfbba2bc88072d264cf22705145 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:44:30 +0530 Subject: [PATCH 18/34] Try done output in the except block --- mutmut/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 0b6f2d19..f88ab785 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -895,6 +895,7 @@ def run_forced_fail_test(runner): # raise SystemExit(1) os._exit(1) except MutmutProgrammaticFailException: + print(' done - c') pass # catcher.stop() print(' done - a') From 7faadf9a3484ca37bbff8eaadaa336fee0e61f9a Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:46:50 +0530 Subject: [PATCH 19/34] Try catcher.dump_output() --- mutmut/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index f88ab785..0c4fd02a 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -899,6 +899,7 @@ def run_forced_fail_test(runner): pass # catcher.stop() print(' done - a') + catcher.dump_output() os.environ['MUTANT_UNDER_TEST'] = '' print(' done - b') From d5de126afb9dd2142cbe6a3fd32a75f224d6a5e7 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:47:23 +0530 Subject: [PATCH 20/34] Use SystemExit again --- mutmut/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 0c4fd02a..77992a16 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -892,8 +892,7 @@ def run_forced_fail_test(runner): if runner.run_forced_fail() == 0: catcher.dump_output() print("FAILED: Unable to force test failures") - # raise SystemExit(1) - os._exit(1) + raise SystemExit(1) except MutmutProgrammaticFailException: print(' done - c') pass From c6ce598b78ae500f2b1aec85eceb3502261bb327 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:52:20 +0530 Subject: [PATCH 21/34] Clutching at straws --- tests/test_mutation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 8c7240e9..b9656096 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -437,6 +437,10 @@ def test_run_forced_fail_test_with_failing_test(capfd): print() print(f"out: {out}") print(f"err: {err}") + + print() + print(f"out: {out}") + print(f"err: {err}") assert 'Running forced fail test' in out assert 'done' in out assert os.environ['MUTANT_UNDER_TEST'] is '' From 134d502ac4786cf4b418a9854ce2bc329c840b9a Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:53:52 +0530 Subject: [PATCH 22/34] Clutching at straws --- tests/test_mutation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_mutation.py b/tests/test_mutation.py index b9656096..b87367ad 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -437,6 +437,9 @@ def test_run_forced_fail_test_with_failing_test(capfd): print() print(f"out: {out}") print(f"err: {err}") + assert 'Running forced fail test' in out + assert 'done' in out + assert os.environ['MUTANT_UNDER_TEST'] is '' print() print(f"out: {out}") From bb002bc7e7dbbb76201fbc94d464f6699d5c0045 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:22:23 +0530 Subject: [PATCH 23/34] sys.stdout.write --- mutmut/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 77992a16..9d1e42ee 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -894,14 +894,14 @@ def run_forced_fail_test(runner): print("FAILED: Unable to force test failures") raise SystemExit(1) except MutmutProgrammaticFailException: - print(' done - c') + sys.stdout.write(' done - c') pass # catcher.stop() - print(' done - a') + sys.stdout.write(' done - a') catcher.dump_output() os.environ['MUTANT_UNDER_TEST'] = '' - print(' done - b') + sys.stdout.write(' done - b') class CatchOutput: From de1020d7827486953a99809799f26d4b54012124 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:26:52 +0530 Subject: [PATCH 24/34] Revert "sys.stdout.write" This reverts commit bb002bc7e7dbbb76201fbc94d464f6699d5c0045. --- mutmut/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 9d1e42ee..77992a16 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -894,14 +894,14 @@ def run_forced_fail_test(runner): print("FAILED: Unable to force test failures") raise SystemExit(1) except MutmutProgrammaticFailException: - sys.stdout.write(' done - c') + print(' done - c') pass # catcher.stop() - sys.stdout.write(' done - a') + print(' done - a') catcher.dump_output() os.environ['MUTANT_UNDER_TEST'] = '' - sys.stdout.write(' done - b') + print(' done - b') class CatchOutput: From 39a748f366be56644225ca1012baff8c70b77606 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:30:39 +0530 Subject: [PATCH 25/34] Run on macos-15 in GitHub Actions --- .github/workflows/tests.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ee117ba7..b8c3beee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,3 +35,36 @@ jobs: # run: python -m pytest run: python -m pytest -vvv --capture=no tests/test_mutation.py::test_run_forced_fail_test_with_failing_test continue-on-error: false + + tests-mac-os: + name: Test Python ${{ matrix.python-version }} on MacOS + # Todo: Revert to ubuntu-latest when Python 3.7 support no longer needed + runs-on: macos-15 + strategy: + fail-fast: false + matrix: + python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7"] + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r test_requirements.txt + + - name: Run tests TEMP + # run: python -m pytest + run: python -m pytest -vvv --capture=no tests/test_mutation.py::test_run_forced_fail_test_with_failing_test_TEMP + continue-on-error: false + + - name: Run tests + # run: python -m pytest + run: python -m pytest -vvv --capture=no tests/test_mutation.py::test_run_forced_fail_test_with_failing_test + continue-on-error: false From 3084a7630279bc1740093a76a141b5e1fb701889 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:37:34 +0530 Subject: [PATCH 26/34] Output some python info --- .github/workflows/tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b8c3beee..1532db6a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,6 +20,10 @@ jobs: with: python-version: ${{ matrix.python-version }} + - run: python --version + + - run: which python + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -53,6 +57,10 @@ jobs: with: python-version: ${{ matrix.python-version }} + - run: python --version + + - run: which python + - name: Install dependencies run: | python -m pip install --upgrade pip From 29add96f6eb074de2b5c47ff5570f63105e37fd5 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:11:01 +0530 Subject: [PATCH 27/34] Revert "Run on macos-15 in GitHub Actions" This reverts commit 39a748f366be56644225ca1012baff8c70b77606. --- .github/workflows/tests.yml | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1532db6a..61649265 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,40 +39,3 @@ jobs: # run: python -m pytest run: python -m pytest -vvv --capture=no tests/test_mutation.py::test_run_forced_fail_test_with_failing_test continue-on-error: false - - tests-mac-os: - name: Test Python ${{ matrix.python-version }} on MacOS - # Todo: Revert to ubuntu-latest when Python 3.7 support no longer needed - runs-on: macos-15 - strategy: - fail-fast: false - matrix: - python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7"] - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - run: python --version - - - run: which python - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r test_requirements.txt - - - name: Run tests TEMP - # run: python -m pytest - run: python -m pytest -vvv --capture=no tests/test_mutation.py::test_run_forced_fail_test_with_failing_test_TEMP - continue-on-error: false - - - name: Run tests - # run: python -m pytest - run: python -m pytest -vvv --capture=no tests/test_mutation.py::test_run_forced_fail_test_with_failing_test - continue-on-error: false From 713e228b806ec85fc7938bcce630510d9d884fa5 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:11:55 +0530 Subject: [PATCH 28/34] setup-python@v4 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 61649265..e2b36748 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 8f8c2cd945799f10e641412f79f9bc89fc22ce2f Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:15:22 +0530 Subject: [PATCH 29/34] Drop setup-python@v5 --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e2b36748..5621e693 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,16 +9,16 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false - matrix: - python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7"] + # matrix: + # python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7"] steps: - name: Checkout uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} - run: python --version From ac1949122c16f563816884f5c2c1b1efccb484f7 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:20:08 +0530 Subject: [PATCH 30/34] Clutching at straws --- mutmut/__main__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 77992a16..350f8353 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -902,6 +902,10 @@ def run_forced_fail_test(runner): os.environ['MUTANT_UNDER_TEST'] = '' print(' done - b') + print('111') + print('222') + print('333') + print() class CatchOutput: From a7ca7d02330b0a6d86ad0617df383d13a674941c Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:23:11 +0530 Subject: [PATCH 31/34] Clutching at straws --- mutmut/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 350f8353..5dc93c9d 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -889,6 +889,7 @@ def run_forced_fail_test(runner): os.environ['MUTANT_UNDER_TEST'] = 'fail' with CatchOutput(spinner_title='Running forced fail test') as catcher: try: + print(f"runner.run_forced_fail(): {runner.run_forced_fail()}") if runner.run_forced_fail() == 0: catcher.dump_output() print("FAILED: Unable to force test failures") From 43a4bdd8b491655ecb9f36414967f0ce0fe1e378 Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:27:04 +0530 Subject: [PATCH 32/34] Clutching at straws --- mutmut/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 5dc93c9d..c9887354 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -891,7 +891,7 @@ def run_forced_fail_test(runner): try: print(f"runner.run_forced_fail(): {runner.run_forced_fail()}") if runner.run_forced_fail() == 0: - catcher.dump_output() + # catcher.dump_output() print("FAILED: Unable to force test failures") raise SystemExit(1) except MutmutProgrammaticFailException: From a94e680f89d82fa97de80165627f415ad22040fd Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:39:33 +0530 Subject: [PATCH 33/34] Clutching at straws --- mutmut/__main__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index c9887354..c6f5702f 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -644,6 +644,7 @@ def run_stats(self, *, tests): raise NotImplementedError() def run_forced_fail(self): + print("In TestRunner.run_forced_fail") raise NotImplementedError() def prepare_main_test_run(self): @@ -728,6 +729,7 @@ def run_tests(self, *, mutant_name, tests): return int(self.execute_pytest(['-x', '-q', '--import-mode=append'] + list(tests))) def run_forced_fail(self): + print("In PytestRunner.run_forced_fail") with change_cwd('mutants'): return int(self.execute_pytest(['-x', '-q', '--import-mode=append'])) @@ -762,6 +764,7 @@ def post_test_callback(_name, **_): return hammett.main(quiet=True, fail_fast=True, disable_assert_analyze=True, post_test_callback=post_test_callback, use_cache=False, insert_cwd=False) def run_forced_fail(self): + print("In HammettRunner.run_forced_fail") import hammett return hammett.main(quiet=True, fail_fast=True, disable_assert_analyze=True, use_cache=False, insert_cwd=False) @@ -899,14 +902,10 @@ def run_forced_fail_test(runner): pass # catcher.stop() print(' done - a') - catcher.dump_output() + # catcher.dump_output() os.environ['MUTANT_UNDER_TEST'] = '' print(' done - b') - print('111') - print('222') - print('333') - print() class CatchOutput: From af2663abbbd103616a8c1e93efe78b1d6f87576d Mon Sep 17 00:00:00 2001 From: Will Gibson <8738245+WillGibson@users.noreply.github.com> Date: Sun, 19 Jan 2025 00:43:58 +0530 Subject: [PATCH 34/34] Drop CatchOutput --- mutmut/__main__.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/mutmut/__main__.py b/mutmut/__main__.py index c6f5702f..c227d260 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -890,19 +890,20 @@ def print_stats(source_file_mutation_data_by_path, force_output=False): def run_forced_fail_test(runner): os.environ['MUTANT_UNDER_TEST'] = 'fail' - with CatchOutput(spinner_title='Running forced fail test') as catcher: - try: - print(f"runner.run_forced_fail(): {runner.run_forced_fail()}") - if runner.run_forced_fail() == 0: - # catcher.dump_output() - print("FAILED: Unable to force test failures") - raise SystemExit(1) - except MutmutProgrammaticFailException: - print(' done - c') - pass - # catcher.stop() - print(' done - a') - # catcher.dump_output() + # with CatchOutput(spinner_title='Running forced fail test') as catcher: + print("'Running forced fail test'") + try: + print(f"runner.run_forced_fail(): {runner.run_forced_fail()}") + if runner.run_forced_fail() == 0: + # catcher.dump_output() + print("FAILED: Unable to force test failures") + raise SystemExit(1) + except MutmutProgrammaticFailException: + print(' done - c') + pass + # catcher.stop() + print(' done - a') + # catcher.dump_output() os.environ['MUTANT_UNDER_TEST'] = '' print(' done - b')