From 9344ad46c7c58dfc7b2106f9a1de5328aea92db8 Mon Sep 17 00:00:00 2001 From: Davide Canton Date: Wed, 10 Jan 2024 22:59:51 +0100 Subject: [PATCH 1/6] Updated repo of black in pre-commit to use mirror --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ef5272..d386003 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: -- repo: https://github.com/psf/black +- repo: https://github.com/psf/black-pre-commit-mirror rev: 23.12.1 hooks: - id: black From fa3e177687926e2944489cdcc1eeaffc3243d48e Mon Sep 17 00:00:00 2001 From: Davide Canton Date: Wed, 10 Jan 2024 22:59:57 +0100 Subject: [PATCH 2/6] Respect order in replay file when re-running tests --- src/pytest_replay/__init__.py | 17 ++++++------ tests/test_replay.py | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/pytest_replay/__init__.py b/src/pytest_replay/__init__.py index 7e3a3d8..5c6b916 100644 --- a/src/pytest_replay/__init__.py +++ b/src/pytest_replay/__init__.py @@ -106,22 +106,23 @@ def pytest_collection_modifyitems(self, items, config): with open(replay_file, "r", encoding="UTF-8") as f: all_lines = f.readlines() - nodeids = { + nodeids = dict.fromkeys( json.loads(line)["nodeid"] for line in all_lines if not line.strip().startswith(("#", "//")) - } + ) + + items_dict = {item.nodeid: item for item in items} remaining = [] - deselected = [] - for item in items: - if item.nodeid in nodeids: + for nodeid in nodeids: + if item := items_dict.pop(nodeid): remaining.append(item) - else: - deselected.append(item) + deselected = list(items_dict.values()) if deselected: config.hook.pytest_deselected(items=deselected) - items[:] = remaining + + items[:] = remaining def append_test_to_script(self, nodeid, line): suffix = "-" + self.xdist_worker_name if self.xdist_worker_name else "" diff --git a/tests/test_replay.py b/tests/test_replay.py index ed63215..be0fae7 100644 --- a/tests/test_replay.py +++ b/tests/test_replay.py @@ -1,3 +1,4 @@ +import itertools as it import json import pytest @@ -216,3 +217,54 @@ def test_2(): expected = {"test_cwd_changed.py::test_1", "test_cwd_changed.py::test_2"} assert contents == expected assert result.ret == 0 + + +@pytest.mark.usefixtures("suite") +def test_execution_different_order(testdir): + dir = testdir.tmpdir / "replay" + options = [f"--replay-record-dir={dir}"] + result = testdir.runpytest(*options) + + replay_file = dir / ".pytest-replay.txt" + + with replay_file.open("r+") as f: + content = f.readlines() + + # pairwise shuffle of replay file + pairs = [(content[i], content[i + 1]) for i in range(0, len(content), 2)] + pairs = [pairs[2], pairs[0], pairs[3], pairs[1]] + content = list(it.chain.from_iterable(pairs)) + + f.seek(0) + f.writelines(content) + + result = testdir.runpytest(f"--replay={replay_file}", "-v") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "test_2.py::test_zz*25%*", + "test_1.py::test_foo*50%*", + "test_3.py::test_foobar*75%*", + "test_1.py::test_bar*100%*", + ], + consecutive=True, + ) + + +@pytest.mark.usefixtures("suite") +def test_filter_out_tests_not_in_file(testdir): + dir = testdir.tmpdir / "replay" + options = [f"--replay-record-dir={dir}", "-k", "foo"] + result = testdir.runpytest(*options) + + replay_file = dir / ".pytest-replay.txt" + + result = testdir.runpytest(f"--replay={replay_file}", "-v") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "test_1.py::test_foo*50%*", + "test_3.py::test_foobar*100%*", + ], + consecutive=True, + ) From 9b3b52fdad58e405900a71fa331a0cb8e00103bd Mon Sep 17 00:00:00 2001 From: Davide Canton Date: Thu, 11 Jan 2024 14:15:20 +0100 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Bruno Oliveira --- src/pytest_replay/__init__.py | 1 + tests/test_replay.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/pytest_replay/__init__.py b/src/pytest_replay/__init__.py index 5c6b916..d628e26 100644 --- a/src/pytest_replay/__init__.py +++ b/src/pytest_replay/__init__.py @@ -114,6 +114,7 @@ def pytest_collection_modifyitems(self, items, config): items_dict = {item.nodeid: item for item in items} remaining = [] + # Make sure to respect the order from the JSON file (#52). for nodeid in nodeids: if item := items_dict.pop(nodeid): remaining.append(item) diff --git a/tests/test_replay.py b/tests/test_replay.py index be0fae7..cebc6d2 100644 --- a/tests/test_replay.py +++ b/tests/test_replay.py @@ -221,6 +221,7 @@ def test_2(): @pytest.mark.usefixtures("suite") def test_execution_different_order(testdir): + """Ensure tests execute in the order defined by the JSON file, not collection (#52).""" dir = testdir.tmpdir / "replay" options = [f"--replay-record-dir={dir}"] result = testdir.runpytest(*options) @@ -253,6 +254,7 @@ def test_execution_different_order(testdir): @pytest.mark.usefixtures("suite") def test_filter_out_tests_not_in_file(testdir): + """Tests not found in the JSON file should not run.""" dir = testdir.tmpdir / "replay" options = [f"--replay-record-dir={dir}", "-k", "foo"] result = testdir.runpytest(*options) From 960c66e41c3089a0fdd266081bf9c019ebc9ccd1 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 11 Jan 2024 10:34:06 -0300 Subject: [PATCH 4/6] Update src/pytest_replay/__init__.py --- src/pytest_replay/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pytest_replay/__init__.py b/src/pytest_replay/__init__.py index d628e26..b830b8f 100644 --- a/src/pytest_replay/__init__.py +++ b/src/pytest_replay/__init__.py @@ -106,6 +106,7 @@ def pytest_collection_modifyitems(self, items, config): with open(replay_file, "r", encoding="UTF-8") as f: all_lines = f.readlines() + # Use a dict to deduplicate the node ids while keeping the order. nodeids = dict.fromkeys( json.loads(line)["nodeid"] for line in all_lines From ee9c0024cbcc8dd7631d6e1e799d511f72c165f4 Mon Sep 17 00:00:00 2001 From: Davide Canton Date: Thu, 11 Jan 2024 14:38:05 +0100 Subject: [PATCH 5/6] Make code compliant with older python versions --- src/pytest_replay/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pytest_replay/__init__.py b/src/pytest_replay/__init__.py index b830b8f..cf6c371 100644 --- a/src/pytest_replay/__init__.py +++ b/src/pytest_replay/__init__.py @@ -117,7 +117,8 @@ def pytest_collection_modifyitems(self, items, config): remaining = [] # Make sure to respect the order from the JSON file (#52). for nodeid in nodeids: - if item := items_dict.pop(nodeid): + item = items_dict.pop(nodeid) + if item: remaining.append(item) deselected = list(items_dict.values()) From fb6f7e82f0de34fa38f5a5916348a0483f0df258 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 11 Jan 2024 11:31:19 -0300 Subject: [PATCH 6/6] Update CHANGELOG.rst --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8000273..35cd516 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,10 @@ +UNRELEASED +========== + +* Test execution order using ``--replay`` now follows the recorded order, not the collection order, as was always intended (`#52`_). + +.. _`#52`: https://github.com/ESSS/pytest-replay/pull/53 + 1.4.0 (2021-06-09) ==================