diff --git a/ddtrace/contrib/internal/pytest/_retry_utils.py b/ddtrace/contrib/internal/pytest/_retry_utils.py index eab45f049be..98133ec505a 100644 --- a/ddtrace/contrib/internal/pytest/_retry_utils.py +++ b/ddtrace/contrib/internal/pytest/_retry_utils.py @@ -117,7 +117,7 @@ def _retry_run_when(item, when, outcomes: RetryOutcomes) -> t.Tuple[CallInfo, _p ) else: call = CallInfo.from_call(lambda: hook(item=item), when=when) - report = pytest.TestReport.from_item_and_call(item=item, call=call) + report = item.ihook.pytest_runtest_makereport(item=item, call=call) if report.outcome == "passed": report.outcome = outcomes.PASSED elif report.outcome == "failed" or report.outcome == "error": diff --git a/releasenotes/notes/ci_visibility-fix-pytest-classes-b54582b09727fdc1.yaml b/releasenotes/notes/ci_visibility-fix-pytest-classes-b54582b09727fdc1.yaml new file mode 100644 index 00000000000..829c05fa761 --- /dev/null +++ b/releasenotes/notes/ci_visibility-fix-pytest-classes-b54582b09727fdc1.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + CI Visibility: fixes an issue where Auto Test Retries with pytest would always consider retries of tests defined + inside unittest classes to be successful. diff --git a/tests/contrib/pytest/test_pytest_atr.py b/tests/contrib/pytest/test_pytest_atr.py index ebb4f8421d8..92012461ed0 100644 --- a/tests/contrib/pytest/test_pytest_atr.py +++ b/tests/contrib/pytest/test_pytest_atr.py @@ -24,17 +24,25 @@ ) _TEST_PASS_CONTENT = """ +import unittest + def test_func_pass(): assert True + +class SomeTestCase(unittest.TestCase): + def test_class_func_pass(self): + assert True """ _TEST_FAIL_CONTENT = """ import pytest +import unittest def test_func_fail(): assert False _test_func_retries_skip_count = 0 + def test_func_retries_skip(): global _test_func_retries_skip_count _test_func_retries_skip_count += 1 @@ -42,9 +50,24 @@ def test_func_retries_skip(): pytest.skip() assert False +_test_class_func_retries_skip_count = 0 + +class SomeTestCase(unittest.TestCase): + def test_class_func_fail(self): + assert False + + def test_class_func_retries_skip(self): + global _test_class_func_retries_skip_count + _test_class_func_retries_skip_count += 1 + if _test_class_func_retries_skip_count > 1: + pytest.skip() + assert False """ + _TEST_PASS_ON_RETRIES_CONTENT = """ +import unittest + _test_func_passes_4th_retry_count = 0 def test_func_passes_4th_retry(): global _test_func_passes_4th_retry_count @@ -56,10 +79,19 @@ def test_func_passes_1st_retry(): global _test_func_passes_1st_retry_count _test_func_passes_1st_retry_count += 1 assert _test_func_passes_1st_retry_count == 2 + +class SomeTestCase(unittest.TestCase): + _test_func_passes_4th_retry_count = 0 + + def test_func_passes_4th_retry(self): + SomeTestCase._test_func_passes_4th_retry_count += 1 + assert SomeTestCase._test_func_passes_4th_retry_count == 5 + """ _TEST_ERRORS_CONTENT = """ import pytest +import unittest @pytest.fixture def fixture_fails_setup(): @@ -79,6 +111,7 @@ def test_func_fails_teardown(fixture_fails_teardown): _TEST_SKIP_CONTENT = """ import pytest +import unittest @pytest.mark.skip def test_func_skip_mark(): @@ -86,6 +119,14 @@ def test_func_skip_mark(): def test_func_skip_inside(): pytest.skip() + +class SomeTestCase(unittest.TestCase): + @pytest.mark.skip + def test_class_func_skip_mark(self): + assert True + + def test_class_func_skip_inside(self): + pytest.skip() """ @@ -105,7 +146,7 @@ def test_pytest_atr_no_ddtrace_does_not_retry(self): self.testdir.makepyfile(test_pass_on_retries=_TEST_PASS_ON_RETRIES_CONTENT) self.testdir.makepyfile(test_skip=_TEST_SKIP_CONTENT) rec = self.inline_run() - rec.assertoutcome(passed=2, failed=6, skipped=2) + rec.assertoutcome(passed=3, failed=9, skipped=4) assert len(self.pop_spans()) == 0 def test_pytest_atr_env_var_disables_retrying(self): @@ -117,7 +158,7 @@ def test_pytest_atr_env_var_disables_retrying(self): with mock.patch("ddtrace.internal.ci_visibility.recorder.ddconfig", _get_default_civisibility_ddconfig()): rec = self.inline_run("--ddtrace", "-s", extra_env={"DD_CIVISIBILITY_FLAKY_RETRY_ENABLED": "0"}) - rec.assertoutcome(passed=2, failed=6, skipped=2) + rec.assertoutcome(passed=3, failed=9, skipped=4) assert len(self.pop_spans()) > 0 def test_pytest_atr_env_var_does_not_override_api(self): @@ -133,7 +174,7 @@ def test_pytest_atr_env_var_does_not_override_api(self): return_value=TestVisibilityAPISettings(flaky_test_retries_enabled=False), ): rec = self.inline_run("--ddtrace", extra_env={"DD_CIVISIBILITY_FLAKY_RETRY_ENABLED": "1"}) - rec.assertoutcome(passed=2, failed=6, skipped=2) + rec.assertoutcome(passed=3, failed=9, skipped=4) assert len(self.pop_spans()) > 0 def test_pytest_atr_spans(self): @@ -174,6 +215,15 @@ def test_pytest_atr_spans(self): func_fail_retries += 1 assert func_fail_retries == 5 + class_func_fail_spans = _get_spans_from_list(spans, "test", "SomeTestCase::test_class_func_fail") + assert len(class_func_fail_spans) == 6 + class_func_fail_retries = 0 + for class_func_fail_span in class_func_fail_spans: + assert class_func_fail_span.get_tag("test.status") == "fail" + if class_func_fail_span.get_tag("test.is_retry") == "true": + class_func_fail_retries += 1 + assert class_func_fail_retries == 5 + func_fail_skip_spans = _get_spans_from_list(spans, "test", "test_func_retries_skip") assert len(func_fail_skip_spans) == 6 func_fail_skip_retries = 0 @@ -184,6 +234,18 @@ def test_pytest_atr_spans(self): func_fail_skip_retries += 1 assert func_fail_skip_retries == 5 + class_func_fail_skip_spans = _get_spans_from_list(spans, "test", "SomeTestCase::test_class_func_retries_skip") + assert len(class_func_fail_skip_spans) == 6 + class_func_fail_skip_retries = 0 + for class_func_fail_skip_span in class_func_fail_skip_spans: + class_func_fail_skip_is_retry = class_func_fail_skip_span.get_tag("test.is_retry") == "true" + assert class_func_fail_skip_span.get_tag("test.status") == ( + "skip" if class_func_fail_skip_is_retry else "fail" + ) + if class_func_fail_skip_is_retry: + class_func_fail_skip_retries += 1 + assert class_func_fail_skip_retries == 5 + func_pass_spans = _get_spans_from_list(spans, "test", "test_func_pass") assert len(func_pass_spans) == 1 assert func_pass_spans[0].get_tag("test.status") == "pass" @@ -201,7 +263,17 @@ def test_pytest_atr_spans(self): assert func_skip_inside_spans[0].get_tag("test.status") == "skip" assert func_skip_inside_spans[0].get_tag("test.is_retry") is None - assert len(spans) == 31 + class_func_skip_mark_spans = _get_spans_from_list(spans, "test", "SomeTestCase::test_class_func_skip_mark") + assert len(class_func_skip_mark_spans) == 1 + assert class_func_skip_mark_spans[0].get_tag("test.status") == "skip" + assert class_func_skip_mark_spans[0].get_tag("test.is_retry") is None + + class_func_skip_inside_spans = _get_spans_from_list(spans, "test", "SomeTestCase::test_class_func_skip_inside") + assert len(class_func_skip_inside_spans) == 1 + assert class_func_skip_inside_spans[0].get_tag("test.status") == "skip" + assert class_func_skip_inside_spans[0].get_tag("test.is_retry") is None + + assert len(spans) == 51 def test_pytest_atr_fails_session_when_test_fails(self): self.testdir.makepyfile(test_pass=_TEST_PASS_CONTENT) @@ -212,7 +284,7 @@ def test_pytest_atr_fails_session_when_test_fails(self): rec = self.inline_run("--ddtrace") spans = self.pop_spans() assert rec.ret == 1 - assert len(spans) == 28 + assert len(spans) == 48 def test_pytest_atr_passes_session_when_test_pass(self): self.testdir.makepyfile(test_pass=_TEST_PASS_CONTENT) @@ -222,9 +294,14 @@ def test_pytest_atr_passes_session_when_test_pass(self): rec = self.inline_run("--ddtrace") spans = self.pop_spans() assert rec.ret == 0 - assert len(spans) == 15 + assert len(spans) == 23 def test_pytest_atr_does_not_retry_failed_setup_or_teardown(self): + # NOTE: This feature only works for regular pytest tests. For tests inside unittest classes, setup and teardown + # happens at the 'call' phase, and we don't have a way to detect that the error happened during setup/teardown, + # so tests will be retried as if they were failing tests. + # See . + self.testdir.makepyfile(test_errors=_TEST_ERRORS_CONTENT) rec = self.inline_run("--ddtrace") spans = self.pop_spans()