Merge pull request #3865 from jobh/cleanup-use-of-example-in-tests

Clean up use of example in tests

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+Internal test refactoring.

hypothesis-python/src/hypothesis/core.py

@@ -628,7 +628,7 @@ def get_random_for_wrapped_test(test, wrapped_test):
         return Random(global_force_seed)
     else:
         global _hypothesis_global_random
-        if _hypothesis_global_random is None:
+        if _hypothesis_global_random is None:  # pragma: no cover
             _hypothesis_global_random = Random()
         seed = _hypothesis_global_random.getrandbits(128)
         wrapped_test._hypothesis_internal_use_generated_seed = seed

hypothesis-python/tests/array_api/test_argument_validation.py

@@ -16,6 +16,7 @@
 from hypothesis.extra.array_api import NominalVersion, make_strategies_namespace
 
 from tests.array_api.common import MIN_VER_FOR_COMPLEX
+from tests.common.debug import check_can_generate_examples
 
 
 def e(name, *, _min_version: Optional[NominalVersion] = None, **kwargs):
@@ -225,7 +226,7 @@ def test_raise_invalid_argument(xp, xps, strat_name, kwargs):
     strat_func = getattr(xps, strat_name)
     strat = strat_func(**kwargs)
     with pytest.raises(InvalidArgument):
-        strat.example()
+        check_can_generate_examples(strat)
 
 
 @pytest.mark.parametrize("api_version", [..., "latest", "1970.01", 42])

hypothesis-python/tests/array_api/test_arrays.py

@@ -20,7 +20,12 @@
     dtype_name_params,
     flushes_to_zero,
 )
-from tests.common.debug import assert_all_examples, find_any, minimal
+from tests.common.debug import (
+    assert_all_examples,
+    check_can_generate_examples,
+    find_any,
+    minimal,
+)
 from tests.common.utils import flaky
 
 
@@ -222,13 +227,17 @@ def test_cannot_draw_unique_arrays_with_too_small_elements(xp, xps):
     """Unique strategy with elements strategy range smaller than its size raises
     helpful error."""
     with pytest.raises(InvalidArgument):
-        xps.arrays(xp.int8, 10, elements=st.integers(0, 5), unique=True).example()
+        check_can_generate_examples(
+            xps.arrays(xp.int8, 10, elements=st.integers(0, 5), unique=True)
+        )
 
 
 def test_cannot_fill_arrays_with_non_castable_value(xp, xps):
     """Strategy with fill not castable to dtype raises helpful error."""
     with pytest.raises(InvalidArgument):
-        xps.arrays(xp.int8, 10, fill=st.just("not a castable value")).example()
+        check_can_generate_examples(
+            xps.arrays(xp.int8, 10, fill=st.just("not a castable value"))
+        )
 
 
 def test_generate_unique_arrays_with_high_collision_elements(xp, xps):
@@ -284,7 +293,7 @@ def test_may_not_fill_unique_array_with_non_nan(xp, xps):
         fill=st.just(0.0),
     )
     with pytest.raises(InvalidArgument):
-        strat.example()
+        check_can_generate_examples(strat)
 
 
 @pytest.mark.parametrize(
@@ -298,7 +307,7 @@ def test_may_not_use_overflowing_integers(xp, xps, kwargs):
     """Strategy with elements strategy range outside the dtype's bounds raises
     helpful error."""
     with pytest.raises(InvalidArgument):
-        xps.arrays(dtype=xp.int8, shape=1, **kwargs).example()
+        check_can_generate_examples(xps.arrays(dtype=xp.int8, shape=1, **kwargs))
 
 
 @pytest.mark.parametrize("fill", [False, True])
@@ -321,7 +330,7 @@ def test_may_not_use_unrepresentable_elements(xp, xps, fill, dtype, strat):
     else:
         kw = {"elements": strat}
     with pytest.raises(InvalidArgument):
-        xps.arrays(dtype=dtype, shape=1, **kw).example()
+        check_can_generate_examples(xps.arrays(dtype=dtype, shape=1, **kw))
 
 
 def test_floats_can_be_constrained(xp, xps):
@@ -507,6 +516,6 @@ def test_subnormal_elements_validation(xp, xps):
     strat = xps.arrays(xp.float32, 10, elements=elements)
     if flushes_to_zero(xp, width=32):
         with pytest.raises(InvalidArgument, match="Generated subnormal float"):
-            strat.example()
+            check_can_generate_examples(strat)
     else:
-        strat.example()
+        check_can_generate_examples(strat)

hypothesis-python/tests/array_api/test_partial_adoptors.py

@@ -26,6 +26,8 @@
     mock_xp,
 )
 
+from tests.common.debug import check_can_generate_examples
+
 MOCK_WARN_MSG = f"determine.*{mock_xp.__name__}.*Array API"
 
 
@@ -56,7 +58,7 @@ def test_error_on_missing_attr(stratname, args, attr):
     xps = make_strategies_namespace(xp, api_version="draft")
     func = getattr(xps, stratname)
     with pytest.raises(InvalidArgument, match=f"{mock_xp.__name__}.*required.*{attr}"):
-        func(*args).example()
+        check_can_generate_examples(func(*args))
 
 
 dtypeless_xp = make_mock_xp(exclude=tuple(DTYPE_NAMES))
@@ -82,7 +84,7 @@ def test_error_on_missing_dtypes(stratname):
     required dtypes."""
     func = getattr(dtypeless_xps, stratname)
     with pytest.raises(InvalidArgument, match=f"{mock_xp.__name__}.*dtype.*namespace"):
-        func().example()
+        check_can_generate_examples(func())
 
 
 @pytest.mark.filterwarnings(f"ignore:.*{MOCK_WARN_MSG}.*")

hypothesis-python/tests/common/debug.py

@@ -89,16 +89,53 @@ def assert_no_examples(strategy, condition=lambda _: True):
         pass
 
 
-def assert_all_examples(strategy, predicate):
+def assert_all_examples(strategy, predicate, settings=None):
     """Asserts that all examples of the given strategy match the predicate.
 
     :param strategy: Hypothesis strategy to check
     :param predicate: (callable) Predicate that takes example and returns bool
     """
 
     @given(strategy)
+    @Settings(parent=settings, database=None)
     def assert_examples(s):
         msg = f"Found {s!r} using strategy {strategy} which does not match"
         assert predicate(s), msg
 
     assert_examples()
+
+
+def assert_simple_property(strategy, predicate, settings=None):
+    """Like assert_all_examples, intended as a self-documenting shortcut for simple constant
+    properties (`is`, `isinstance`, `==`, ...) that can be adequately verified in just a few
+    examples.
+
+    For more thorough checking, use assert_all_examples.
+    """
+
+    assert_all_examples(
+        strategy,
+        predicate,
+        Settings(
+            parent=settings,
+            max_examples=15,
+            suppress_health_check=list(HealthCheck),
+        ),
+    )
+
+
+def check_can_generate_examples(strategy, settings=None):
+    """Tries to generate a small number of examples from the strategy, to verify that it can
+    do so without raising.
+
+    Nothing is returned, it only checks that no error is raised.
+    """
+
+    assert_simple_property(
+        strategy,
+        lambda _: True,
+        settings=Settings(
+            parent=settings,
+            phases=(Phase.generate,),
+        ),
+    )

hypothesis-python/tests/common/setup.py

@@ -13,7 +13,6 @@
 
 from hypothesis import Phase, Verbosity, settings
 from hypothesis._settings import not_set
-from hypothesis.errors import NonInteractiveExampleWarning
 from hypothesis.internal.coverage import IN_COVERAGE_TESTS
 
 
@@ -36,9 +35,6 @@ def run():
         category=UserWarning,
     )
 
-    # User-facing warning which does not apply to our own tests
-    filterwarnings("ignore", category=NonInteractiveExampleWarning)
-
     # We do a smoke test here before we mess around with settings.
     x = settings()
 

hypothesis-python/tests/conftest.py

@@ -17,6 +17,8 @@
 import pytest
 
 from hypothesis._settings import is_in_ci
+from hypothesis.errors import NonInteractiveExampleWarning
+from hypothesis.internal.compat import add_note
 from hypothesis.internal.detection import is_hypothesis_test
 
 from tests.common import TIME_INCREMENT
@@ -106,20 +108,20 @@ def pytest_runtest_call(item):
     # This hookwrapper checks for PRNG state leaks from Hypothesis tests.
     # See: https://github.com/HypothesisWorks/hypothesis/issues/1919
     if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)):
-        yield
+        outcome = yield
     elif "pytest_randomly" in sys.modules:
         # See https://github.com/HypothesisWorks/hypothesis/issues/3041 - this
         # branch exists to make it easier on external contributors, but should
         # never run in our CI (because that would disable the check entirely).
         assert not is_in_ci()
-        yield
+        outcome = yield
     else:
         # We start by peturbing the state of the PRNG, because repeatedly
         # leaking PRNG state resets state_after to the (previously leaked)
         # state_before, and that just shows as "no use of random".
         random.seed(independent_random.randrange(2**32))
         before = random.getstate()
-        yield
+        outcome = yield
         after = random.getstate()
         if before != after:
             if after in random_states_after_tests:
@@ -129,3 +131,11 @@ def pytest_runtest_call(item):
                     "same global `random.getstate()`; this is probably a nasty bug!"
                 )
             random_states_after_tests[after] = item.nodeid
+
+    # Annotate usage of .example() with a hint about alternatives
+    if isinstance(getattr(outcome, "exception", None), NonInteractiveExampleWarning):
+        add_note(
+            outcome.exception,
+            "For hypothesis' own test suite, consider using one of the helper "
+            "methods in tests.common.debug instead.",
+        )

hypothesis-python/tests/cover/test_annotations.py

@@ -8,6 +8,7 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import warnings
 from inspect import Parameter as P, signature
 
 import attr
@@ -21,6 +22,8 @@
     get_pretty_function_description,
 )
 
+from tests.common.debug import check_can_generate_examples
+
 
 @given(st.integers())
 def test_has_an_annotation(i: int):
@@ -127,5 +130,6 @@ def test_attrs_inference_builds(c):
 
 def test_attrs_inference_from_type():
     s = st.from_type(Inferrables)
-    with pytest.warns(SmallSearchSpaceWarning):
-        s.example()
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", SmallSearchSpaceWarning)
+        check_can_generate_examples(s)

hypothesis-python/tests/cover/test_arbitrary_data.py

@@ -75,10 +75,5 @@ def test_errors_when_normal_strategy_functions_are_used(f):
         getattr(st.data(), f)(lambda x: 1)
 
 
-def test_errors_when_asked_for_example():
-    with raises(InvalidArgument):
-        st.data().example()
-
-
 def test_nice_repr():
     assert repr(st.data()) == "data()"

hypothesis-python/tests/cover/test_attrs_inference.py

@@ -16,6 +16,8 @@
 from hypothesis import given, strategies as st
 from hypothesis.errors import ResolutionFailed
 
+from tests.common.debug import check_can_generate_examples
+
 
 @attr.s
 class Inferrables:
@@ -83,12 +85,14 @@ def test_attrs_inference_from_type(c):
 @pytest.mark.parametrize("c", [Required, UnhelpfulConverter])
 def test_cannot_infer(c):
     with pytest.raises(ResolutionFailed):
-        st.builds(c).example()
+        check_can_generate_examples(st.builds(c))
 
 
 def test_cannot_infer_takes_self():
     with pytest.raises(ResolutionFailed):
-        st.builds(Inferrables, has_default_factory_takes_self=...).example()
+        check_can_generate_examples(
+            st.builds(Inferrables, has_default_factory_takes_self=...)
+        )
 
 
 @attr.s
@@ -98,12 +102,12 @@ class HasPrivateAttribute:
 
 @pytest.mark.parametrize("s", [st.just(42), ...])
 def test_private_attribute(s):
-    st.builds(HasPrivateAttribute, x=s).example()
+    check_can_generate_examples(st.builds(HasPrivateAttribute, x=s))
 
 
 def test_private_attribute_underscore_fails():
     with pytest.raises(TypeError, match="unexpected keyword argument '_x'"):
-        st.builds(HasPrivateAttribute, _x=st.just(42)).example()
+        check_can_generate_examples(st.builds(HasPrivateAttribute, _x=st.just(42)))
 
 
 def test_private_attribute_underscore_infer_fails():
@@ -112,7 +116,7 @@ def test_private_attribute_underscore_infer_fails():
     with pytest.raises(
         TypeError, match="Unexpected keyword argument _x for attrs class"
     ):
-        st.builds(HasPrivateAttribute, _x=...).example()
+        check_can_generate_examples(st.builds(HasPrivateAttribute, _x=...))
 
 
 @attr.s
@@ -122,4 +126,4 @@ class HasAliasedAttribute:
 
 @pytest.mark.parametrize("s", [st.just(42), ...])
 def test_aliased_attribute(s):
-    st.builds(HasAliasedAttribute, crazyname=s).example()
+    check_can_generate_examples(st.builds(HasAliasedAttribute, crazyname=s))

hypothesis-python/tests/cover/test_complex_numbers.py

@@ -17,7 +17,12 @@
 from hypothesis.errors import InvalidArgument
 from hypothesis.strategies import complex_numbers
 
-from tests.common.debug import assert_no_examples, find_any, minimal
+from tests.common.debug import (
+    assert_no_examples,
+    check_can_generate_examples,
+    find_any,
+    minimal,
+)
 
 
 def test_minimal():
@@ -129,4 +134,4 @@ def test_allow_subnormal(allow_subnormal, min_magnitude, max_magnitude):
 @pytest.mark.parametrize("allow_subnormal", [1, 0.0, "False"])
 def test_allow_subnormal_validation(allow_subnormal):
     with pytest.raises(InvalidArgument):
-        complex_numbers(allow_subnormal=allow_subnormal).example()
+        check_can_generate_examples(complex_numbers(allow_subnormal=allow_subnormal))

hypothesis-python/tests/cover/test_datetimes.py

@@ -15,7 +15,7 @@
 from hypothesis import given, settings
 from hypothesis.strategies import dates, datetimes, timedeltas, times
 
-from tests.common.debug import find_any, minimal
+from tests.common.debug import assert_simple_property, find_any, minimal
 
 
 def test_can_find_positive_delta():
@@ -51,7 +51,7 @@ def test_max_value_is_respected():
 
 @given(timedeltas())
 def test_single_timedelta(val):
-    assert find_any(timedeltas(val, val)) is val
+    assert_simple_property(timedeltas(val, val), lambda v: v is val)
 
 
 def test_simplifies_towards_millenium():