Merge remote-tracking branch 'upstream/master' into next

# Conflicts:
#	hypothesis/RELEASE.rst

Author: Liam-DeVoe

AUTHORS.rst

@@ -84,6 +84,7 @@ their individual contributions.
 * `Hal Blackburn <https://github.com/h4l>`_
 * `Hugo van Kemenade <https://github.com/hugovk>`_
 * `Humberto Rocha <https://github.com/humrochagf>`_
+* `Ian Hunt-Isaak <https://github.com/ianhi>`_
 * `Ilya Lebedev <https://github.com/melevir>`_ (melevir@gmail.com)
 * `Israel Fruchter <https://github.com/fruch>`_
 * `Ivan Tham <https://github.com/pickfire>`_

hypothesis/RELEASE.rst

@@ -1,3 +1,12 @@
 RELEASE_TYPE: minor
 
 |event|'s ``payload`` is now typed as accepting |Any|, matching its runtime behavior of accepting any string-coercible object.
+
+When Hypothesis detects that your data generation is flaky and raises
+|FlakyStrategyDefinition|, the error message now describes *what* differed
+between the two runs - such as a different choice type, different constraints,
+or drawing more or less data - as well as the stack of strategies being drawn
+from, instead of only reporting that generation was inconsistent. In stateful
+tests, it also reports the steps leading up to the error.
+
+Thanks to Ian Hunt-Isaak for this improvement!

hypothesis/docs/changelog.rst

@@ -24,13 +24,26 @@ Hypothesis 6.x
 6.152.11 - 2026-05-26
 ---------------------
 
-This patch adds support for recursive forward references in
-:func:`~hypothesis.strategies.from_type`, such as
-``A = list[Union["A", str]]`` (:issue:`4542`).
-Previously, such recursive type aliases would raise a ``ResolutionFailed``
-error. Now, Hypothesis can automatically resolve the forward reference
-by looking it up in the caller's namespace. This also resolves forward
-references inside ``type[...]``, such as ``type["MyClass"]``.
+This patch adds support for resolving forward references in |st.from_type|
+(:issue:`4542`):
+
+.. code-block:: python
+
+    # this now works
+    A = list[Union["A", str]]
+    st.from_type(A).example()
+
+    # this also now works
+    s = st.from_type(list["B"])
+
+    @dataclass
+    class B:
+        v: int
+
+    s.example()
+
+Previously, these would raise an error. Now, Hypothesis automatically resolves
+the forward reference by looking it up in the caller's namespace.
 
 .. _v6.152.10:
 

hypothesis/docs/explanation/domain.rst

@@ -45,5 +45,6 @@ The test case distribution remains an active area of research and development, a
 * some are statically designed into strategies - for example, |st.integers| upweights range endpoints, and samples from a mixed distribution over integer bit-widths.
 * some are dynamic features of the engine - like replaying prior examples with subsections of the input 'cloned' or otherwise altered, for bugs which trigger only when different fields have the same value (which is otherwise exponentially unlikely).
 * some vary depending on the code under test - we collect interesting-looking constants from imported source files as seeds for test cases.
+* `swarm testing <https://www.cs.utah.edu/~regehr/papers/swarm12.pdf>`__ adds further randomization when choosing which rules to execute in stateful testing.
 
 And as if that wasn't enough, :ref:`alternative backends <alternative-backends>` can radically change the distribution again - for example :pypi:`hypofuzz` uses runtime feedback to modify the distribution of inputs as the test runs, to maximize the rate at which we trigger new behaviors in that particular test and code.  If Hypothesis' defaults aren't strong enough, we recommend trying Hypofuzz!

hypothesis/src/hypothesis/errors.py

@@ -8,11 +8,17 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+from collections.abc import Mapping
 from datetime import timedelta
-from typing import Any, Literal
+from typing import TYPE_CHECKING, Any, Literal
 
 from hypothesis.internal.compat import ExceptionGroup
 
+if TYPE_CHECKING:
+    from hypothesis.internal.conjecture.choice import ChoiceConstraintsT
+else:
+    ChoiceConstraintsT = Mapping
+
 
 class HypothesisException(Exception):
     """Generic parent class for exceptions thrown by Hypothesis."""
@@ -85,6 +91,13 @@ def __init__(self, reason, interesting_origins=None):
         self._interesting_origins = interesting_origins
 
 
+def _render_constraints(show: Mapping[str, object], other: Mapping[str, object]) -> str:
+    assert show.keys() == other.keys()
+    return ", ".join(
+        f"{k}={'...' if v == other[k] else repr(v)}" for k, v in show.items()
+    )
+
+
 class FlakyStrategyDefinition(Flaky):
     """
     This function appears to cause inconsistent data generation.
@@ -99,6 +112,39 @@ class FlakyStrategyDefinition(Flaky):
         See also the :doc:`flaky failures tutorial </tutorial/flaky>`.
     """
 
+    _BASE_MESSAGE = (
+        "Inconsistent data generation! Data generation behaved differently "
+        "between test cases. Is your data generation depending on external "
+        "state?"
+    )
+
+    @classmethod
+    def with_detail(cls, detail: str) -> "FlakyStrategyDefinition":
+        return cls(f"{cls._BASE_MESSAGE}\n\n{detail}")
+
+    @classmethod
+    def from_mismatch(
+        cls,
+        expected_type: str,
+        expected_constraints: ChoiceConstraintsT,
+        actual_type: str,
+        actual_constraints: ChoiceConstraintsT,
+    ) -> "FlakyStrategyDefinition":
+        if actual_type != expected_type:
+            detail = (
+                "The second test case drew a different type of value than the first.\n"
+                f"  first:  {expected_type}\n"
+                f"  second: {actual_type}\n"
+            )
+        else:
+            detail = (
+                f"The second test case drew type {actual_type} with different constraints "
+                "than the first.\n"
+                f"  first:  {_render_constraints(expected_constraints, actual_constraints)}\n"
+                f"  second: {_render_constraints(actual_constraints, expected_constraints)}\n"
+            )
+        return cls.with_detail(detail)
+
 
 class _WrappedBaseException(Exception):
     """Used internally for wrapping BaseExceptions as components of FlakyFailure."""

hypothesis/src/hypothesis/internal/conjecture/data.py

@@ -30,6 +30,7 @@
 from hypothesis.errors import (
     CannotProceedScopeT,
     ChoiceTooLarge,
+    FlakyStrategyDefinition,
     Frozen,
     InvalidArgument,
     StopTest,
@@ -1202,7 +1203,15 @@ def draw(
         self.start_span(label=label)
         try:
             if not at_top_level:
-                return unwrapped.do_draw(self)
+                try:
+                    return unwrapped.do_draw(self)
+                except FlakyStrategyDefinition as err:
+                    # Record the strategy stack as the error unwinds, so that an
+                    # inconsistent-generation failure is explained in terms of the
+                    # strategies being drawn from, not just the choice sequence.
+                    # The top-level draw adds its own "while generating ..." note.
+                    add_note(err, f"while drawing from {strategy!r}")
+                    raise
             assert start_time is not None
             key = observe_as or f"generate:unlabeled_{len(self.draw_times)}"
             try:

hypothesis/src/hypothesis/internal/conjecture/datatree.py

@@ -53,13 +53,6 @@ class PreviouslyUnseenBehaviour(HypothesisException):
     pass
 
 
-_FLAKY_STRAT_MSG = (
-    "Inconsistent data generation! Data generation behaved differently "
-    "between different runs. Is your data generation depending on external "
-    "state?"
-)
-
-
 EMPTY: frozenset[int] = frozenset()
 
 
@@ -442,7 +435,7 @@ def mark_forced(self, i: int) -> None:
             self.__forced = set()
         self.__forced.add(i)
 
-    def split_at(self, i: int) -> None:
+    def split_at(self, i: int, *, new_value: object = None) -> None:
         """
         Splits the tree so that it can incorporate a decision at the draw call
         corresponding to the node at position i.
@@ -451,7 +444,11 @@ def split_at(self, i: int) -> None:
         """
 
         if i in self.forced:
-            raise FlakyStrategyDefinition(_FLAKY_STRAT_MSG)
+            raise FlakyStrategyDefinition.with_detail(
+                f"The {self.choice_types[i]} value was forced to "
+                f"{self.values[i]!r} in the first run, but the second run "
+                f"drew {new_value!r}.\n"
+            )
 
         assert not self.is_exhausted
 
@@ -1050,17 +1047,25 @@ def draw_value(
                 choice_type != node.choice_types[i]
                 or constraints != node.constraints[i]
             ):
-                raise FlakyStrategyDefinition(_FLAKY_STRAT_MSG)
+                raise FlakyStrategyDefinition.from_mismatch(
+                    node.choice_types[i],
+                    node.constraints[i],
+                    choice_type,
+                    constraints,
+                )
             # Note that we don't check whether a previously
             # forced value is now free. That will be caught
             # if we ever split the node there, but otherwise
             # may pass silently. This is acceptable because it
             # means we skip a hash set lookup on every
             # draw and that's a pretty niche failure mode.
             if was_forced and i not in node.forced:
-                raise FlakyStrategyDefinition(_FLAKY_STRAT_MSG)
+                raise FlakyStrategyDefinition.with_detail(
+                    f"The {choice_type} value was forced to a specific value "
+                    f"but was not forced on the first run.\n"
+                )
             if value != node.values[i]:
-                node.split_at(i)
+                node.split_at(i, new_value=value)
                 assert i == len(node.values)
                 new_node = TreeNode()
                 assert isinstance(node.transition, Branch)
@@ -1095,19 +1100,26 @@ def draw_value(
                     compute_max_children(choice_type, constraints) == 1
                     and not was_forced
                 ):
-                    node.split_at(i)
+                    node.split_at(i, new_value=value)
                     assert isinstance(node.transition, Branch)
                     self._current_node = node.transition.children[value]
                     self._index_in_current_node = 0
             elif isinstance(trans, Conclusion):
                 assert trans.status != Status.OVERRUN
                 # We tried to draw where history says we should have
                 # stopped
-                raise FlakyStrategyDefinition(_FLAKY_STRAT_MSG)
+                raise FlakyStrategyDefinition.with_detail(
+                    "The second run drew more data than the first run.\n"
+                )
             else:
                 assert isinstance(trans, Branch), trans
                 if choice_type != trans.choice_type or constraints != trans.constraints:
-                    raise FlakyStrategyDefinition(_FLAKY_STRAT_MSG)
+                    raise FlakyStrategyDefinition.from_mismatch(
+                        trans.choice_type,
+                        trans.constraints,
+                        choice_type,
+                        constraints,
+                    )
                 try:
                     self._current_node = trans.children[value]
                 except KeyError:
@@ -1127,7 +1139,10 @@ def kill_branch(self) -> None:
             self._current_node.transition is not None
             and not isinstance(self._current_node.transition, Killed)
         ):
-            raise FlakyStrategyDefinition(_FLAKY_STRAT_MSG)
+            raise FlakyStrategyDefinition.with_detail(
+                "The second run stopped drawing earlier than the first run, "
+                "which continued to draw more data.\n"
+            )
 
         if self._current_node.transition is None:
             self._current_node.transition = Killed(TreeNode())
@@ -1148,7 +1163,10 @@ def conclude_test(
         node = self._current_node
 
         if i < len(node.values) or isinstance(node.transition, Branch):
-            raise FlakyStrategyDefinition(_FLAKY_STRAT_MSG)
+            raise FlakyStrategyDefinition.with_detail(
+                "The second run stopped drawing earlier than the first run, "
+                "which continued to draw more data.\n"
+            )
 
         new_transition = Conclusion(status, interesting_origin)
 

hypothesis/src/hypothesis/internal/conjecture/engine.py

@@ -27,6 +27,7 @@
 from hypothesis.errors import (
     BackendCannotProceed,
     FlakyBackendFailure,
+    FlakyStrategyDefinition,
     HypothesisException,
     InvalidArgument,
     StopTest,
@@ -562,8 +563,15 @@ def _backend_cannot_proceed(
             interrupted = True
             data.freeze()
             return
-        except BaseException:
+        except BaseException as err:
             data.freeze()
+            if isinstance(err, FlakyStrategyDefinition) and data._stateful_repr_parts:
+                # In a stateful test, surface the steps leading up to the
+                # inconsistency.
+                report(
+                    "Steps leading up to this error:\n"
+                    + "\n".join(f"  {s}" for s in data._stateful_repr_parts)
+                )
             if self.settings.backend != "hypothesis":
                 try:
                     realize_choices(data, for_failure=True)

hypothesis/src/hypothesis/stateful.py

@@ -132,8 +132,7 @@ def run_state_machine(data):
         def output(s):
             if print_steps:
                 report(s)
-            if observability_enabled():
-                cd._stateful_repr_parts.append(s)
+            cd._stateful_repr_parts.append(s)
 
         try:
             output(f"state = {machine.__class__.__name__}()")

hypothesis/tests/conjecture/test_data_tree.py

@@ -226,7 +226,7 @@ def test_concluding_at_prefix_is_flaky():
         data.conclude_test(Status.INTERESTING)
 
     data = ConjectureData.for_choices([], observer=tree.new_observer())
-    with pytest.raises(Flaky):
+    with pytest.raises(Flaky, match="stopped drawing earlier"):
         data.conclude_test(Status.INVALID)
 
 
@@ -250,7 +250,7 @@ def test_changing_n_bits_is_flaky_in_prefix():
         data.conclude_test(Status.INTERESTING)
 
     data = ConjectureData.for_choices((1,), observer=tree.new_observer())
-    with pytest.raises(Flaky):
+    with pytest.raises(Flaky, match="different constraints"):
         data.draw_integer(0, 3)
 
 
@@ -264,7 +264,7 @@ def test_changing_n_bits_is_flaky_in_branch():
             data.conclude_test(Status.INTERESTING)
 
     data = ConjectureData.for_choices((1,), observer=tree.new_observer())
-    with pytest.raises(Flaky):
+    with pytest.raises(Flaky, match="different constraints"):
         data.draw_integer(0, 3)
 
 
@@ -278,7 +278,7 @@ def test_extending_past_conclusion_is_flaky():
     data = ConjectureData.for_choices((True, False), observer=tree.new_observer())
     data.draw_boolean()
 
-    with pytest.raises(Flaky):
+    with pytest.raises(Flaky, match="more data"):
         data.draw_boolean()
 
 
@@ -291,7 +291,7 @@ def test_changing_to_forced_is_flaky():
 
     data = ConjectureData.for_choices((True, False), observer=tree.new_observer())
 
-    with pytest.raises(Flaky):
+    with pytest.raises(Flaky, match="was not forced on the first run"):
         data.draw_boolean(forced=True)
 
 
@@ -304,10 +304,22 @@ def test_changing_value_of_forced_is_flaky():
 
     data = ConjectureData.for_choices((True, False), observer=tree.new_observer())
 
-    with pytest.raises(Flaky):
+    with pytest.raises(Flaky, match=r"forced to True.*drew False"):
         data.draw_boolean(forced=False)
 
 
+def test_drawing_different_type_is_flaky():
+    tree = DataTree()
+    data = ConjectureData.for_choices((1,), observer=tree.new_observer())
+    data.draw_integer(0, 1)
+    with pytest.raises(StopTest):
+        data.conclude_test(Status.INTERESTING)
+
+    data = ConjectureData.for_choices((True,), observer=tree.new_observer())
+    with pytest.raises(Flaky, match="different type"):
+        data.draw_boolean()
+
+
 def test_does_not_truncate_if_unseen():
     tree = DataTree()
     nodes = (1, 2, 3, 4)