test: if a test fails randomly, let it retry with @flaky

tests/helpers.py

@@ -23,6 +23,7 @@
     TypeVar, Union, cast,
 )
 
+import flaky
 import pytest
 
 from coverage import env
@@ -397,3 +398,12 @@ def __init__(self, options: Iterable[str]) -> None:
     def get_output(self) -> str:
         """Get the output text from the `DebugControl`."""
         return self.io.getvalue()
+
+
+TestMethod = Callable[[Any], None]
+
+def flaky_method(max_runs: int) -> Callable[[TestMethod], TestMethod]:
+    """flaky.flaky, but with type annotations."""
+    def _decorator(fn: TestMethod) -> TestMethod:
+        return cast(TestMethod, flaky.flaky(max_runs)(fn))
+    return _decorator

tests/test_concurrency.py

@@ -30,6 +30,7 @@
 
 from tests import testenv
 from tests.coveragetest import CoverageTest
+from tests.helpers import flaky_method
 
 
 # These libraries aren't always available, we'll skip tests if they aren't.
@@ -317,6 +318,7 @@ def do():
             """
         self.try_some_code(BUG_330, "eventlet", eventlet, "0\n")
 
+    @flaky_method(max_runs=3)   # Sometimes a test fails due to inherent randomness. Try more times.
     def test_threads_with_gevent(self) -> None:
         self.make_file("both.py", """\
             import queue