From 432ebd1fd45119c6fdc1427f66bf79109c050622 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Max=20G=C3=B6rner?= <max@familie-goerner.eu>
Date: Wed, 2 Sep 2020 15:07:47 +0200
Subject: [PATCH] Add property-based fuzz test

A test is added that runs mccabe against plenty of random yet valid
Python source code. Right now, the test does not do much - it only tests
whether mccabe accepts the code. Already this is an achievement because
now weird edge cases are covered.

Having this test suite integrated will also allow to build up on that.
The author of Hypothesis claimed to work on a coverage guided test
runner. Also more general properties to hold on any valid source code
now can be tested.
---
 .gitignore     |  1 +
 test_mccabe.py | 38 ++++++++++++++++++++++++++++++++++++++
 tox.ini        |  2 ++
 3 files changed, 41 insertions(+)

diff --git a/.gitignore b/.gitignore
index 413f6e2..69504d4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 *.egg
 *.egg-info
 *.py[cod]
+.hypothesis/
 .tox
 dist
diff --git a/test_mccabe.py b/test_mccabe.py
index b53db6a..fbf2650 100644
--- a/test_mccabe.py
+++ b/test_mccabe.py
@@ -1,11 +1,14 @@
 import unittest
 import sys
+
 try:
     from StringIO import StringIO
 except ImportError:
     from io import StringIO
 
 import pytest
+import hypothesmith
+from hypothesis import HealthCheck, given, settings, strategies as st
 
 import mccabe
 from mccabe import get_code_complexity
@@ -233,5 +236,40 @@ def test_get_module_complexity(self):
         self.assertEqual(0, mccabe.get_module_complexity("mccabe.py"))
 
 
+# This test uses the Hypothesis and Hypothesmith libraries to generate random
+# syntatically-valid Python source code and applies McCabe on it.
+@settings(
+    max_examples=1000,  # roughly 1k tests/minute, or half that under coverage
+    derandomize=False,  # deterministic mode to avoid CI flakiness
+    deadline=None,  # ignore Hypothesis' health checks; we already know that
+    suppress_health_check=HealthCheck.all(),  # this is slow and filter-heavy.
+)
+@given(
+    # Note that while Hypothesmith might generate code unlike that written by
+    # humans, it's a general test that should pass for any *valid* source code.
+    # (so e.g. running it against code scraped of the internet might also help)
+    src_contents=hypothesmith.from_grammar() | hypothesmith.from_node(),
+    max_complexity=st.integers(min_value=1),
+)
+def test_idempotent_any_syntatically_valid_python(
+    src_contents: str, max_complexity: int
+) -> None:
+    """Property-based tests for mccabe.
+
+    This test case is based on a similar test for Black, the code formatter.
+    Black's test was written by Zac Hatfield-Dodds, the author of Hypothesis
+    and the Hypothesmith tool for source code generation.  You can run this
+    file with `python`, `pytest`, or (soon) a coverage-guided fuzzer Zac is
+    working on.
+    """
+
+    # Before starting, let's confirm that the input string is valid Python:
+    compile(src_contents, "<string>", "exec")  # else the bug is in hypothesmith
+
+    # Then try to apply get_complexity_number to the code...
+    get_code_complexity(src_contents, max_complexity)
+
+
 if __name__ == "__main__":
+    test_idempotent_any_syntatically_valid_python()
     unittest.main()
diff --git a/tox.ini b/tox.ini
index d74138a..0ea8c13 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,6 +5,8 @@ envlist =
 [testenv]
 deps =
     pytest
+    hypothesis
+    hypothesmith
 commands =
     pytest