diff --git a/hypothesis-python/tests/pytest/test_no_plugin_side_effects.py b/hypothesis-python/tests/pytest/test_no_plugin_side_effects.py
new file mode 100644
index 0000000000..65f1e9ec74
--- /dev/null
+++ b/hypothesis-python/tests/pytest/test_no_plugin_side_effects.py
@@ -0,0 +1,49 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2021 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+import os
+import subprocess
+
+DOES_NOT_WRITE_CHARMAP = """
+from hypothesis import given, strategies as st
+
+def test():
+    pass
+"""
+WRITES_CHARMAP = """
+from hypothesis import given, strategies as st
+
+@given(st.text())
+def test(s):
+    pass
+"""
+
+
+def names(path):
+    return {p.name for p in path.iterdir() if "cache" not in p.name}
+
+
+def test_does_not_create_dir_unless_actually_using_hypothesis(tmp_path):
+    pyfile = tmp_path / "test.py"
+    env = {**os.environ, "HYPOTHESIS_STORAGE_DIRECTORY": ".hypothesis"}
+    expected = {"test.py", ".pytest_cache"}
+
+    pyfile.write_text(DOES_NOT_WRITE_CHARMAP)
+    subprocess.check_call(["pytest", "test.py"], env=env, cwd=tmp_path)
+    assert names(tmp_path) == {"test.py"}
+
+    pyfile.write_text(WRITES_CHARMAP)
+    subprocess.check_call(["pytest", "test.py"], env=env, cwd=tmp_path)
+    assert names(tmp_path) == {"test.py", ".hypothesis"}