Use pytest-perf to capture performance expectations and regressions

.coveragerc

@@ -5,6 +5,7 @@ omit =
 	tests/*
 	prepare/*
 	*/_itertools.py
+	exercises.py
 
 [report]
 show_missing = True

.github/workflows/main.yml

@@ -24,22 +24,6 @@ jobs:
       - name: Run tests
         run: tox
 
-  benchmark:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - name: Install tox
-        run: |
-          python -m pip install tox
-      - name: Run benchmarks
-        run: tox
-        env:
-          TOXENV: perf{,-ref}
-
   diffcov:
     runs-on: ubuntu-latest
     steps:

exercises.py

@@ -0,0 +1,36 @@
+from pytest_perf.deco import extras
+
+
+@extras('perf')
+def discovery_perf():
+    "discovery"
+    import importlib_metadata  # end warmup
+
+    importlib_metadata.distribution('ipython')
+
+
+def entry_points_perf():
+    "entry_points()"
+    import importlib_metadata  # end warmup
+
+    importlib_metadata.entry_points()
+
+
+@extras('perf')
+def cached_distribution_perf():
+    "cached distribution"
+    import importlib_metadata
+
+    importlib_metadata.distribution('ipython')  # end warmup
+    importlib_metadata.distribution('ipython')
+
+
+@extras('perf')
+def uncached_distribution_perf():
+    "uncached distribution"
+    import importlib
+    import importlib_metadata
+
+    # end warmup
+    importlib.invalidate_caches()
+    importlib_metadata.distribution('ipython')

setup.cfg

@@ -51,6 +51,7 @@ testing =
 	pep517
 	pyfakefs
 	flufl.flake8
+	pytest-perf >= 0.9.2
 
 docs =
 	# upstream

tox.ini

@@ -10,6 +10,8 @@ toxworkdir={env:TOX_WORK_DIR:.tox}
 deps =
 commands =
 	pytest {posargs}
+passenv =
+	HOME
 usedevelop = True
 extras = testing
 setenv =
@@ -33,45 +35,6 @@ commands =
 	diff-cover coverage.xml --compare-branch=origin/main --html-report diffcov.html
 	diff-cover coverage.xml --compare-branch=origin/main --fail-under=100
 
-[perf]
-deps =
-	ipython
-	pyperf
-	path
-commands =
-	python -c "import path; path.Path('{env:SCOPE}.json').remove_p()"
-
-	python -m pyperf timeit --name discovery --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.distribution("ipython")'
-
-	python -m pyperf timeit --name 'entry_points()' --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.entry_points()'
-
-	python -m pyperf timeit --name 'cached distribution' --append {env:SCOPE}.json -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' 'importlib_metadata.distribution("ipython")'
-
-	python -m pyperf timeit --name 'uncached distribution' --append {env:SCOPE}.json -s 'import importlib, importlib_metadata' 'importlib.invalidate_caches(); importlib_metadata.distribution("ipython")'
-
-[testenv:perf]
-use_develop = False
-# change dir to avoid picking up local package.
-changedir = {toxworkdir}
-setenv =
-	SCOPE = local
-deps = {[perf]deps}
-commands = {[perf]commands}
-
-[testenv:perf-ref]
-# compare perf results to the main branch
-skip_install = True
-# change dir to avoid picking up local package.
-changedir = {toxworkdir}
-setenv =
-	SCOPE = main
-deps =
-	{[perf]deps}
-	git+https://github.com/python/importlib_metadata
-commands =
-	{[perf]commands}
-	python -m pyperf compare_to --verbose main.json local.json --table
-
 [testenv:release]
 skip_install = True
 deps =