From aedbde938a7d88436551bdbf05020d552ef060a8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 21 Apr 2024 23:44:09 +0200 Subject: [PATCH] tests: speedups, lowest-version, ... (#6812) --- .github/workflows/main.yml | 16 ++- mitmproxy/net/local_ip.py | 8 +- mitmproxy/proxy/mode_servers.py | 12 ++- pyproject.toml | 116 ++++++++++++++++++--- setup.cfg | 31 ------ test/conftest.py | 2 - test/full_coverage_plugin.py | 144 -------------------------- test/individual_coverage.py | 172 ++++++++++++++++---------------- test/mitmproxy/test_options.py | 6 +- 9 files changed, 221 insertions(+), 286 deletions(-) delete mode 100644 setup.cfg delete mode 100644 test/full_coverage_plugin.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a42c4469ee..603272c698 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,6 +77,19 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml + test-old-dependencies: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version-file: .github/python-version.txt + - run: pip install tox-uv + - run: tox -e old-dependencies + build: strategy: fail-fast: false @@ -208,8 +221,9 @@ jobs: - mypy - individual-coverage - test - - build + - test-old-dependencies - test-web-ui + - build - docs uses: mhils/workflows/.github/workflows/alls-green.yml@main with: diff --git a/mitmproxy/net/local_ip.py b/mitmproxy/net/local_ip.py index bc3087263c..f6183a886c 100644 --- a/mitmproxy/net/local_ip.py +++ b/mitmproxy/net/local_ip.py @@ -13,9 +13,9 @@ def get_local_ip(reachable: str = "8.8.8.8") -> str | None: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: s.connect((reachable, 80)) - return s.getsockname()[0] + return s.getsockname()[0] # pragma: no cover except OSError: - return None + return None # pragma: no cover finally: s.close() @@ -29,8 +29,8 @@ def get_local_ip6(reachable: str = "2001:4860:4860::8888") -> str | None: s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) try: s.connect((reachable, 80)) - return s.getsockname()[0] + return s.getsockname()[0] # pragma: no cover except OSError: - return None + return None # pragma: no cover finally: s.close() diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index 8919776e88..921789a0a0 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -204,7 +204,9 @@ async def handle_stream( else: handler.layer.context.client.sockname = original_dst handler.layer.context.server.address = original_dst - elif isinstance(self.mode, (mode_specs.WireGuardMode, mode_specs.LocalMode)): + elif isinstance( + self.mode, (mode_specs.WireGuardMode, mode_specs.LocalMode) + ): # pragma: no cover on platforms without wg-test-client handler.layer.context.server.address = writer.get_extra_info( "remote_endpoint", handler.layer.context.client.sockname ) @@ -325,7 +327,9 @@ class WireGuardServerInstance(ServerInstance[mode_specs.WireGuardMode]): server_key: str client_key: str - def make_top_layer(self, context: Context) -> Layer: + def make_top_layer( + self, context: Context + ) -> Layer: # pragma: no cover on platforms without wg-test-client return layers.modes.TransparentProxy(context) @property @@ -418,7 +422,9 @@ async def _stop(self) -> None: finally: self._server = None - async def wg_handle_stream(self, stream: mitmproxy_rs.Stream) -> None: + async def wg_handle_stream( + self, stream: mitmproxy_rs.Stream + ) -> None: # pragma: no cover on platforms without wg-test-client await self.handle_stream(stream, stream) diff --git a/pyproject.toml b/pyproject.toml index 7b332e52c9..0441092142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "Brotli>=1.0,<1.2", "certifi>=2019.9.11", # no semver here - this should always be on the last release! "cryptography>=42.0,<42.1", - "flask>=1.1.1,<3.1", + "flask>=3.0,<3.1", "h11>=0.11,<0.15", "h2>=4.1,<5", "hyperframe>=6.0,<7", @@ -57,7 +57,7 @@ dependencies = [ "urwid-mitmproxy>=2.1.1,<2.2", "wsproto>=1.0,<1.3", "publicsuffix2>=2.20190812,<3", - "zstandard>=0.11,<0.23", + "zstandard>=0.15,<0.23", ] [project.optional-dependencies] @@ -66,11 +66,11 @@ dev = [ "hypothesis>=5.8,<7", "pdoc>=4.0.0", "pyinstaller==6.5.0", - "pytest-asyncio>=0.23,<0.24", - "pytest-cov>=2.7.1,<5.1", - "pytest-timeout>=1.3.3,<2.4", - "pytest-xdist>=2.1.0,<3.6", - "pytest>=6.1.0,<9", + "pytest-asyncio>=0.23.6,<0.24", + "pytest-cov>=5.0.0,<5.1", + "pytest-timeout>=2.3.1,<2.4", + "pytest-xdist>=3.6.0,<3.7", + "pytest>=8.1.1,<9", "requests>=2.9.1,<3", "tox>=3.5,<5", "wheel>=0.36.2,<0.44", @@ -137,6 +137,93 @@ filterwarnings = [ "default:coroutine 'ConnectionHandler.hook_task' was never awaited:RuntimeWarning", ] +[tool.pytest.individual_coverage] +exclude = [ + "mitmproxy/addons/__init__.py", + "mitmproxy/addons/onboarding.py", + "mitmproxy/addons/onboardingapp/__init__.py", + "mitmproxy/contentviews/__init__.py", + "mitmproxy/contentviews/base.py", + "mitmproxy/contentviews/grpc.py", + "mitmproxy/contentviews/image/__init__.py", + "mitmproxy/contrib/*", + "mitmproxy/ctx.py", + "mitmproxy/exceptions.py", + "mitmproxy/flow.py", + "mitmproxy/io/__init__.py", + "mitmproxy/io/io.py", + "mitmproxy/io/tnetstring.py", + "mitmproxy/log.py", + "mitmproxy/master.py", + "mitmproxy/net/check.py", + "mitmproxy/net/http/cookies.py", + "mitmproxy/net/http/http1/__init__.py", + "mitmproxy/net/http/multipart.py", + "mitmproxy/net/tls.py", + "mitmproxy/platform/__init__.py", + "mitmproxy/platform/linux.py", + "mitmproxy/platform/openbsd.py", + "mitmproxy/platform/osx.py", + "mitmproxy/platform/pf.py", + "mitmproxy/platform/windows.py", + "mitmproxy/proxy/__init__.py", + "mitmproxy/proxy/layers/__init__.py", + "mitmproxy/proxy/layers/http/__init__.py", + "mitmproxy/proxy/layers/http/_base.py", + "mitmproxy/proxy/layers/http/_events.py", + "mitmproxy/proxy/layers/http/_hooks.py", + "mitmproxy/proxy/layers/http/_http1.py", + "mitmproxy/proxy/layers/http/_http2.py", + "mitmproxy/proxy/layers/http/_http3.py", + "mitmproxy/proxy/layers/http/_http_h2.py", + "mitmproxy/proxy/layers/http/_http_h3.py", + "mitmproxy/proxy/layers/http/_upstream_proxy.py", + "mitmproxy/proxy/layers/tls.py", + "mitmproxy/proxy/server.py", + "mitmproxy/script/__init__.py", + "mitmproxy/test/taddons.py", + "mitmproxy/test/tflow.py", + "mitmproxy/test/tutils.py", + "mitmproxy/tools/console/__init__.py", + "mitmproxy/tools/console/commander/commander.py", + "mitmproxy/tools/console/commandexecutor.py", + "mitmproxy/tools/console/commands.py", + "mitmproxy/tools/console/common.py", + "mitmproxy/tools/console/consoleaddons.py", + "mitmproxy/tools/console/eventlog.py", + "mitmproxy/tools/console/flowdetailview.py", + "mitmproxy/tools/console/flowlist.py", + "mitmproxy/tools/console/flowview.py", + "mitmproxy/tools/console/grideditor/__init__.py", + "mitmproxy/tools/console/grideditor/base.py", + "mitmproxy/tools/console/grideditor/col_bytes.py", + "mitmproxy/tools/console/grideditor/col_subgrid.py", + "mitmproxy/tools/console/grideditor/col_text.py", + "mitmproxy/tools/console/grideditor/col_viewany.py", + "mitmproxy/tools/console/grideditor/editors.py", + "mitmproxy/tools/console/help.py", + "mitmproxy/tools/console/keybindings.py", + "mitmproxy/tools/console/keymap.py", + "mitmproxy/tools/console/layoutwidget.py", + "mitmproxy/tools/console/master.py", + "mitmproxy/tools/console/options.py", + "mitmproxy/tools/console/overlay.py", + "mitmproxy/tools/console/quickhelp.py", + "mitmproxy/tools/console/searchable.py", + "mitmproxy/tools/console/signals.py", + "mitmproxy/tools/console/statusbar.py", + "mitmproxy/tools/console/tabs.py", + "mitmproxy/tools/console/window.py", + "mitmproxy/tools/main.py", + "mitmproxy/tools/web/__init__.py", + "mitmproxy/tools/web/app.py", + "mitmproxy/tools/web/master.py", + "mitmproxy/tools/web/webaddons.py", + "mitmproxy/utils/bits.py", + "mitmproxy/utils/pyinstaller/*", + "mitmproxy/utils/vt_codes.py", +] + [tool.mypy] check_untyped_defs = true ignore_missing_imports = true @@ -164,12 +251,14 @@ module = "test.*" ignore_errors = true [tool.ruff] -select = ["E", "F", "I"] extend-exclude = ["mitmproxy/contrib/"] + +[tool.ruff.lint] +select = ["E", "F", "I"] ignore = ["F541", "E501"] -[tool.ruff.isort] +[tool.ruff.lint.isort] # these rules are a bit weird, but they mimic our existing reorder_python_imports style. # if we break compatibility here, consider removing all customization + enforce absolute imports. force-single-line = true @@ -191,15 +280,18 @@ deps = setenv = HOME = {envtmpdir} commands = mitmdump --version - pytest --timeout 60 -vv --cov-report xml \ + pytest --timeout 60 -vv \ + --cov-report xml \ --continue-on-collection-errors \ --cov=mitmproxy --cov=release \ - --full-cov=mitmproxy/ \ {posargs} +[testenv:old-dependencies] +uv_resolution = lowest-direct + [testenv:lint] deps = - ruff>=0.1.3,<0.2 + ruff>=0.4.1,<0.5 commands = ruff . diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 8cd2a7ab80..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,31 +0,0 @@ -[tool:full_coverage] -exclude = - mitmproxy/tools/ - release/hooks - -[tool:individual_coverage] -exclude = - mitmproxy/addons/onboarding.py - mitmproxy/connections.py - mitmproxy/contentviews/base.py - mitmproxy/contentviews/grpc.py - mitmproxy/ctx.py - mitmproxy/exceptions.py - mitmproxy/flow.py - mitmproxy/io/io.py - mitmproxy/io/tnetstring.py - mitmproxy/log.py - mitmproxy/master.py - mitmproxy/net/check.py - mitmproxy/net/http/cookies.py - mitmproxy/net/http/message.py - mitmproxy/net/http/multipart.py - mitmproxy/net/tls.py - mitmproxy/net/udp_wireguard.py - mitmproxy/options.py - mitmproxy/proxy/config.py - mitmproxy/proxy/server.py - mitmproxy/proxy/layers/tls.py - mitmproxy/utils/bits.py - mitmproxy/utils/vt_codes.py - mitmproxy/utils/pyinstaller diff --git a/test/conftest.py b/test/conftest.py index b002c450bf..0d8fb23dfa 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -9,8 +9,6 @@ from mitmproxy.utils import data -pytest_plugins = ("test.full_coverage_plugin",) - skip_windows = pytest.mark.skipif(os.name == "nt", reason="Skipping due to Windows") skip_not_windows = pytest.mark.skipif( diff --git a/test/full_coverage_plugin.py b/test/full_coverage_plugin.py deleted file mode 100644 index 36ac7f263d..0000000000 --- a/test/full_coverage_plugin.py +++ /dev/null @@ -1,144 +0,0 @@ -import configparser -import os -import sys - -import pytest - -here = os.path.abspath(os.path.dirname(__file__)) - - -enable_coverage = False -coverage_values = [] -coverage_passed = True -no_full_cov = [] - - -def pytest_addoption(parser): - parser.addoption( - "--full-cov", - action="append", - dest="full_cov", - default=[], - help="Require full test coverage of 100%% for this module/path/filename (multi-allowed). Default: none", - ) - - parser.addoption( - "--no-full-cov", - action="append", - dest="no_full_cov", - default=[], - help="Exclude file from a parent 100%% coverage requirement (multi-allowed). Default: none", - ) - - -def pytest_configure(config): - global enable_coverage - global no_full_cov - - enable_coverage = ( - config.getoption("file_or_dir") - and len(config.getoption("file_or_dir")) == 0 - and config.getoption("full_cov") - and len(config.getoption("full_cov")) > 0 - and config.pluginmanager.getplugin("_cov") is not None - and config.pluginmanager.getplugin("_cov").cov_controller is not None - and config.pluginmanager.getplugin("_cov").cov_controller.cov is not None - ) - - c = configparser.ConfigParser() - c.read(os.path.join(here, "..", "setup.cfg")) - fs = c["tool:full_coverage"]["exclude"].split("\n") - no_full_cov = config.option.no_full_cov + [f.strip() for f in fs] - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtestloop(session): - global enable_coverage - global coverage_values - global coverage_passed - global no_full_cov - - if not enable_coverage: - yield - return - - cov = session.config.pluginmanager.getplugin("_cov").cov_controller.cov - - if os.name == "nt": - cov.exclude("pragma: windows no cover") - - if sys.platform == "darwin": - cov.exclude("pragma: osx no cover") - - if os.environ.get("OPENSSL") == "old": - cov.exclude("pragma: openssl-old no cover") - - yield - - coverage_values = {name: 0 for name in session.config.option.full_cov} - - prefix = os.getcwd() - - excluded_files = [os.path.normpath(f) for f in no_full_cov] - measured_files = [ - os.path.normpath(os.path.relpath(f, prefix)) - for f in cov.get_data().measured_files() - ] - measured_files = [ - f - for f in measured_files - if not any(f.startswith(excluded_f) for excluded_f in excluded_files) - ] - - for name in coverage_values.keys(): - files = [f for f in measured_files if f.startswith(os.path.normpath(name))] - try: - with open(os.devnull, "w") as null: - overall = cov.report(files, ignore_errors=True, file=null) - singles = [ - (s, cov.report(s, ignore_errors=True, file=null)) for s in files - ] - coverage_values[name] = (overall, singles) - except Exception: - pass - - if any(v < 100 for v, _ in coverage_values.values()): - # make sure we get the EXIT_TESTSFAILED exit code - session.testsfailed += 1 - coverage_passed = False - - -def pytest_terminal_summary(terminalreporter, exitstatus, config): - global enable_coverage - global coverage_values - global coverage_passed - global no_full_cov - - if not enable_coverage: - return - - terminalreporter.write("\n") - if not coverage_passed: - markup = {"red": True, "bold": True} - msg = "FAIL: Full test coverage not reached!\n" - terminalreporter.write(msg, **markup) - - for name in sorted(coverage_values.keys()): - msg = f"Coverage for {name}: {coverage_values[name][0]:.2f}%\n" - if coverage_values[name][0] < 100: - markup = {"red": True, "bold": True} - for s, v in sorted(coverage_values[name][1]): - if v < 100: - msg += f" {s}: {v:.2f}%\n" - else: - markup = {"green": True} - terminalreporter.write(msg, **markup) - else: - msg = "SUCCESS: Full test coverage reached in modules and files:\n" - msg += "{}\n\n".format("\n".join(config.option.full_cov)) - terminalreporter.write(msg, green=True) - - msg = "\nExcluded files:\n" - for s in sorted(no_full_cov): - msg += f" {s}\n" - terminalreporter.write(msg) diff --git a/test/individual_coverage.py b/test/individual_coverage.py index 72ba8e1048..cd992f2fbc 100755 --- a/test/individual_coverage.py +++ b/test/individual_coverage.py @@ -1,108 +1,104 @@ #!/usr/bin/env python3 -import configparser -import contextlib -import glob -import io -import itertools -import multiprocessing +import asyncio +import fnmatch import os +import re +import subprocess import sys +from pathlib import Path -import pytest +import tomllib +root = Path(__file__).parent.parent.absolute() -def run_tests(src, test, fail): - stderr = io.StringIO() - stdout = io.StringIO() - with contextlib.redirect_stderr(stderr): - with contextlib.redirect_stdout(stdout): - e = pytest.main( - [ + +async def main(): + with open("pyproject.toml", "rb") as f: + data = tomllib.load(f) + + exclude = re.compile( + "|".join( + f"({fnmatch.translate(x)})" + for x in data["tool"]["pytest"]["individual_coverage"]["exclude"] + ) + ) + + sem = asyncio.Semaphore(os.cpu_count() or 1) + + async def run_tests(f: Path, should_fail: bool) -> None: + if f.name == "__init__.py": + test_file = Path("test") / f.parent.with_name(f"test_{f.parent.name}.py") + else: + test_file = Path("test") / f.with_name(f"test_{f.name}") + + coverage_file = f".coverage-{str(f).replace('/','-')}" + + async with sem: + try: + proc = await asyncio.create_subprocess_exec( + "pytest", "-qq", "--disable-pytest-warnings", "--cov", - src.replace(".py", "").replace("/", "."), + str(f.with_suffix("")).replace("/", "."), "--cov-fail-under", "100", "--cov-report", "term-missing:skip-covered", - "-o", - "faulthandler_timeout=0", - test, - ] - ) - - if e == 0: - if fail: - print( - "FAIL DUE TO UNEXPECTED SUCCESS:", - src, - "Please remove this file from setup.cfg tool:individual_coverage/exclude.", - ) - e = 42 - else: - print(".") - else: - if fail: - print("Ignoring allowed fail:", src) - e = 0 - else: - cov = [ - line - for line in stdout.getvalue().split("\n") - if (src in line) or ("was never imported" in line) - ] - if len(cov) == 1: - print("FAIL:", cov[0]) + test_file, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={ + "COVERAGE_FILE": coverage_file, + **os.environ, + }, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), 60) + except TimeoutError: + raise RuntimeError(f"{f}: timeout") + finally: + Path(coverage_file).unlink(missing_ok=True) + + if should_fail: + if proc.returncode != 0: + print(f"{f}: excluded") + else: + raise RuntimeError( + f"{f} is now fully covered by {test_file}. Remove it from tool.pytest.individual_coverage in pyproject.toml." + ) else: - print("FAIL:", src, test, stdout.getvalue(), stdout.getvalue()) - print(stderr.getvalue()) - print(stdout.getvalue()) - - sys.exit(e) - - -def start_pytest(src, test, fail): - # run pytest in a new process, otherwise imports and modules might conflict - proc = multiprocessing.Process(target=run_tests, args=(src, test, fail)) - proc.start() - proc.join() - return (src, test, proc.exitcode) - - -def main(): - c = configparser.ConfigParser() - c.read("setup.cfg") - fs = c["tool:individual_coverage"]["exclude"].strip().split("\n") - no_individual_cov = [f.strip() for f in fs] - - excluded = [ - "mitmproxy/contrib/", - "mitmproxy/test/", - "mitmproxy/tools/", - "mitmproxy/platform/", - ] - src_files = glob.glob("mitmproxy/**/*.py", recursive=True) - src_files = [f for f in src_files if os.path.basename(f) != "__init__.py"] - src_files = [ - f for f in src_files if not any(os.path.normpath(p) in f for p in excluded) - ] - if len(sys.argv) > 1: - src_files = [f for f in src_files if sys.argv[1] in str(f)] - - ps = [] - for src in sorted(src_files): - test = os.path.join( - "test", os.path.dirname(src), "test_" + os.path.basename(src) + if proc.returncode == 0: + print(f"{f}: ok") + else: + raise RuntimeError( + f"{f} is not fully covered by {test_file}:\n{stdout.decode(errors='ignore')}\n{stderr.decode(errors='ignore')}" + ) + + tasks = [] + for f in (root / "mitmproxy").glob("**/*.py"): + f = f.relative_to(root) + + if len(sys.argv) > 1 and sys.argv[1] not in str(f): + continue + + if f.name == "__init__.py" and f.stat().st_size == 0: + print(f"{f}: empty") + continue + + tasks.append( + asyncio.create_task(run_tests(f, should_fail=exclude.match(str(f)))) ) - if os.path.isfile(test): - ps.append((src, test, src in no_individual_cov)) - result = list(itertools.starmap(start_pytest, ps)) + exit_code = 0 + for task in asyncio.as_completed(tasks): + try: + await task + except RuntimeError as e: + print(e) + exit_code = 1 - if any(e != 0 for _, _, e in result): - sys.exit(1) + sys.exit(exit_code) if __name__ == "__main__": - main() + asyncio.run(main()) diff --git a/test/mitmproxy/test_options.py b/test/mitmproxy/test_options.py index 777ab4dd18..dbaafe1a9d 100644 --- a/test/mitmproxy/test_options.py +++ b/test/mitmproxy/test_options.py @@ -1 +1,5 @@ -# TODO: write tests +from mitmproxy import options + + +def test_simple(): + assert options.Options()