Improve python healthy() and eliminate python_venv

pre_commit/languages/all.py

@@ -14,7 +14,6 @@
 from pre_commit.languages import perl
 from pre_commit.languages import pygrep
 from pre_commit.languages import python
-from pre_commit.languages import python_venv
 from pre_commit.languages import ruby
 from pre_commit.languages import rust
 from pre_commit.languages import script
@@ -49,12 +48,13 @@ class Language(NamedTuple):
     'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, healthy=perl.healthy, install_environment=perl.install_environment, run_hook=perl.run_hook),  # noqa: E501
     'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, healthy=pygrep.healthy, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook),  # noqa: E501
     'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, healthy=python.healthy, install_environment=python.install_environment, run_hook=python.run_hook),  # noqa: E501
-    'python_venv': Language(name='python_venv', ENVIRONMENT_DIR=python_venv.ENVIRONMENT_DIR, get_default_version=python_venv.get_default_version, healthy=python_venv.healthy, install_environment=python_venv.install_environment, run_hook=python_venv.run_hook),  # noqa: E501
     'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, healthy=ruby.healthy, install_environment=ruby.install_environment, run_hook=ruby.run_hook),  # noqa: E501
     'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, healthy=rust.healthy, install_environment=rust.install_environment, run_hook=rust.run_hook),  # noqa: E501
     'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, healthy=script.healthy, install_environment=script.install_environment, run_hook=script.run_hook),  # noqa: E501
     'swift': Language(name='swift', ENVIRONMENT_DIR=swift.ENVIRONMENT_DIR, get_default_version=swift.get_default_version, healthy=swift.healthy, install_environment=swift.install_environment, run_hook=swift.run_hook),  # noqa: E501
     'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, healthy=system.healthy, install_environment=system.install_environment, run_hook=system.run_hook),  # noqa: E501
     # END GENERATED
 }
+# TODO: fully deprecate `python_venv`
+languages['python_venv'] = languages['python']
 all_languages = sorted(languages)

pre_commit/languages/python.py

@@ -2,8 +2,7 @@
 import functools
 import os
 import sys
-from typing import Callable
-from typing import ContextManager
+from typing import Dict
 from typing import Generator
 from typing import Optional
 from typing import Sequence
@@ -26,6 +25,28 @@
 ENVIRONMENT_DIR = 'py_env'
 
 
+@functools.lru_cache(maxsize=None)
+def _version_info(exe: str) -> str:
+    prog = 'import sys;print(".".join(str(p) for p in sys.version_info))'
+    try:
+        return cmd_output(exe, '-S', '-c', prog)[1].strip()
+    except CalledProcessError:
+        return f'<<error retrieving version from {exe}>>'
+
+
+def _read_pyvenv_cfg(filename: str) -> Dict[str, str]:
+    ret = {}
+    with open(filename) as f:
+        for line in f:
+            try:
+                k, v = line.split('=')
+            except ValueError:  # blank line / comment / etc.
+                continue
+            else:
+                ret[k.strip()] = v.strip()
+    return ret
+
+
 def bin_dir(venv: str) -> str:
     """On windows there's a different directory for the virtualenv"""
     bin_part = 'Scripts' if os.name == 'nt' else 'bin'
@@ -116,6 +137,9 @@ def _sys_executable_matches(version: str) -> bool:
 
 
 def norm_version(version: str) -> str:
+    if version == C.DEFAULT:
+        return os.path.realpath(sys.executable)
+
     # first see if our current executable is appropriate
     if _sys_executable_matches(version):
         return sys.executable
@@ -140,70 +164,59 @@ def norm_version(version: str) -> str:
     return os.path.expanduser(version)
 
 
-def py_interface(
-        _dir: str,
-        _make_venv: Callable[[str, str], None],
-) -> Tuple[
-    Callable[[Prefix, str], ContextManager[None]],
-    Callable[[Prefix, str], bool],
-    Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]],
-    Callable[[Prefix, str, Sequence[str]], None],
-]:
-    @contextlib.contextmanager
-    def in_env(
-            prefix: Prefix,
-            language_version: str,
-    ) -> Generator[None, None, None]:
-        envdir = prefix.path(helpers.environment_dir(_dir, language_version))
-        with envcontext(get_env_patch(envdir)):
-            yield
-
-    def healthy(prefix: Prefix, language_version: str) -> bool:
-        envdir = helpers.environment_dir(_dir, language_version)
-        exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
-        py_exe = prefix.path(bin_dir(envdir), exe_name)
-        with in_env(prefix, language_version):
-            retcode, _, _ = cmd_output_b(
-                py_exe, '-c', 'import ctypes, datetime, io, os, ssl, weakref',
-                cwd='/',
-                retcode=None,
-            )
-        return retcode == 0
-
-    def run_hook(
-            hook: Hook,
-            file_args: Sequence[str],
-            color: bool,
-    ) -> Tuple[int, bytes]:
-        with in_env(hook.prefix, hook.language_version):
-            return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
-
-    def install_environment(
-            prefix: Prefix,
-            version: str,
-            additional_dependencies: Sequence[str],
-    ) -> None:
-        directory = helpers.environment_dir(_dir, version)
-        install = ('python', '-mpip', 'install', '.', *additional_dependencies)
-
-        env_dir = prefix.path(directory)
-        with clean_path_on_failure(env_dir):
-            if version != C.DEFAULT:
-                python = norm_version(version)
-            else:
-                python = os.path.realpath(sys.executable)
-            _make_venv(env_dir, python)
-            with in_env(prefix, version):
-                helpers.run_setup_cmd(prefix, install)
+@contextlib.contextmanager
+def in_env(
+        prefix: Prefix,
+        language_version: str,
+) -> Generator[None, None, None]:
+    directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
+    envdir = prefix.path(directory)
+    with envcontext(get_env_patch(envdir)):
+        yield
 
-    return in_env, healthy, run_hook, install_environment
 
+def healthy(prefix: Prefix, language_version: str) -> bool:
+    directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
+    envdir = prefix.path(directory)
+    pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg')
 
-def make_venv(envdir: str, python: str) -> None:
-    env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1')
-    cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
-    cmd_output_b(*cmd, env=env, cwd='/')
+    # created with "old" virtualenv
+    if not os.path.exists(pyvenv_cfg):
+        return False
+
+    exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
+    py_exe = prefix.path(bin_dir(envdir), exe_name)
+    cfg = _read_pyvenv_cfg(pyvenv_cfg)
+
+    return (
+        'version_info' in cfg and
+        _version_info(py_exe) == cfg['version_info'] and (
+            'base-executable' not in cfg or
+            _version_info(cfg['base-executable']) == cfg['version_info']
+        )
+    )
 
 
-_interface = py_interface(ENVIRONMENT_DIR, make_venv)
-in_env, healthy, run_hook, install_environment = _interface
+def install_environment(
+        prefix: Prefix,
+        version: str,
+        additional_dependencies: Sequence[str],
+) -> None:
+    envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
+    python = norm_version(version)
+    venv_cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
+    install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies)
+
+    with clean_path_on_failure(envdir):
+        cmd_output_b(*venv_cmd, cwd='/')
+        with in_env(prefix, version):
+            helpers.run_setup_cmd(prefix, install_cmd)
+
+
+def run_hook(
+        hook: Hook,
+        file_args: Sequence[str],
+        color: bool,
+) -> Tuple[int, bytes]:
+    with in_env(hook.prefix, hook.language_version):
+        return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

setup.cfg

@@ -27,7 +27,7 @@ install_requires =
     nodeenv>=0.11.1
     pyyaml>=5.1
     toml
-    virtualenv>=15.2
+    virtualenv>=20.0.8
     importlib-metadata;python_version<"3.8"
     importlib-resources;python_version<"3.7"
 python_requires = >=3.6.1

testing/gen-languages-all

@@ -3,8 +3,7 @@ import sys
 
 LANGUAGES = [
     'conda', 'docker', 'docker_image', 'fail', 'golang', 'node', 'perl',
-    'pygrep', 'python', 'python_venv', 'ruby', 'rust', 'script', 'swift',
-    'system',
+    'pygrep', 'python', 'ruby', 'rust', 'script', 'swift', 'system',
 ]
 FIELDS = [
     'ENVIRONMENT_DIR', 'get_default_version', 'healthy', 'install_environment',

testing/util.py

@@ -45,20 +45,6 @@ def cmd_output_mocked_pre_commit_home(
 xfailif_windows = pytest.mark.xfail(os.name == 'nt', reason='windows')
 
 
-def supports_venv():  # pragma: no cover (platform specific)
-    try:
-        __import__('ensurepip')
-        __import__('venv')
-        return True
-    except ImportError:
-        return False
-
-
-xfailif_no_venv = pytest.mark.xfail(
-    not supports_venv(), reason='Does not support venv module',
-)
-
-
 def run_opts(
         all_files=False,
         files=(),