diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py index 8f4ffa8c5..5609631b0 100644 --- a/pre_commit/languages/all.py +++ b/pre_commit/languages/all.py @@ -14,7 +14,6 @@ from pre_commit.languages import perl from pre_commit.languages import pygrep from pre_commit.languages import python -from pre_commit.languages import python_venv from pre_commit.languages import ruby from pre_commit.languages import rust from pre_commit.languages import script @@ -49,7 +48,6 @@ class Language(NamedTuple): 'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, healthy=perl.healthy, install_environment=perl.install_environment, run_hook=perl.run_hook), # noqa: E501 'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, healthy=pygrep.healthy, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook), # noqa: E501 'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, healthy=python.healthy, install_environment=python.install_environment, run_hook=python.run_hook), # noqa: E501 - 'python_venv': Language(name='python_venv', ENVIRONMENT_DIR=python_venv.ENVIRONMENT_DIR, get_default_version=python_venv.get_default_version, healthy=python_venv.healthy, install_environment=python_venv.install_environment, run_hook=python_venv.run_hook), # noqa: E501 'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, healthy=ruby.healthy, install_environment=ruby.install_environment, run_hook=ruby.run_hook), # noqa: E501 'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, healthy=rust.healthy, install_environment=rust.install_environment, run_hook=rust.run_hook), # noqa: E501 'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, healthy=script.healthy, install_environment=script.install_environment, run_hook=script.run_hook), # noqa: E501 @@ -57,4 +55,6 @@ class Language(NamedTuple): 'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, healthy=system.healthy, install_environment=system.install_environment, run_hook=system.run_hook), # noqa: E501 # END GENERATED } +# TODO: fully deprecate `python_venv` +languages['python_venv'] = languages['python'] all_languages = sorted(languages) diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index de3dd4527..e17376e1f 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -2,8 +2,7 @@ import functools import os import sys -from typing import Callable -from typing import ContextManager +from typing import Dict from typing import Generator from typing import Optional from typing import Sequence @@ -26,6 +25,28 @@ ENVIRONMENT_DIR = 'py_env' +@functools.lru_cache(maxsize=None) +def _version_info(exe: str) -> str: + prog = 'import sys;print(".".join(str(p) for p in sys.version_info))' + try: + return cmd_output(exe, '-S', '-c', prog)[1].strip() + except CalledProcessError: + return f'<>' + + +def _read_pyvenv_cfg(filename: str) -> Dict[str, str]: + ret = {} + with open(filename) as f: + for line in f: + try: + k, v = line.split('=') + except ValueError: # blank line / comment / etc. + continue + else: + ret[k.strip()] = v.strip() + return ret + + def bin_dir(venv: str) -> str: """On windows there's a different directory for the virtualenv""" bin_part = 'Scripts' if os.name == 'nt' else 'bin' @@ -116,6 +137,9 @@ def _sys_executable_matches(version: str) -> bool: def norm_version(version: str) -> str: + if version == C.DEFAULT: + return os.path.realpath(sys.executable) + # first see if our current executable is appropriate if _sys_executable_matches(version): return sys.executable @@ -140,70 +164,59 @@ def norm_version(version: str) -> str: return os.path.expanduser(version) -def py_interface( - _dir: str, - _make_venv: Callable[[str, str], None], -) -> Tuple[ - Callable[[Prefix, str], ContextManager[None]], - Callable[[Prefix, str], bool], - Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]], - Callable[[Prefix, str, Sequence[str]], None], -]: - @contextlib.contextmanager - def in_env( - prefix: Prefix, - language_version: str, - ) -> Generator[None, None, None]: - envdir = prefix.path(helpers.environment_dir(_dir, language_version)) - with envcontext(get_env_patch(envdir)): - yield - - def healthy(prefix: Prefix, language_version: str) -> bool: - envdir = helpers.environment_dir(_dir, language_version) - exe_name = 'python.exe' if sys.platform == 'win32' else 'python' - py_exe = prefix.path(bin_dir(envdir), exe_name) - with in_env(prefix, language_version): - retcode, _, _ = cmd_output_b( - py_exe, '-c', 'import ctypes, datetime, io, os, ssl, weakref', - cwd='/', - retcode=None, - ) - return retcode == 0 - - def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, - ) -> Tuple[int, bytes]: - with in_env(hook.prefix, hook.language_version): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) - - def install_environment( - prefix: Prefix, - version: str, - additional_dependencies: Sequence[str], - ) -> None: - directory = helpers.environment_dir(_dir, version) - install = ('python', '-mpip', 'install', '.', *additional_dependencies) - - env_dir = prefix.path(directory) - with clean_path_on_failure(env_dir): - if version != C.DEFAULT: - python = norm_version(version) - else: - python = os.path.realpath(sys.executable) - _make_venv(env_dir, python) - with in_env(prefix, version): - helpers.run_setup_cmd(prefix, install) +@contextlib.contextmanager +def in_env( + prefix: Prefix, + language_version: str, +) -> Generator[None, None, None]: + directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) + envdir = prefix.path(directory) + with envcontext(get_env_patch(envdir)): + yield - return in_env, healthy, run_hook, install_environment +def healthy(prefix: Prefix, language_version: str) -> bool: + directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) + envdir = prefix.path(directory) + pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg') -def make_venv(envdir: str, python: str) -> None: - env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1') - cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python) - cmd_output_b(*cmd, env=env, cwd='/') + # created with "old" virtualenv + if not os.path.exists(pyvenv_cfg): + return False + + exe_name = 'python.exe' if sys.platform == 'win32' else 'python' + py_exe = prefix.path(bin_dir(envdir), exe_name) + cfg = _read_pyvenv_cfg(pyvenv_cfg) + + return ( + 'version_info' in cfg and + _version_info(py_exe) == cfg['version_info'] and ( + 'base-executable' not in cfg or + _version_info(cfg['base-executable']) == cfg['version_info'] + ) + ) -_interface = py_interface(ENVIRONMENT_DIR, make_venv) -in_env, healthy, run_hook, install_environment = _interface +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: + envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + python = norm_version(version) + venv_cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python) + install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies) + + with clean_path_on_failure(envdir): + cmd_output_b(*venv_cmd, cwd='/') + with in_env(prefix, version): + helpers.run_setup_cmd(prefix, install_cmd) + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + with in_env(hook.prefix, hook.language_version): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/python_venv.py b/pre_commit/languages/python_venv.py deleted file mode 100644 index 5404c8be5..000000000 --- a/pre_commit/languages/python_venv.py +++ /dev/null @@ -1,46 +0,0 @@ -import os.path - -from pre_commit.languages import python -from pre_commit.util import CalledProcessError -from pre_commit.util import cmd_output -from pre_commit.util import cmd_output_b - -ENVIRONMENT_DIR = 'py_venv' -get_default_version = python.get_default_version - - -def orig_py_exe(exe: str) -> str: # pragma: no cover (platform specific) - """A -mvenv virtualenv made from a -mvirtualenv virtualenv installs - packages to the incorrect location. Attempt to find the _original_ exe - and invoke `-mvenv` from there. - - See: - - https://github.com/pre-commit/pre-commit/issues/755 - - https://github.com/pypa/virtualenv/issues/1095 - - https://bugs.python.org/issue30811 - """ - try: - prefix_script = 'import sys; print(sys.real_prefix)' - _, prefix, _ = cmd_output(exe, '-c', prefix_script) - prefix = prefix.strip() - except CalledProcessError: - # not created from -mvirtualenv - return exe - - if os.name == 'nt': - expected = os.path.join(prefix, 'python.exe') - else: - expected = os.path.join(prefix, 'bin', os.path.basename(exe)) - - if os.path.exists(expected): - return expected - else: - return exe - - -def make_venv(envdir: str, python: str) -> None: - cmd_output_b(orig_py_exe(python), '-mvenv', envdir, cwd='/') - - -_interface = python.py_interface(ENVIRONMENT_DIR, make_venv) -in_env, healthy, run_hook, install_environment = _interface diff --git a/setup.cfg b/setup.cfg index 2e69d5032..2ca5b3150 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ install_requires = nodeenv>=0.11.1 pyyaml>=5.1 toml - virtualenv>=15.2 + virtualenv>=20.0.8 importlib-metadata;python_version<"3.8" importlib-resources;python_version<"3.7" python_requires = >=3.6.1 diff --git a/testing/gen-languages-all b/testing/gen-languages-all index 6d0b26ff9..2bff7beb0 100755 --- a/testing/gen-languages-all +++ b/testing/gen-languages-all @@ -3,8 +3,7 @@ import sys LANGUAGES = [ 'conda', 'docker', 'docker_image', 'fail', 'golang', 'node', 'perl', - 'pygrep', 'python', 'python_venv', 'ruby', 'rust', 'script', 'swift', - 'system', + 'pygrep', 'python', 'ruby', 'rust', 'script', 'swift', 'system', ] FIELDS = [ 'ENVIRONMENT_DIR', 'get_default_version', 'healthy', 'install_environment', diff --git a/testing/util.py b/testing/util.py index 439bee794..ff3537a47 100644 --- a/testing/util.py +++ b/testing/util.py @@ -45,20 +45,6 @@ def cmd_output_mocked_pre_commit_home( xfailif_windows = pytest.mark.xfail(os.name == 'nt', reason='windows') -def supports_venv(): # pragma: no cover (platform specific) - try: - __import__('ensurepip') - __import__('venv') - return True - except ImportError: - return False - - -xfailif_no_venv = pytest.mark.xfail( - not supports_venv(), reason='Does not support venv module', -) - - def run_opts( all_files=False, files=(), diff --git a/tests/languages/python_test.py b/tests/languages/python_test.py index 34c6c7fc5..66f6c8bec 100644 --- a/tests/languages/python_test.py +++ b/tests/languages/python_test.py @@ -5,10 +5,23 @@ import pytest import pre_commit.constants as C +from pre_commit.envcontext import envcontext from pre_commit.languages import python from pre_commit.prefix import Prefix +def test_read_pyvenv_cfg(tmpdir): + pyvenv_cfg = tmpdir.join('pyvenv.cfg') + pyvenv_cfg.write( + '# I am a comment\n' + '\n' + 'foo = bar\n' + 'version-info=123\n', + ) + expected = {'foo': 'bar', 'version-info': '123'} + assert python._read_pyvenv_cfg(pyvenv_cfg) == expected + + def test_norm_version_expanduser(): home = os.path.expanduser('~') if os.name == 'nt': # pragma: nt cover @@ -21,6 +34,10 @@ def test_norm_version_expanduser(): assert result == expected_path +def test_norm_version_of_default_is_sys_executable(): + assert python.norm_version('default') == os.path.realpath(sys.executable) + + @pytest.mark.parametrize('v', ('python3.6', 'python3', 'python')) def test_sys_executable_matches(v): with mock.patch.object(sys, 'version_info', (3, 6, 7)): @@ -49,27 +66,67 @@ def test_find_by_sys_executable(exe, realpath, expected): assert python._find_by_sys_executable() == expected -def test_healthy_types_py_in_cwd(tmpdir): +@pytest.fixture +def python_dir(tmpdir): with tmpdir.as_cwd(): prefix = tmpdir.join('prefix').ensure_dir() prefix.join('setup.py').write('import setuptools; setuptools.setup()') prefix = Prefix(str(prefix)) - python.install_environment(prefix, C.DEFAULT, ()) + yield prefix, tmpdir - # even if a `types.py` file exists, should still be healthy - tmpdir.join('types.py').ensure() - assert python.healthy(prefix, C.DEFAULT) is True +def test_healthy_default_creator(python_dir): + prefix, tmpdir = python_dir -def test_healthy_python_goes_missing(tmpdir): - with tmpdir.as_cwd(): - prefix = tmpdir.join('prefix').ensure_dir() - prefix.join('setup.py').write('import setuptools; setuptools.setup()') - prefix = Prefix(str(prefix)) + python.install_environment(prefix, C.DEFAULT, ()) + + # should be healthy right after creation + assert python.healthy(prefix, C.DEFAULT) is True + + # even if a `types.py` file exists, should still be healthy + tmpdir.join('types.py').ensure() + assert python.healthy(prefix, C.DEFAULT) is True + + +def test_healthy_venv_creator(python_dir): + # venv creator produces slightly different pyvenv.cfg + prefix, tmpdir = python_dir + + with envcontext((('VIRTUALENV_CREATOR', 'venv'),)): python.install_environment(prefix, C.DEFAULT, ()) - exe_name = 'python' if sys.platform != 'win32' else 'python.exe' - py_exe = prefix.path(python.bin_dir('py_env-default'), exe_name) - os.remove(py_exe) + assert python.healthy(prefix, C.DEFAULT) is True + + +def test_unhealthy_python_goes_missing(python_dir): + prefix, tmpdir = python_dir + + python.install_environment(prefix, C.DEFAULT, ()) + + exe_name = 'python' if sys.platform != 'win32' else 'python.exe' + py_exe = prefix.path(python.bin_dir('py_env-default'), exe_name) + os.remove(py_exe) + + assert python.healthy(prefix, C.DEFAULT) is False + + +def test_unhealthy_with_version_change(python_dir): + prefix, tmpdir = python_dir + + python.install_environment(prefix, C.DEFAULT, ()) + + with open(prefix.path('py_env-default/pyvenv.cfg'), 'w') as f: + f.write('version_info = 1.2.3\n') + + assert python.healthy(prefix, C.DEFAULT) is False + + +def test_unhealthy_system_version_changes(python_dir): + prefix, tmpdir = python_dir + + python.install_environment(prefix, C.DEFAULT, ()) + + with open(prefix.path('py_env-default/pyvenv.cfg'), 'a') as f: + f.write('base-executable = /does/not/exist\n') - assert python.healthy(prefix, C.DEFAULT) is False + assert python.healthy(prefix, C.DEFAULT) is False diff --git a/tests/repository_test.py b/tests/repository_test.py index f55c34c82..56e2bba8e 100644 --- a/tests/repository_test.py +++ b/tests/repository_test.py @@ -33,7 +33,6 @@ from testing.util import get_resource_path from testing.util import skipif_cant_run_docker from testing.util import skipif_cant_run_swift -from testing.util import xfailif_no_venv from testing.util import xfailif_windows_no_ruby @@ -163,7 +162,6 @@ def test_python_hook_weird_setup_cfg(in_git_dir, tempdir_factory, store): ) -@xfailif_no_venv def test_python_venv(tempdir_factory, store): # pragma: no cover (no venv) _test_hook_repo( tempdir_factory, store, 'python_venv_hooks_repo',