From 592ba8894edd13ed8ce8b0171c485c3a3897dce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Eustace?= Date: Tue, 14 Sep 2021 16:11:36 +0200 Subject: [PATCH] Fix system env detection --- poetry/utils/env.py | 197 ++++++++++++++++++++++++++++++++++++---- tests/utils/test_env.py | 125 +++++++++++++++++++++++++ 2 files changed, 304 insertions(+), 18 deletions(-) diff --git a/poetry/utils/env.py b/poetry/utils/env.py index cf39e271c26..b6b7b67271f 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -145,6 +145,38 @@ def _version_nodot(version): print(json.dumps(sysconfig.get_paths())) """ +GET_PATHS_FOR_GENERIC_ENVS = """\ +# We can't use sysconfig.get_paths() because +# on some distributions it does not return the proper paths +# (those used by pip for instance). We go through distutils +# to get the proper ones. +import json +import site +import sysconfig + +from distutils.command.install import SCHEME_KEYS # noqa +from distutils.core import Distribution + +d = Distribution() +d.parse_config_files() +obj = d.get_command_obj("install", create=True) +obj.finalize_options() + +paths = sysconfig.get_paths().copy() +for key in SCHEME_KEYS: + if key == "headers": + # headers is not a path returned by sysconfig.get_paths() + continue + + paths[key] = getattr(obj, f"install_{key}") + +if site.check_enableusersite() and hasattr(obj, "install_usersite"): + paths["usersite"] = getattr(obj, "install_usersite") + paths["userbase"] = getattr(obj, "install_userbase") + +print(json.dumps(paths)) +""" + class SitePackages: def __init__( @@ -615,7 +647,7 @@ def remove(self, python): # type: (str) -> Env self.remove_venv(venv) - return VirtualEnv(venv) + return VirtualEnv(venv, venv) def create_venv( self, io, name=None, executable=None, force=False @@ -848,15 +880,21 @@ def get_system_env( (e.g. plugin installation or self update). """ prefix, base_prefix = Path(sys.prefix), Path(cls.get_base_prefix()) + env = SystemEnv(prefix) if not naive: - try: - Path(__file__).relative_to(prefix) - except ValueError: - pass + if prefix.joinpath("poetry_env").exists(): + env = GenericEnv(base_prefix, child_env=env) else: - return GenericEnv(base_prefix) + from poetry.locations import data_dir + + try: + prefix.relative_to(data_dir()) + except ValueError: + pass + else: + env = GenericEnv(base_prefix, child_env=env) - return SystemEnv(prefix) + return env @classmethod def get_base_prefix(cls): # type: () -> str @@ -892,6 +930,11 @@ def __init__(self, path, base=None): # type: (Path, Optional[Path]) -> None self._base = base or path + self._executable = "python" + self._pip_executable = "pip" + + self.find_executables() + self._marker_env = None self._pip_version = None self._site_packages = None @@ -922,7 +965,7 @@ def python(self): # type: () -> str """ Path to current python executable """ - return self._bin("python") + return self._bin(self._executable) @property def marker_env(self): @@ -931,12 +974,16 @@ def marker_env(self): return self._marker_env + @property + def parent_env(self): # type: () -> GenericEnv + return GenericEnv(self.base, child_env=self) + @property def pip(self): # type: () -> str """ Path to current pip executable """ - return self._bin("pip") + return self._bin(self._pip_executable) @property def platform(self): # type: () -> str @@ -1028,6 +1075,35 @@ def get_base_prefix(cls): # type: () -> str return sys.prefix + def find_executables(self): # type: () -> None + python_executables = sorted( + [ + p.name + for p in self._bin_dir.glob("python*") + if re.match(r"python(?:\d+(?:\.\d+)?)?(?:\.exe)?$", p.name) + ] + ) + if python_executables: + executable = python_executables[0] + if executable.endswith(".exe"): + executable = executable[:-4] + + self._executable = executable + + pip_executables = sorted( + [ + p.name + for p in self._bin_dir.glob("pip*") + if re.match(r"pip(?:\d+(?:\.\d+)?)?(?:\.exe)?$", p.name) + ] + ) + if pip_executables: + pip_executable = pip_executables[0] + if pip_executable.endswith(".exe"): + pip_executable = pip_executable[:-4] + + self._pip_executable = pip_executable + def get_version_info(self): # type: () -> Tuple[int] raise NotImplementedError() @@ -1133,7 +1209,11 @@ def _bin(self, bin): # type: (str) -> str """ Return path to the given executable. """ - bin_path = (self._bin_dir / bin).with_suffix(".exe" if self._is_windows else "") + if self._is_windows and not bin.endswith(".exe"): + bin_path = self._bin_dir / (bin + ".exe") + else: + bin_path = self._bin_dir / bin + if not bin_path.exists(): # On Windows, some executables can be in the base path # This is especially true when installing Python with @@ -1144,7 +1224,11 @@ def _bin(self, bin): # type: (str) -> str # that creates a fake virtual environment pointing to # a base Python install. if self._is_windows: - bin_path = (self._path / bin).with_suffix(".exe") + if not bin.endswith(".exe"): + bin_path = self._bin_dir / (bin + ".exe") + else: + bin_path = self._path / bin + if bin_path.exists(): return str(bin_path) @@ -1270,16 +1354,18 @@ def __init__(self, path, base=None): # type: (Path, Optional[Path]) -> None # In this case we need to get sys.base_prefix # from inside the virtualenv. if base is None: - self._base = Path(self.run("python", "-", input_=GET_BASE_PREFIX).strip()) + self._base = Path( + self.run(self._executable, "-", input_=GET_BASE_PREFIX).strip() + ) @property def sys_path(self): # type: () -> List[str] - output = self.run("python", "-", input_=GET_SYS_PATH) + output = self.run(self._executable, "-", input_=GET_SYS_PATH) return json.loads(output) def get_version_info(self): # type: () -> Tuple[int] - output = self.run("python", "-", input_=GET_PYTHON_VERSION) + output = self.run(self._executable, "-", input_=GET_PYTHON_VERSION) return tuple([int(s) for s in output.strip().split(".")]) @@ -1289,7 +1375,7 @@ def get_python_implementation(self): # type: () -> str def get_pip_command(self): # type: () -> List[str] # We're in a virtualenv that is known to be sane, # so assume that we have a functional pip - return [self._bin("pip")] + return [self._bin(self._pip_executable)] def get_supported_tags(self): # type: () -> List[Tag] file_path = Path(packaging.tags.__file__) @@ -1317,12 +1403,12 @@ def get_supported_tags(self): # type: () -> List[Tag] """ ) - output = self.run("python", "-", input_=script) + output = self.run(self._executable, "-", input_=script) return [Tag(*t) for t in json.loads(output)] def get_marker_env(self): # type: () -> Dict[str, Any] - output = self.run("python", "-", input_=GET_ENVIRONMENT_INFO) + output = self.run(self._executable, "-", input_=GET_ENVIRONMENT_INFO) return json.loads(output) @@ -1335,7 +1421,7 @@ def get_pip_version(self): # type: () -> Version return Version.parse(m.group(1)) def get_paths(self): # type: () -> Dict[str, str] - output = self.run("python", "-", input_=GET_PATHS) + output = self.run(self._executable, "-", input_=GET_PATHS) return json.loads(output) @@ -1388,6 +1474,81 @@ def _updated_path(self): class GenericEnv(VirtualEnv): + def __init__( + self, path, base=None, child_env=None + ): # type: (Path, Optional[Path], Optional[Env]) -> None + self._child_env = child_env + + super(GenericEnv, self).__init__(path, base=base) + + def find_executables(self): # type: () -> None + patterns = [("python*", "pip*")] + + if self._child_env: + minor_version = "{}.{}".format( + self._child_env.version_info[0], self._child_env.version_info[1] + ) + major_version = "{}".format(self._child_env.version_info[0]) + patterns = [ + ("python{}".format(minor_version), "pip{}".format(minor_version)), + ("python{}".format(major_version), "pip{}".format(major_version)), + ] + + python_executable = None + pip_executable = None + + for python_pattern, pip_pattern in patterns: + if python_executable and pip_executable: + break + + if not python_executable: + python_executables = sorted( + [ + p.name + for p in self._bin_dir.glob(python_pattern) + if re.match(r"python(?:\d+(?:\.\d+)?)?(?:\.exe)?$", p.name) + ] + ) + + if python_executables: + executable = python_executables[0] + if executable.endswith(".exe"): + executable = executable[:-4] + + python_executable = executable + + if not pip_executable: + pip_executables = sorted( + [ + p.name + for p in self._bin_dir.glob(pip_pattern) + if re.match(r"pip(?:\d+(?:\.\d+)?)?(?:\.exe)?$", p.name) + ] + ) + if pip_executables: + pip_executable = pip_executables[0] + if pip_executable.endswith(".exe"): + pip_executable = pip_executable[:-4] + + pip_executable = pip_executable + + if python_executable: + self._executable = python_executable + + if pip_executable: + self._pip_executable = pip_executable + + def get_paths(self): # type: () -> Dict[str, str] + output = self.run(self._executable, "-", input_=GET_PATHS_FOR_GENERIC_ENVS) + + return json.loads(output) + + def execute(self, bin, *args, **kwargs): # type: (str, str, Any) -> Optional[int] + return super(VirtualEnv, self).execute(bin, *args, **kwargs) + + def _run(self, cmd, **kwargs): # type: (List[str], Any) -> Optional[int] + return super(VirtualEnv, self)._run(cmd, **kwargs) + def is_venv(self): # type: () -> bool return self._path != self._base diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index cd5e1b9fe16..369cf7c9874 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -14,10 +14,12 @@ from poetry.core.toml.file import TOMLFile from poetry.factory import Factory from poetry.utils._compat import PY2 +from poetry.utils._compat import WINDOWS from poetry.utils._compat import Path from poetry.utils.env import GET_BASE_PREFIX from poetry.utils.env import EnvCommandError from poetry.utils.env import EnvManager +from poetry.utils.env import GenericEnv from poetry.utils.env import NoCompatiblePythonVersionFound from poetry.utils.env import SystemEnv from poetry.utils.env import VirtualEnv @@ -869,3 +871,126 @@ def test_venv_has_correct_paths(tmp_venv): assert paths.get("platlib") is not None assert paths.get("scripts") is not None assert tmp_venv.site_packages.path == Path(paths["purelib"]) + + +def test_env_finds_the_correct_executables(tmp_dir, manager): + venv_path = Path(tmp_dir) / "Virtual Env" + manager.build_venv(str(venv_path)) + venv = VirtualEnv(venv_path) + + default_executable = expected_executable = "python" + (".exe" if WINDOWS else "") + default_pip_executable = expected_pip_executable = "pip" + ( + ".exe" if WINDOWS else "" + ) + major_executable = "python{}{}".format( + sys.version_info[0], ".exe" if WINDOWS else "" + ) + major_pip_executable = "pip{}{}".format( + sys.version_info[0], ".exe" if WINDOWS else "" + ) + + if ( + venv._bin_dir.joinpath(default_executable).exists() + and venv._bin_dir.joinpath(major_executable).exists() + ): + venv._bin_dir.joinpath(default_executable).unlink() + expected_executable = major_executable + + if ( + venv._bin_dir.joinpath(default_pip_executable).exists() + and venv._bin_dir.joinpath(major_pip_executable).exists() + ): + venv._bin_dir.joinpath(default_pip_executable).unlink() + expected_pip_executable = major_pip_executable + + venv = VirtualEnv(venv_path) + + assert Path(venv.python).name == expected_executable + assert Path(venv.pip).name.startswith(expected_pip_executable.split(".")[0]) + + +def test_env_finds_the_correct_executables_for_generic_env(tmp_dir, manager): + venv_path = Path(tmp_dir) / "Virtual Env" + child_venv_path = Path(tmp_dir) / "Child Virtual Env" + manager.build_venv(str(venv_path)) + parent_venv = VirtualEnv(venv_path) + manager.build_venv(str(child_venv_path), executable=parent_venv.python) + venv = GenericEnv(parent_venv.path, child_env=VirtualEnv(child_venv_path)) + + expected_executable = "python{}.{}{}".format( + sys.version_info[0], sys.version_info[1], ".exe" if WINDOWS else "" + ) + expected_pip_executable = "pip{}.{}{}".format( + sys.version_info[0], sys.version_info[1], ".exe" if WINDOWS else "" + ) + + if WINDOWS: + expected_executable = "python.exe" + expected_pip_executable = "pip.exe" + + assert Path(venv.python).name == expected_executable + assert Path(venv.pip).name == expected_pip_executable + + +def test_env_finds_fallback_executables_for_generic_env(tmp_dir, manager): + venv_path = Path(tmp_dir) / "Virtual Env" + child_venv_path = Path(tmp_dir) / "Child Virtual Env" + manager.build_venv(str(venv_path)) + parent_venv = VirtualEnv(venv_path) + manager.build_venv(str(child_venv_path), executable=parent_venv.python) + venv = GenericEnv(parent_venv.path, child_env=VirtualEnv(child_venv_path)) + + default_executable = "python" + (".exe" if WINDOWS else "") + major_executable = "python{}{}".format( + sys.version_info[0], ".exe" if WINDOWS else "" + ) + minor_executable = "python{}.{}{}".format( + sys.version_info[0], sys.version_info[1], ".exe" if WINDOWS else "" + ) + expected_executable = minor_executable + if ( + venv._bin_dir.joinpath(expected_executable).exists() + and venv._bin_dir.joinpath(major_executable).exists() + ): + venv._bin_dir.joinpath(expected_executable).unlink() + expected_executable = major_executable + + if ( + venv._bin_dir.joinpath(expected_executable).exists() + and venv._bin_dir.joinpath(default_executable).exists() + ): + venv._bin_dir.joinpath(expected_executable).unlink() + expected_executable = default_executable + + default_pip_executable = "pip" + (".exe" if WINDOWS else "") + major_pip_executable = "pip{}{}".format( + sys.version_info[0], ".exe" if WINDOWS else "" + ) + minor_pip_executable = "pip{}.{}{}".format( + sys.version_info[0], sys.version_info[1], ".exe" if WINDOWS else "" + ) + expected_pip_executable = minor_pip_executable + if ( + venv._bin_dir.joinpath(expected_pip_executable).exists() + and venv._bin_dir.joinpath(major_pip_executable).exists() + ): + venv._bin_dir.joinpath(expected_pip_executable).unlink() + expected_pip_executable = major_pip_executable + + if ( + venv._bin_dir.joinpath(expected_pip_executable).exists() + and venv._bin_dir.joinpath(default_pip_executable).exists() + ): + venv._bin_dir.joinpath(expected_pip_executable).unlink() + expected_pip_executable = default_pip_executable + + if not venv._bin_dir.joinpath(expected_executable).exists(): + expected_executable = default_executable + + if not venv._bin_dir.joinpath(expected_pip_executable).exists(): + expected_pip_executable = default_pip_executable + + venv = GenericEnv(parent_venv.path, child_env=VirtualEnv(child_venv_path)) + + assert Path(venv.python).name == expected_executable + assert Path(venv.pip).name == expected_pip_executable