diff --git a/docs/changelog/2310.bugfix.rst b/docs/changelog/2310.bugfix.rst new file mode 100644 index 000000000..b7226d619 --- /dev/null +++ b/docs/changelog/2310.bugfix.rst @@ -0,0 +1 @@ +Avoid symlinking the contents of ``/usr`` into PyPy3.8+ virtualenvs - by :user:`stefanor`. diff --git a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py index f1726ec96..cc72c1459 100644 --- a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py +++ b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py @@ -42,6 +42,11 @@ def to_lib(self, src): def sources(cls, interpreter): for src in super(PyPy3Posix, cls).sources(interpreter): yield src + # PyPy >= 3.8 supports a standard prefix installation, where older + # versions always used a portable/developent style installation. + # If this is a standard prefix installation, skip the below: + if interpreter.system_prefix == "/usr": + return # Also copy/symlink anything under prefix/lib, which, for "portable" # PyPy builds, includes the tk,tcl runtime and a number of shared # objects. In distro-specific builds or on conda this should be empty diff --git a/src/virtualenv/util/path/_pathlib/via_os_path.py b/src/virtualenv/util/path/_pathlib/via_os_path.py index ac78d4f00..b876f025e 100644 --- a/src/virtualenv/util/path/_pathlib/via_os_path.py +++ b/src/virtualenv/util/path/_pathlib/via_os_path.py @@ -49,6 +49,9 @@ def __ne__(self, other): def __hash__(self): return hash(self._path) + def as_posix(self): + return str(self).replace(os.sep, "/") + def exists(self): return os.path.exists(self._path) diff --git a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json new file mode 100644 index 000000000..eb694a840 --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json @@ -0,0 +1,64 @@ +{ + "platform": "linux", + "implementation": "PyPy", + "pypy_version_info": [7, 3, 7, "final", 0], + "version_info": { + "major": 3, + "minor": 7, + "micro": 12, + "releaselevel": "final", + "serial": 0 + }, + "architecture": 64, + "version": "3.7.12 (7.3.7+dfsg-5, Jan 27 2022, 12:27:44)\\n[PyPy 7.3.7 with GCC 11.2.0]", + "os": "posix", + "prefix": "/usr/lib/pypy3", + "base_prefix": "/usr/lib/pypy3", + "real_prefix": null, + "base_exec_prefix": "/usr/lib/pypy3", + "exec_prefix": "/usr/lib/pypy3", + "executable": "/usr/bin/pypy3", + "original_executable": "/usr/bin/pypy3", + "system_executable": "/usr/bin/pypy3", + "has_venv": true, + "path": [ + "/usr/lib/pypy3/lib_pypy/__extensions__", + "/usr/lib/pypy3/lib_pypy", + "/usr/lib/pypy3/lib-python/3", + "/usr/lib/pypy3/lib-python/3/lib-tk", + "/usr/lib/pypy3/lib-python/3/plat-linux2", + "/usr/local/lib/pypy3.7/dist-packages", + "/usr/lib/python3/dist-packages" + ], + "file_system_encoding": "utf-8", + "stdout_encoding": "UTF-8", + "sysconfig_scheme": null, + "sysconfig_paths": { + "stdlib": "{base}/lib-python/{py_version_short}", + "platstdlib": "{base}/lib-python/{py_version_short}", + "purelib": "{base}/../../local/lib/pypy{py_version_short}/lib-python", + "platlib": "{base}/../../local/lib/pypy{py_version_short}/lib-python", + "include": "{base}/include", + "scripts": "{base}/../../local/bin", + "data": "{base}/../../local" + }, + "distutils_install": { + "purelib": "site-packages", + "platlib": "site-packages", + "headers": "include/UNKNOWN", + "scripts": "bin", + "data": "" + }, + "sysconfig": { + "makefile_filename": "/usr/lib/pypy3/lib-python/3.7/config-3.7-x86_64-linux-gnu/Makefile" + }, + "sysconfig_vars": { + "base": "/usr/lib/pypy3", + "py_version_short": "3.7", + "PYTHONFRAMEWORK": "" + }, + "system_stdlib": "/usr/lib/pypy3/lib-python/3.7", + "system_stdlib_platform": "/usr/lib/pypy3/lib-python/3.7", + "max_size": 9223372036854775807, + "_creators": null +} diff --git a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json new file mode 100644 index 000000000..478d79977 --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json @@ -0,0 +1,60 @@ +{ + "platform": "linux", + "implementation": "PyPy", + "pypy_version_info": [7, 3, 8, "final", 0], + "version_info": { + "major": 3, + "minor": 8, + "micro": 12, + "releaselevel": "final", + "serial": 0 + }, + "architecture": 64, + "version": "3.8.12 (7.3.8+dfsg-2, Mar 05 2022, 02:04:42)\\n[PyPy 7.3.8 with GCC 11.2.0]", + "os": "posix", + "prefix": "/usr", + "base_prefix": "/usr", + "real_prefix": null, + "base_exec_prefix": "/usr", + "exec_prefix": "/usr", + "executable": "/usr/bin/pypy3", + "original_executable": "/usr/bin/pypy3", + "system_executable": "/usr/bin/pypy3", + "has_venv": true, + "path": ["/usr/lib/pypy3.8", "/usr/local/lib/pypy3.8/dist-packages", "/usr/lib/python3/dist-packages"], + "file_system_encoding": "utf-8", + "stdout_encoding": "UTF-8", + "sysconfig_scheme": null, + "sysconfig_paths": { + "stdlib": "{installed_base}/lib/{implementation_lower}{py_version_short}", + "platstdlib": "{platbase}/lib/{implementation_lower}{py_version_short}", + "purelib": "{base}/local/lib/{implementation_lower}{py_version_short}/dist-packages", + "platlib": "{platbase}/local/lib/{implementation_lower}{py_version_short}/dist-packages", + "include": "{installed_base}/local/include/{implementation_lower}{py_version_short}{abiflags}", + "scripts": "{base}/local/bin", + "data": "{base}" + }, + "distutils_install": { + "purelib": "lib/pypy3.8/site-packages", + "platlib": "lib/pypy3.8/site-packages", + "headers": "include/pypy3.8/UNKNOWN", + "scripts": "bin", + "data": "" + }, + "sysconfig": { + "makefile_filename": "/usr/lib/pypy3.8/config-3.8-x86_64-linux-gnu/Makefile" + }, + "sysconfig_vars": { + "installed_base": "/usr", + "implementation_lower": "pypy", + "py_version_short": "3.8", + "platbase": "/usr", + "base": "/usr", + "abiflags": "", + "PYTHONFRAMEWORK": "" + }, + "system_stdlib": "/usr/lib/pypy3.8", + "system_stdlib_platform": "/usr/lib/pypy3.8", + "max_size": 9223372036854775807, + "_creators": null +} diff --git a/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json b/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json new file mode 100644 index 000000000..2264fa432 --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json @@ -0,0 +1,60 @@ +{ + "platform": "linux", + "implementation": "PyPy", + "pypy_version_info": [7, 3, 8, "final", 0], + "version_info": { + "major": 3, + "minor": 8, + "micro": 12, + "releaselevel": "final", + "serial": 0 + }, + "architecture": 64, + "version": "3.8.12 (d00b0afd2a5dd3c13fcda75d738262c864c62fa7, Feb 18 2022, 09:52:33)\\n[PyPy 7.3.8 with GCC 10.2.1 20210130 (Red Hat 10.2.1-11)]", + "os": "posix", + "prefix": "/tmp/pypy3.8-v7.3.8-linux64", + "base_prefix": "/tmp/pypy3.8-v7.3.8-linux64", + "real_prefix": null, + "base_exec_prefix": "/tmp/pypy3.8-v7.3.8-linux64", + "exec_prefix": "/tmp/pypy3.8-v7.3.8-linux64", + "executable": "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy", + "original_executable": "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy", + "system_executable": "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy", + "has_venv": true, + "path": ["/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8/site-packages"], + "file_system_encoding": "utf-8", + "stdout_encoding": "UTF-8", + "sysconfig_scheme": null, + "sysconfig_paths": { + "stdlib": "{installed_base}/lib/{implementation_lower}{py_version_short}", + "platstdlib": "{platbase}/lib/{implementation_lower}{py_version_short}", + "purelib": "{base}/lib/{implementation_lower}{py_version_short}/site-packages", + "platlib": "{platbase}/lib/{implementation_lower}{py_version_short}/site-packages", + "include": "{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}", + "scripts": "{base}/bin", + "data": "{base}" + }, + "distutils_install": { + "purelib": "lib/pypy3.8/site-packages", + "platlib": "lib/pypy3.8/site-packages", + "headers": "include/pypy3.8/UNKNOWN", + "scripts": "bin", + "data": "" + }, + "sysconfig": { + "makefile_filename": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8/config-3.8-x86_64-linux-gnu/Makefile" + }, + "sysconfig_vars": { + "installed_base": "/tmp/pypy3.8-v7.3.8-linux64", + "implementation_lower": "pypy", + "py_version_short": "3.8", + "platbase": "/tmp/pypy3.8-v7.3.8-linux64", + "base": "/tmp/pypy3.8-v7.3.8-linux64", + "abiflags": "", + "PYTHONFRAMEWORK": "" + }, + "system_stdlib": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", + "system_stdlib_platform": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", + "max_size": 9223372036854775807, + "_creators": null +} diff --git a/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py b/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py new file mode 100644 index 000000000..c4d6860e7 --- /dev/null +++ b/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py @@ -0,0 +1,104 @@ +from __future__ import absolute_import, unicode_literals + +import fnmatch + +from virtualenv.create.via_global_ref.builtin.pypy.pypy3 import PyPy3Posix +from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest +from virtualenv.discovery.py_info import PythonInfo +from virtualenv.util.path import Path + + +class FakePath(Path): + """ + A Path() fake that only knows about files in existing_paths and the + directories that contain them. + """ + + existing_paths = [] + + if hasattr(Path(""), "_flavour"): + _flavour = Path("")._flavour + + def exists(self): + return self.as_posix() in self.existing_paths or self.is_dir() + + def glob(self, glob): + pattern = self.as_posix() + "/" + glob + for path in fnmatch.filter(self.existing_paths, pattern): + yield FakePath(path) + + def is_dir(self): + prefix = self.as_posix() + "/" + return any(True for path in self.existing_paths if path.startswith(prefix)) + + def iterdir(self): + prefix = self.as_posix() + "/" + for path in self.existing_paths: + if path.startswith(prefix) and "/" not in path[len(prefix) :]: + yield FakePath(path) + + def resolve(self): + return self + + def __div__(self, key): + return FakePath(super(FakePath, self).__div__(key)) + + def __truediv__(self, key): + return FakePath(super(FakePath, self).__truediv__(key)) + + +def assert_contains_exe(sources, src): + """Assert that the one and only executeable in sources is src""" + exes = [source for source in sources if isinstance(source, ExePathRefToDest)] + assert len(exes) == 1 + exe = exes[0] + assert exe.src.as_posix() == src + + +def assert_contains_ref(sources, src): + """Assert that src appears in sources""" + assert any(source for source in sources if isinstance(source, PathRefToDest) and source.src.as_posix() == src) + + +def inject_fake_path(mocker, existing_paths): + """Inject FakePath in all the correct places, and set existing_paths""" + FakePath.existing_paths = existing_paths + mocker.patch("virtualenv.create.via_global_ref.builtin.pypy.common.Path", FakePath) + mocker.patch("virtualenv.create.via_global_ref.builtin.pypy.pypy3.Path", FakePath) + + +def _load_pypi_info(name): + return PythonInfo._from_json((Path(__file__).parent / "{}.json".format(name)).read_text()) + + +def test_portable_pypy3_virtualenvs_get_their_libs(mocker): + paths = ["/tmp/pypy3.8-v7.3.8-linux64/bin/pypy", "/tmp/pypy3.8-v7.3.8-linux64/lib/libgdbm.so.4"] + inject_fake_path(mocker, paths) + path = Path("/tmp/pypy3.8-v7.3.8-linux64/bin/libpypy3-c.so") + mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[path]) + + sources = list(PyPy3Posix.sources(interpreter=_load_pypi_info("portable_pypy38"))) + assert_contains_exe(sources, "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy") + assert len(sources) > 2 + assert_contains_ref(sources, "/tmp/pypy3.8-v7.3.8-linux64/bin/libpypy3-c.so") + assert_contains_ref(sources, "/tmp/pypy3.8-v7.3.8-linux64/lib/libgdbm.so.4") + + +def test_debian_pypy37_virtualenvs(mocker): + # Debian's pypy3 layout, installed to /usr, before 3.8 allowed a /usr prefix + inject_fake_path(mocker, ["/usr/bin/pypy3"]) + mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[Path("/usr/lib/pypy3/bin/libpypy3-c.so")]) + sources = list(PyPy3Posix.sources(interpreter=_load_pypi_info("deb_pypy37"))) + assert_contains_exe(sources, "/usr/bin/pypy3") + assert_contains_ref(sources, "/usr/lib/pypy3/bin/libpypy3-c.so") + assert len(sources) == 2 + + +def test_debian_pypy38_virtualenvs_exclude_usr(mocker): + inject_fake_path(mocker, ["/usr/bin/pypy3", "/usr/lib/foo"]) + # libpypy3-c.so lives on the ld search path + mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[]) + + sources = list(PyPy3Posix.sources(interpreter=_load_pypi_info("deb_pypy38"))) + assert_contains_exe(sources, "/usr/bin/pypy3") + assert len(sources) == 1