From 1c153f7fd325de492953f2eb980808d6ba099f78 Mon Sep 17 00:00:00 2001 From: bwoodsend Date: Sat, 3 Oct 2020 14:58:29 +0100 Subject: [PATCH 1/5] Tests: Remove test for collection of config.h and Makefile. The pyconfig.h and makefile are no longer used directly. Rather configuration is baked into `.py` files. --- .../functional/scripts/pyi_python_makefile.py | 36 ------------------- tests/functional/test_basic.py | 4 --- 2 files changed, 40 deletions(-) delete mode 100644 tests/functional/scripts/pyi_python_makefile.py diff --git a/tests/functional/scripts/pyi_python_makefile.py b/tests/functional/scripts/pyi_python_makefile.py deleted file mode 100644 index ef975ace3d..0000000000 --- a/tests/functional/scripts/pyi_python_makefile.py +++ /dev/null @@ -1,36 +0,0 @@ -#----------------------------------------------------------------------------- -# Copyright (c) 2005-2021, PyInstaller Development Team. -# -# Distributed under the terms of the GNU General Public License (version 2 -# or later) with exception for distributing the bootloader. -# -# The full license is in the file COPYING.txt, distributed with this software. -# -# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) -#----------------------------------------------------------------------------- - - -# distutils module requires Makefile and pyconfig.h files from Python -# installation. - - -import os -import sys -import sysconfig - - -config_h = sysconfig.get_config_h_filename() -print(('pyconfig.h: ' + config_h)) -files = [config_h] - - -# On Windows Makefile does not exist. -if not sys.platform.startswith('win'): - makefile = sysconfig.get_makefile_filename() - print(('Makefile: ' + makefile)) - files.append(makefile) - - -for f in files: - if not os.path.exists(f): - raise SystemExit('File does not exist: %s' % f) diff --git a/tests/functional/test_basic.py b/tests/functional/test_basic.py index 4c025aff9d..e5ab05c380 100644 --- a/tests/functional/test_basic.py +++ b/tests/functional/test_basic.py @@ -332,10 +332,6 @@ def MyEXE(*args, **kwargs): assert "'import warnings' failed" not in err -@pytest.mark.darwin -@pytest.mark.linux -def test_python_makefile(pyi_builder): - pyi_builder.test_script('pyi_python_makefile.py') def test_set_icon(pyi_builder, data_dir): From b5be6af7c024bc4e311c4f34f2fbb51ea8bd33b1 Mon Sep 17 00:00:00 2001 From: bwoodsend Date: Tue, 29 Dec 2020 12:06:17 +0000 Subject: [PATCH 2/5] Tests: Add test for [distutils.]sysconfig.get_config_vars(). --- tests/functional/test_basic.py | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/functional/test_basic.py b/tests/functional/test_basic.py index e5ab05c380..a8893e06e4 100644 --- a/tests/functional/test_basic.py +++ b/tests/functional/test_basic.py @@ -332,6 +332,41 @@ def MyEXE(*args, **kwargs): assert "'import warnings' failed" not in err +@pytest.mark.parametrize("distutils", ["", "from distutils "]) +def test_python_makefile(pyi_builder, distutils): + """Tests hooks for ``sysconfig`` and its near-duplicate + ``distutils.sysconfig``. Raises an import error if we failed to collect the + special module that contains the details from pyconfig.h and the makefile. + """ + # Ideally we'd test that the contents of `sysconfig.get_config_vars()` dict + # are the same frozen vs unfrozen but because some values are paths into + # a Python installation's guts, these will point into the frozen app when + # frozen and therefore noy match. Without some fiddly filtering away paths, + # this is impossible. + + # As a compromise, test that the dictionary keys are the same to be sure + # that there is no conditional initialisation of get_config_vars(). i.e. + # get_config_vars() doesn't silently return an empty dictionary if it can't + # find the information it needs. + if distutils: + from distutils import sysconfig + else: + import sysconfig + unfrozen_keys = sorted(sysconfig.get_config_vars().keys()) + + pyi_builder.test_source(""" + # The error is raised immediately on import. + {}import sysconfig + + # But just in case, Python later opt for some lazy loading, force + # configuration retrieval: + from pprint import pprint + pprint(sysconfig.get_config_vars()) + + unfrozen_keys = {} + assert sorted(sysconfig.get_config_vars()) == unfrozen_keys + + """.format(distutils, unfrozen_keys)) def test_set_icon(pyi_builder, data_dir): From ea3a2e1582be6aef6bdcb7c12ea1730a184d0ef0 Mon Sep 17 00:00:00 2001 From: bwoodsend Date: Wed, 9 Sep 2020 10:18:43 +0100 Subject: [PATCH 3/5] Hooks: sysconfig: Don't collect pyconfig.h and makefile. As of Python 3.x, the config header and makefile are not longer used. Trying to include them causes build errors in pyenv/venv/equivalents. Applying this fixes 5018, fixes 4775, and closes 1545 (a cleanup issue). On Windows: The pyconfig.h data is hardcoded directly into sysconfig.py. And the makefile does not exist. On Unix: A Python extension module contains all the details. This module is a hidden import since 3.6 but is already marked as such in the hook. The functions to find and parse the files are still left over in sysconfig.py but they are almost fully replaced by functions to get configuration parameters directly. Should a user attempt to invoke the parser in a PyInstaller build, it will now fail. They should instead use get_config_var() which will always work (and is the recommended usage anyway). --- PyInstaller/hooks/hook-sysconfig.py | 17 ----------------- doc/conf.py | 1 + news/5018.hooks.rst | 1 + news/5218.break.rst | 4 ++++ 4 files changed, 6 insertions(+), 17 deletions(-) create mode 100644 news/5018.hooks.rst create mode 100644 news/5218.break.rst diff --git a/PyInstaller/hooks/hook-sysconfig.py b/PyInstaller/hooks/hook-sysconfig.py index 225afab966..07e35d6404 100644 --- a/PyInstaller/hooks/hook-sysconfig.py +++ b/PyInstaller/hooks/hook-sysconfig.py @@ -9,27 +9,10 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# The 'sysconfig' module requires Makefile and pyconfig.h files from -# Python installation. 'sysconfig' parses these files to get some -# information from them. -# TODO Verify that bundling Makefile and pyconfig.h is still required for Python 3. - import sysconfig -import os -from PyInstaller.utils.hooks import relpath_to_config_or_make from PyInstaller.compat import is_win -_CONFIG_H = sysconfig.get_config_h_filename() -_MAKEFILE = sysconfig.get_makefile_filename() - - -datas = [(_CONFIG_H, relpath_to_config_or_make(_CONFIG_H))] - -# The Makefile does not exist on all platforms, eg. on Windows -if os.path.exists(_MAKEFILE): - datas.append((_MAKEFILE, relpath_to_config_or_make(_MAKEFILE))) if not is_win and hasattr(sysconfig, '_get_sysconfigdata_name'): # Python 3.6 uses additional modules like diff --git a/doc/conf.py b/doc/conf.py index b9530b7d18..a5d4ce5ef8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -69,6 +69,7 @@ intersphinx_mapping = { 'website': ('http://www.pyinstaller.org//', None), + 'python': ('http://docs.python.org/3', None), } # Add any paths that contain templates here, relative to this directory. diff --git a/news/5018.hooks.rst b/news/5018.hooks.rst new file mode 100644 index 0000000000..2687fd91ee --- /dev/null +++ b/news/5018.hooks.rst @@ -0,0 +1 @@ +Update hook for ``sysconfig`` to be compatible with pyenv-virtualenv. diff --git a/news/5218.break.rst b/news/5218.break.rst new file mode 100644 index 0000000000..da621adad7 --- /dev/null +++ b/news/5218.break.rst @@ -0,0 +1,4 @@ +No longer collect ``pyconfig.h`` and ``makefile`` for :mod:`sysconfig`. Instead +of :func:`~sysconfig.get_config_h_filename` and +:func:`~sysconfig.get_makefile_filename`, you should use +:func:`~sysconfig.get_config_vars` which no longer depends on those files. From 5806950707d817958c2389853d0416b7add69051 Mon Sep 17 00:00:00 2001 From: bwoodsend Date: Fri, 2 Oct 2020 22:32:13 +0100 Subject: [PATCH 4/5] Hooks: distutils: Fix no suitable dest location found for pyconfig.h. Locating `pyconfig.h` and the `makefile` gets into a mess when using certain environment managers (pyenv-virtualenv) because PyInstaller, whilst trying to find an appropriate `dest` path to put the files in, gets confused by `sys.prefix` (or its varients) not being a parent dir of the config and makefiles. As of Python 3.6, direct parsing of `pyconfig.h` and `makefile` are replaced by a Python module generated on building Python so that these files are no longer required. --- PyInstaller/hooks/hook-distutils.py | 25 ++++++++++--------------- news/5218.hooks.rst | 1 + 2 files changed, 11 insertions(+), 15 deletions(-) create mode 100644 news/5218.hooks.rst diff --git a/PyInstaller/hooks/hook-distutils.py b/PyInstaller/hooks/hook-distutils.py index efb98e8740..36f322726f 100644 --- a/PyInstaller/hooks/hook-distutils.py +++ b/PyInstaller/hooks/hook-distutils.py @@ -17,19 +17,14 @@ runtime for platform-specific metadata. """ -# TODO Verify that bundling Makefile and pyconfig.h is still required for Python 3. - -import os +from PyInstaller import compat + +# From Python 3.6 and later ``distutils.sysconfig`` takes on the same +# behaviour as regular ``sysconfig`` of moving the config vars to a +# module (see hook-sysconfig.py). It doesn't use a nice +# `get module name` function like ``sysconfig`` does to help us +# locate it but the module is the same file that ``sysconfig`` uses so +# we can use the ``_get_sysconfigdata_name()`` from regular ``sysconfig``. import sysconfig - -from PyInstaller.utils.hooks import relpath_to_config_or_make - -_CONFIG_H = sysconfig.get_config_h_filename() -_MAKEFILE = sysconfig.get_makefile_filename() - -# Data files in PyInstaller hook format. -datas = [(_CONFIG_H, relpath_to_config_or_make(_CONFIG_H))] - -# The Makefile does not exist on all platforms, eg. on Windows -if os.path.exists(_MAKEFILE): - datas.append((_MAKEFILE, relpath_to_config_or_make(_MAKEFILE))) +if not compat.is_win and hasattr(sysconfig, '_get_sysconfigdata_name'): + hiddenimports = [sysconfig._get_sysconfigdata_name()] diff --git a/news/5218.hooks.rst b/news/5218.hooks.rst new file mode 100644 index 0000000000..9de05c4e18 --- /dev/null +++ b/news/5218.hooks.rst @@ -0,0 +1 @@ +Update hook for ``distutils.sysconfig`` to be compatible with pyenv-virtualenv. From 1557991b96f3119243a8ff420529b95f5a9d44df Mon Sep 17 00:00:00 2001 From: bwoodsend Date: Fri, 2 Oct 2020 23:33:34 +0100 Subject: [PATCH 5/5] Cleanup: Remove now unused hooks utilities. [skip-ci] --- PyInstaller/utils/hooks/__init__.py | 34 ----------------------------- 1 file changed, 34 deletions(-) diff --git a/PyInstaller/utils/hooks/__init__.py b/PyInstaller/utils/hooks/__init__.py index 43435bf316..d4d78bb1ad 100755 --- a/PyInstaller/utils/hooks/__init__.py +++ b/PyInstaller/utils/hooks/__init__.py @@ -892,40 +892,6 @@ def collect_system_data_files(path, destdir=None, include_py_files=False): return datas -def _find_prefix(filename): - """ - In virtualenv, _CONFIG_H and _MAKEFILE may have same or different - prefixes, depending on the version of virtualenv. - Try to find the correct one, which is assumed to be the longest one. - """ - if not is_venv: - return sys.prefix - filename = os.path.abspath(filename) - prefixes = [os.path.abspath(sys.prefix), base_prefix] - possible_prefixes = [] - for prefix in prefixes: - common = os.path.commonprefix([prefix, filename]) - if common == prefix: - possible_prefixes.append(prefix) - if not possible_prefixes: - # no matching prefix, assume running from build directory - possible_prefixes = [os.path.dirname(filename)] - possible_prefixes.sort(key=lambda p: len(p), reverse=True) - return possible_prefixes[0] - - -def relpath_to_config_or_make(filename): - """ - The following is refactored out of hook-sysconfig and hook-distutils, - both of which need to generate "datas" tuples for pyconfig.h and - Makefile, under the same conditions. - """ - - # Relative path in the dist directory. - prefix = _find_prefix(filename) - return os.path.relpath(os.path.dirname(filename), prefix) - - def copy_metadata(package_name): """ This function returns a list to be assigned to the ``datas`` global