Hooks: Fix sysconfig and distutils.sysconfig.

PyInstaller/hooks/hook-distutils.py

@@ -17,19 +17,14 @@
 runtime for platform-specific metadata.
 """
 
-# TODO Verify that bundling Makefile and pyconfig.h is still required for Python 3.
-
-import os
+from PyInstaller import compat
+
+# From Python 3.6 and later ``distutils.sysconfig`` takes on the same
+# behaviour as regular ``sysconfig`` of moving the config vars to a
+# module (see hook-sysconfig.py). It doesn't use a nice
+# `get module name` function like ``sysconfig`` does to help us
+# locate it but the module is the same file that ``sysconfig`` uses so
+# we can use the ``_get_sysconfigdata_name()`` from regular ``sysconfig``.
 import sysconfig
-
-from PyInstaller.utils.hooks import relpath_to_config_or_make
-
-_CONFIG_H = sysconfig.get_config_h_filename()
-_MAKEFILE = sysconfig.get_makefile_filename()
-
-# Data files in PyInstaller hook format.
-datas = [(_CONFIG_H, relpath_to_config_or_make(_CONFIG_H))]
-
-# The Makefile does not exist on all platforms, eg. on Windows
-if os.path.exists(_MAKEFILE):
-    datas.append((_MAKEFILE, relpath_to_config_or_make(_MAKEFILE)))
+if not compat.is_win and hasattr(sysconfig, '_get_sysconfigdata_name'):
+    hiddenimports = [sysconfig._get_sysconfigdata_name()]

PyInstaller/hooks/hook-sysconfig.py

@@ -9,27 +9,10 @@
 # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception)
 #-----------------------------------------------------------------------------
 
-
-# The 'sysconfig' module requires Makefile and pyconfig.h files from
-# Python installation. 'sysconfig' parses these files to get some
-# information from them.
-# TODO Verify that bundling Makefile and pyconfig.h is still required for Python 3.
-
 import sysconfig
-import os
 
-from PyInstaller.utils.hooks import relpath_to_config_or_make
 from PyInstaller.compat import is_win
 
-_CONFIG_H = sysconfig.get_config_h_filename()
-_MAKEFILE = sysconfig.get_makefile_filename()
-
-
-datas = [(_CONFIG_H, relpath_to_config_or_make(_CONFIG_H))]
-
-# The Makefile does not exist on all platforms, eg. on Windows
-if os.path.exists(_MAKEFILE):
-    datas.append((_MAKEFILE, relpath_to_config_or_make(_MAKEFILE)))
 
 if not is_win and hasattr(sysconfig, '_get_sysconfigdata_name'):
     # Python 3.6 uses additional modules like

PyInstaller/utils/hooks/__init__.py

@@ -892,40 +892,6 @@ def collect_system_data_files(path, destdir=None, include_py_files=False):
     return datas
 
 
-def _find_prefix(filename):
-    """
-    In virtualenv, _CONFIG_H and _MAKEFILE may have same or different
-    prefixes, depending on the version of virtualenv.
-    Try to find the correct one, which is assumed to be the longest one.
-    """
-    if not is_venv:
-        return sys.prefix
-    filename = os.path.abspath(filename)
-    prefixes = [os.path.abspath(sys.prefix), base_prefix]
-    possible_prefixes = []
-    for prefix in prefixes:
-        common = os.path.commonprefix([prefix, filename])
-        if common == prefix:
-            possible_prefixes.append(prefix)
-    if not possible_prefixes:
-        # no matching prefix, assume running from build directory
-        possible_prefixes = [os.path.dirname(filename)]
-    possible_prefixes.sort(key=lambda p: len(p), reverse=True)
-    return possible_prefixes[0]
-
-
-def relpath_to_config_or_make(filename):
-    """
-    The following is refactored out of hook-sysconfig and hook-distutils,
-    both of which need to generate "datas" tuples for pyconfig.h and
-    Makefile, under the same conditions.
-    """
-
-    # Relative path in the dist directory.
-    prefix = _find_prefix(filename)
-    return os.path.relpath(os.path.dirname(filename), prefix)
-
-
 def copy_metadata(package_name):
     """
     This function returns a list to be assigned to the ``datas`` global

doc/conf.py

@@ -69,6 +69,7 @@
 
 intersphinx_mapping = {
     'website': ('http://www.pyinstaller.org//', None),
+    'python': ('http://docs.python.org/3', None),
 }
 
 # Add any paths that contain templates here, relative to this directory.

news/5018.hooks.rst

@@ -0,0 +1 @@
+Update hook for ``sysconfig`` to be compatible with pyenv-virtualenv.

news/5218.break.rst

@@ -0,0 +1,4 @@
+No longer collect ``pyconfig.h`` and ``makefile`` for :mod:`sysconfig`. Instead
+of :func:`~sysconfig.get_config_h_filename` and
+:func:`~sysconfig.get_makefile_filename`, you should use
+:func:`~sysconfig.get_config_vars` which no longer depends on those files.

news/5218.hooks.rst

@@ -0,0 +1 @@
+Update hook for ``distutils.sysconfig`` to be compatible with pyenv-virtualenv.

tests/functional/test_basic.py

@@ -332,10 +332,41 @@ def MyEXE(*args, **kwargs):
     assert "'import warnings' failed" not in err
 
 
-@pytest.mark.darwin
-@pytest.mark.linux
-def test_python_makefile(pyi_builder):
-    pyi_builder.test_script('pyi_python_makefile.py')
+@pytest.mark.parametrize("distutils", ["", "from distutils "])
+def test_python_makefile(pyi_builder, distutils):
+    """Tests hooks for ``sysconfig`` and its near-duplicate
+    ``distutils.sysconfig``. Raises an import error if we failed to collect the
+    special module that contains the details from pyconfig.h and the makefile.
+    """
+    # Ideally we'd test that the contents of `sysconfig.get_config_vars()` dict
+    # are the same frozen vs unfrozen but because some values are paths into
+    # a Python installation's guts, these will point into the frozen app when
+    # frozen and therefore noy match. Without some fiddly filtering away paths,
+    # this is impossible.
+
+    # As a compromise, test that the dictionary keys are the same to be sure
+    # that there is no conditional initialisation of get_config_vars(). i.e.
+    # get_config_vars() doesn't silently return an empty dictionary if it can't
+    # find the information it needs.
+    if distutils:
+        from distutils import sysconfig
+    else:
+        import sysconfig
+    unfrozen_keys = sorted(sysconfig.get_config_vars().keys())
+
+    pyi_builder.test_source("""
+    # The error is raised immediately on import.
+    {}import sysconfig
+
+    # But just in case, Python later opt for some lazy loading, force
+    # configuration retrieval:
+    from pprint import pprint
+    pprint(sysconfig.get_config_vars())
+
+    unfrozen_keys = {}
+    assert sorted(sysconfig.get_config_vars()) == unfrozen_keys
+
+    """.format(distutils, unfrozen_keys))
 
 
 def test_set_icon(pyi_builder, data_dir):