building: collect built-in extensions into lib-dynload sub-directory

PyInstaller/building/api.py

@@ -840,7 +840,11 @@ def _set_dependencies(self, analysis, path):
                     # relative path needs to be reconstructed from the
                     # name components.
                     if tpl[2] == 'EXTENSION':
-                        ext_components = tpl[0].split('.')[:-1]
+                        # Split on os.path.sep first, to handle additional
+                        # path prefix (e.g., lib-dynload)
+                        ext_components = tpl[0].split(os.path.sep)
+                        ext_components = ext_components[:-1] \
+                            + ext_components[-1].split('.')[:-1]
                         if ext_components:
                             rel_path = os.path.join(*ext_components)
                         else:

PyInstaller/building/build_main.py

@@ -480,6 +480,18 @@ def assemble(self):
             self.binding_redirects[:] = list(set(self.binding_redirects))
             logger.info("Found binding redirects: \n%s", self.binding_redirects)
 
+        # Filter binaries to adjust path of extensions that come from
+        # python's lib-dynload directory. Prefix them with lib-dynload
+        # so that we'll collect them into subdirectory instead of
+        # directly into _MEIPASS
+        for idx, tpl in enumerate(self.binaries):
+            name, path, typecode = tpl
+            if typecode == 'EXTENSION' \
+               and not os.path.dirname(os.path.normpath(name)) \
+               and os.path.basename(os.path.dirname(path)) == 'lib-dynload':
+                name = os.path.join('lib-dynload', name)
+                self.binaries[idx] = (name, path, typecode)
+
         # Place Python source in data files for the noarchive case.
         if self.noarchive:
             # Create a new TOC of ``(dest path for .pyc, source for .py, type)``.

bootloader/src/pyi_pythonlib.c

@@ -392,10 +392,11 @@ pyi_pylib_start_python(ARCHIVE_STATUS *status)
      * its contents nor free its memory.
      *
      * NOTE: Statics are zero-initialized. */
-    static char pypath[2 * PATH_MAX + 14];
+    #define MAX_PYPATH_SIZE (3 * PATH_MAX + 32)
+    static char pypath[MAX_PYPATH_SIZE];
 
     /* Wide string forms of the above, for Python 3. */
-    static wchar_t pypath_w[PATH_MAX + 1];
+    static wchar_t pypath_w[MAX_PYPATH_SIZE];
     static wchar_t pyhome_w[PATH_MAX + 1];
     static wchar_t progname_w[PATH_MAX + 1];
 
@@ -419,23 +420,27 @@ pyi_pylib_start_python(ARCHIVE_STATUS *status)
     PI_Py_SetPythonHome(pyhome_w);
 
     /* Set sys.path */
-    /* sys.path = [base_library, mainpath] */
-    if (snprintf(pypath, sizeof pypath, "%s%cbase_library.zip%c%s",
-                 status->mainpath, PYI_SEP, PYI_PATHSEP, status->mainpath)
-        >= sizeof pypath) {
+    /* sys.path = [mainpath/base_library.zip, mainpath/lib-dynload, mainpath] */
+    if (snprintf(pypath, MAX_PYPATH_SIZE, "%s%c%s" "%c" "%s%c%s" "%c" "%s",
+                 status->mainpath, PYI_SEP, "base_library.zip",
+                 PYI_PATHSEP,
+                 status->mainpath, PYI_SEP, "lib-dynload",
+                 PYI_PATHSEP,
+                 status->mainpath)
+        >= MAX_PYPATH_SIZE) {
         // This should never happen, since mainpath is < PATH_MAX and pypath is
         // huge enough
         FATALERROR("sys.path (based on %s) exceeds buffer[%d] space\n",
-                   status->mainpath, sizeof pypath);
+                   status->mainpath, MAX_PYPATH_SIZE);
         return -1;
     }
 
     /*
-     * E must set sys.path to have base_library.zip before
+     * We must set sys.path to have base_library.zip before
      * calling Py_Initialize as it needs `encodings` and other modules.
      */
     /* Decode using current locale */
-    if (!pyi_locale_char2wchar(pypath_w, pypath, PATH_MAX)) {
+    if (!pyi_locale_char2wchar(pypath_w, pypath, MAX_PYPATH_SIZE)) {
         FATALERROR("Failed to convert pypath to wchar_t\n");
         return -1;
     }

news/5583.bugfix.rst

@@ -0,0 +1 @@
+(OSX) Fix issues with ``pycryptodomex`` on macOS.

news/5604.core.rst

@@ -0,0 +1,5 @@
+Collect python extension modules that correspond to built-ins into
+``lib-dynload`` sub-directory instead of directly into bundle's root
+directory. This prevents them from shadowing shared libraries with the
+same basename that are located in a package and loaded via ``ctypes`` or
+``cffi``, and also declutters the bundle's root directory.

tests/functional/test_import.py

@@ -11,6 +11,7 @@
 #-----------------------------------------------------------------------------
 
 import os
+import sys
 import glob
 import ctypes
 import ctypes.util
@@ -371,6 +372,31 @@ def g():
     pyi_builder.test_source(source % locals(), test_id=test_id)
 
 
+def test_ctypes_cdll_builtin_extension(pyi_builder):
+    # Take a built-in that is provided as an extension
+    builtin_ext = '_sha256'
+    if builtin_ext in sys.builtin_module_names:
+        # On Windows, built-ins do not seem to be extensions
+        pytest.skip(f"{builtin_ext} is a built-in module without extension.")
+
+    pyi_builder.test_source(
+        """
+        import ctypes
+        import importlib.machinery
+
+        # Try to load CDLL with all possible extension suffices; this
+        # should fail in all cases, as built-in extensions should not
+        # be in the ctypes' search path.
+        builtin_ext = '{0}'
+        for suffix in importlib.machinery.EXTENSION_SUFFIXES:
+            try:
+                lib = ctypes.CDLL(builtin_ext + suffix)
+            except OSError:
+                lib = None
+            assert lib is None, "Built-in extension picked up by ctypes.CDLL!"
+        """.format(builtin_ext))
+
+
 # TODO: Add test-cases for the prefabricated library loaders supporting
 # attribute accesses on windows. Example::
 #