hook-numpy.py

#!/usr/bin/env python3

# --- Copyright Disclaimer ---
#
# In order to support PyInstaller with numpy<1.20.0 this file will be
# duplicated for a short period inside PyInstaller's repository [1]. However
# this file is the intellectual property of the NumPy team and is under the
# terms and conditions outlined their repository [2].
#
# .. refs:
#
#   [1] PyInstaller: https://github.com/pyinstaller/pyinstaller/
#   [2] NumPy's license: https://github.com/numpy/numpy/blob/master/LICENSE.txt
#

"""
This hook should collect all binary files and any hidden modules that numpy
needs.

Our (some-what inadequate) docs for writing PyInstaller hooks are kept here:
https://pyinstaller.readthedocs.io/en/stable/hooks.html

PyInstaller has a lot of numpy users so we'd consider maintaining this hook to
be high priority. Feel free to @mention either bwoodsend or Legorooj on Github
for help keeping it working.
"""

import os
import re
from pathlib import Path

from PyInstaller.utils.hooks import collect_dynamic_libs, exec_statement, logger
from PyInstaller import compat

# --- Plain official numpy from PyPI ---

# Supporting regular numpy is actually dead easy.

# We need to collect all used dll/so/dylibs. As these are all inside of numpy's
# `site-packages/numpy` directory we can just use the following:
binaries = collect_dynamic_libs("numpy", ".")
# Note: this does not collect Python extension modules. They are found using
# Python's native import scheme.

# PyInstaller can't detect imports from a cython or C extension module. If a
# submodule is *only* ever `import`ed by a cython or C module then it wont be
# found. Likewise __import__ and importlib are invisible to PyInstaller.
# These issues should be very easy to track. You'll get a ModuleNotFoundError
# for that hidden module at runtime (usually straight away) if you forget to
# include one.
hiddenimports = ['numpy.core._dtype_ctypes']


# --- Additional support for less official mkl builds ---

# Check if MKL is being used.
# We avoid using `import numpy` directly in hooks in-case doing so alters either
# sys.path or PATH which could confuse the build.
is_mkl = exec_statement("""
# XXX: Numpy devs - is this a good way to test if using MKL?
import numpy
print(bool(numpy.__config__.blas_mkl_info))
""") == "True"

# The MKL binaries themselves are included inside the numpy folder and will
# therefore already have been found by `collect_dynamic_libs()` above.

def find_library(name):
    """Glob-find and include a dll (like) binary file which is usually found by
    searching PATH.
    """
    # We'll hopefully include this in later versions of PyInstaller. So that
    # NumPy remains compatible with PyInstaller 4.0, I'm copying it here.

    names = set()
    binaries = []
    for folder in os.environ["PATH"].split(os.pathsep):
        for path in Path(folder).glob(name):
            if not path.name in names:
                binaries.append((str(path), "."))
                names.add(path.name)
    if not binaries:
        logger.warning(
            "Failed to find '%s' DLL in PATH. Your app will likely crash if run"
            " on a different machine that doesn't already have it.", name
        )
    return binaries


if is_mkl:
    # Other dlls that MKL uses which PyInstaller can't detect itself. These are
    # determined empirically using dynamic dependency sniffing and will lead
    # to issues in the future should they change...
    for lib in ["libcrypto*", "libffi*", "libssl*"]:
        binaries.extend(find_library(lib))


# --- A vain attempt at Conda's numpy support ---

# Regular numpy, even with unofficial mkl builds, is pretty trivial to support
# with PyInstaller. Unfortunately Conda's numpy is the opposite. We need
# their help to maintain this because our own attempts have been a disaster.

if compat.is_conda:
    hiddenimports.append("six")

    # There are so many hidden binary dependencies. This list is heavily, OS,
    # Python and NumPy versions dependent. Omitting any of these can lead to
    # obscure and often traceback-less crashes.
    # XXX: As you can see, this is really not a scalable solution. Needs help!
    conda_dll_patterns = [re.compile(i) for i in (
        'apphelp.*', 'crypt32.*', 'imagehlp.*', 'libblas.*',
        'libcblas.*', 'libcrypto.*', 'libffi.*', 'libgcc_.*',
        'libgfortran.*', 'libifcoremd.*', r'libiomp\d+md.*', 'liblapack.*',
        'libmmd.*', 'libomp.*',
        'libopenblas.*', 'libquadmath.*', 'libssl.*', 'libuuid.*',
        'libz.*', 'mkl_avx.*', 'mkl_core.*',
        'mkl_intel_thread.*', 'mkl_rt.*', 'mkl_vml_avx.*',
        'mkl_vml_avx.*', 'msasn.*', 'mswsock.*', 'ole.*',
        'oleaut.*', 'tbbmalloc.*', 'urandom'
    )]

    if compat.is_win:
        lib_dir = os.path.join(compat.base_prefix, "Library", "bin")
    else:
        lib_dir = os.path.join(compat.base_prefix, "lib")

    def _is_required(name):
        return any(pattern.match(name) for pattern in conda_dll_patterns)

    _to_add = set(filter(_is_required, os.listdir(lib_dir)))

    for name in _to_add:
        binaries.append((os.path.join(lib_dir, name), "."))


# --- Remove testing and building code ---

excludedimports = ["scipy", "pytest", "nose", "distutils", "f2py", "setuptools",
                   "numpy.f2py", "numpy.distutils"]

# I would suggest using the following to remove all the `tests` submodules but
# we don't need it. They will be included if any modules that are included
# contain an explicit `import numpy.xxx.tests`. Should you're tests structure
# change so that they start to get sucked in, uncomment the lines below.

# from PyInstaller.utils.hooks import collect_submodules
# is_tests = lambda x: "tests" in x.split(".")
# excludedimports += collect_submodules("numpy", filter=is_tests)


# --- Remove binaries that aren't DLLs ---


import ctypes

def _is_valid(source, dest):
    # There really should be a less brute-force way of doing this.
    if source.endswith(".pdb"):
        # Attempting to load a pdb causes a pop-up window. Check for and exclude
        # them here.
        return False
    try:
        ctypes.CDLL(source)
        return True
    except OSError:
        return False

binaries = [i for i in binaries if _is_valid(*i)]