ENH: add hook and test for PyInstaller.

Adding this special hook file tells PyInstaller what files a self contained
NumPy application needs to run. A test is included to verify that the hook still
finds all necessary files.

Closes numpy#17184.

numpy/__init__.py

@@ -413,6 +413,11 @@ def _mac_os_check():
     # it is tidier organized.
     core.multiarray._multiarray_umath._reload_guard()
 
+    # Tell PyInstaller where to find hook-numpy.py
+    def _pyinstaller_hooks_dir():
+        from pathlib import Path
+        return [str(Path(__file__).with_name("_pyinstaller").resolve())]
+
 
 # get the version using versioneer
 from .version import __version__, git_revision as __git_version__

numpy/_pyinstaller/hook-numpy.py

@@ -0,0 +1,47 @@
+"""This hook should collect all binary files and any hidden modules that numpy
+needs.
+
+Our (some-what inadequate) docs for writing PyInstaller hooks are kept here:
+https://pyinstaller.readthedocs.io/en/stable/hooks.html
+
+"""
+from PyInstaller.compat import is_conda, is_pure_conda
+from PyInstaller.utils.hooks import collect_dynamic_libs, is_module_satisfies
+
+# Collect all DLLs inside numpy's installation folder, dump them into built
+# app's root.
+binaries = collect_dynamic_libs("numpy", ".")
+
+# If using Conda without any non-conda virtual environment manager:
+if is_pure_conda:
+    # Assume running the NumPy from Conda-forge and collect it's DLLs from the
+    # communal Conda bin directory. DLLs from NumPy's dependencies must also be
+    # collected to capture MKL, OpenBlas, OpenMP, etc.
+    from PyInstaller.utils.hooks import conda_support
+    datas = conda_support.collect_dynamic_libs("numpy", dependencies=True)
+
+# Submodules PyInstaller cannot detect (probably because they are only imported
+# by extension modules, which PyInstaller cannot read).
+hiddenimports = ['numpy.core._dtype_ctypes']
+if is_conda:
+    hiddenimports.append("six")
+
+# Remove testing and building code and packages that are referenced throughout
+# NumPy but are not really dependencies.
+excludedimports = [
+    "scipy",
+    "pytest",
+    "nose",
+    "f2py",
+    "setuptools",
+    "numpy.f2py",
+]
+
+# As of version 1.22, numpy.testing (imported for example by some scipy
+# modules) requires numpy.distutils and distutils. So exclude them only for
+# earlier versions. See #20769.
+if is_module_satisfies("numpy < 1.22"):
+    excludedimports += [
+        "distutils",
+        "numpy.distutils",
+    ]

numpy/_pyinstaller/pyinstaller-smoke.py

@@ -0,0 +1,32 @@
+"""A crude *bit of everything* smoke test to verify PyInstaller compatibility.
+
+PyInstaller typically goes wrong by forgetting to package modules, extension
+modules or shared libraries. This script should aim to touch as many of those
+as possible in an attempt to trip a ModuleNotFoundError or a DLL load failure
+due to an uncollected resource. Missing resources are unlikely to lead to
+arithmitic errors so there's generally no need to verify any calculation's
+output - merely that it made it to the end OK. This script should not
+explicitly import any of numpy's submodules as that gives PyInstaller undue
+hints that those submodules exist and should be collected (accessing implicitly
+loaded submodules is OK).
+
+"""
+import numpy as np
+
+a = np.arange(1., 10.).reshape((3, 3)) % 5
+np.linalg.det(a)
+a @ a
+a @ a.T
+np.linalg.inv(a)
+np.sin(np.exp(a))
+np.linalg.svd(a)
+np.linalg.eigh(a)
+
+np.unique(np.random.randint(0, 10, 100))
+np.sort(np.random.uniform(0, 10, 100))
+
+np.fft.fft(np.exp(2j * np.pi * np.arange(8) / 8))
+np.ma.masked_array(np.arange(10), np.random.rand(10) < .5).sum()
+np.polynomial.Legendre([7, 8, 9]).roots()
+
+print("I made it!")

numpy/_pyinstaller/test_pyinstaller.py

@@ -0,0 +1,35 @@
+import subprocess
+from pathlib import Path
+
+import pytest
+
+
+# PyInstaller has been very unproactive about replacing 'imp' with 'importlib'.
+@pytest.mark.filterwarnings('ignore::DeprecationWarning')
+# It also leaks io.BytesIO()s.
+@pytest.mark.filterwarnings('ignore::ResourceWarning')
+@pytest.mark.parametrize("mode", ["--onedir", "--onefile"])
+@pytest.mark.slow
+def test_pyinstaller(mode, tmp_path):
+    """Compile and run pyinstaller-smoke.py using PyInstaller."""
+
+    pyinstaller_cli = pytest.importorskip("PyInstaller.__main__").run
+
+    source = Path(__file__).with_name("pyinstaller-smoke.py").resolve()
+    args = [
+        # Place all generated files in ``tmp_path``.
+        '--workpath', str(tmp_path / "build"),
+        '--distpath', str(tmp_path / "dist"),
+        '--specpath', str(tmp_path),
+        mode,
+        str(source),
+    ]
+    pyinstaller_cli(args)
+
+    if mode == "--onefile":
+        exe = tmp_path / "dist" / source.stem
+    else:
+        exe = tmp_path / "dist" / source.stem / source.stem
+
+    p = subprocess.run([str(exe)], check=True, stdout=subprocess.PIPE)
+    assert p.stdout.strip() == b"I made it!"

numpy/setup.py

@@ -23,6 +23,7 @@ def configuration(parent_package='',top_path=None):
     config.add_data_files('py.typed')
     config.add_data_files('*.pyi')
     config.add_subpackage('tests')
+    config.add_subpackage('_pyinstaller')
     config.make_config_py() # installs __config__.py
     return config
 

setup.py

@@ -417,6 +417,7 @@ def setup_package():
         entry_points={
             'console_scripts': f2py_cmds,
             'array_api': ['numpy = numpy.array_api'],
+            'pyinstaller40': ['hook-dirs = numpy:_pyinstaller_hooks_dir'],
         },
     )
 

tools/travis-test.sh

@@ -83,7 +83,7 @@ run_test()
   # in test_requirements.txt) does not provide a wheel, and the source tar
   # file does not install correctly when Python's optimization level is set
   # to strip docstrings (see https://github.com/eliben/pycparser/issues/291).
-  PYTHONOPTIMIZE="" $PIP install -r test_requirements.txt
+  PYTHONOPTIMIZE="" $PIP install -r test_requirements.txt pyinstaller
   DURATIONS_FLAG="--durations 10"
 
   if [ -n "$USE_DEBUG" ]; then