Merge 'Moving hooks to the packages they support' into develop

Closes pyinstaller#4582, pyinstaller#4232, pyinstaller#4301.

.travis.yml

@@ -57,7 +57,7 @@ jobs:
         #   Print the 10 longest tests; see
         #   https://docs.pytest.org/en/latest/usage.html#profiling-test-execution-duration
         - >
-          py.test -n 3 --maxfail 3 --durations 10
+          pytest -n 3 --maxfail 3 --durations 10
           tests/unit tests/functional
           --ignore tests/functional/test_libraries.py
 
@@ -73,8 +73,14 @@ jobs:
       before_install: *before_install-osx
       script: &script-test-libraries
         - >
-          py.test -n 3 --maxfail 3 --durations 10
+          pytest -n 3 --maxfail 3 --durations 10
           tests/functional/test_libraries.py
+        # The ``run_tests`` script is invoked with the ``-c`` option to
+        # specify a ``pytest.ini``, rather than allowing pytest to find
+        # something unexpected in the filesystem (it searches from the root
+        # dir all the way to the top of the filesystem per
+        # https://docs.pytest.org/en/latest/customize.html).
+        - python -m PyInstaller.utils.run_tests -c PyInstaller/utils/pytest.ini
 
     - &test-pyinstaller
       stage: Test - PyInstaller
@@ -158,6 +164,10 @@ install:
   # Install dependencies for tests.
   - pip install --progress-bar=off -U -r tests/requirements-tools.txt -r tests/requirements-libraries.txt
 
+  # Install PyInstaller Hook Sample, to ensure that tests declared in
+  # entry-points are discovered.
+  - pip install https://github.com/pyinstaller/hooksample/archive/v4.0rc1.zip
+
   # Compile bootloader
   - cd bootloader
   - python waf distclean all
@@ -169,7 +179,6 @@ install:
   # Make sure the help options print.
   - python -m pyinstaller -h
 
-
 script:
   - true
 

PyInstaller/building/build_main.py

@@ -23,6 +23,8 @@
 import shutil
 import sys
 
+import pkg_resources
+
 
 # Relative imports to PyInstaller modules.
 from .. import HOMEPATH, DEFAULT_DISTPATH, DEFAULT_WORKPATH
@@ -40,7 +42,6 @@
 from ..depend.utils import create_py3_base_library, scan_code_for_ctypes
 from ..archive import pyz_crypto
 from ..utils.misc import get_path_to_toplevel_modules, get_unicode_modules, mtime
-from ..configure import get_importhooks_dir
 
 if is_win:
     from ..utils.win32 import winmanifest
@@ -207,6 +208,11 @@ def __init__(self, scripts, pathex=None, binaries=None, datas=None,
         # Include modules detected when parsing options, like 'codecs' and encodings.
         self.hiddenimports.extend(CONF['hiddenimports'])
 
+        # Add hook directories from PyInstaller entry points.
+        hookspath = hookspath or []
+        for entry_point in pkg_resources.iter_entry_points(
+                'pyinstaller40', 'hook-dirs'):
+            hookspath += list(entry_point.load()())
         self.hookspath = hookspath
 
         # Custom runtime hook files that should be included and started before

PyInstaller/configure.py

@@ -86,16 +86,6 @@ def _get_pyinst_cache_dir():
     return cache_dir
 
 
-#FIXME: Rename to get_official_hooks_dir().
-#FIXME: Remove the "hook_type" parameter after unifying hook types.
-def get_importhooks_dir(hook_type=None):
-    from . import PACKAGEPATH
-    if not hook_type:
-        return os.path.join(PACKAGEPATH, 'hooks')
-    else:
-        return os.path.join(PACKAGEPATH, 'hooks', hook_type)
-
-
 def get_config(upx_dir, **kw):
     config = {}
     test_UPX(config, upx_dir)

PyInstaller/depend/analysis.py

@@ -38,10 +38,13 @@
 import re
 import sys
 import traceback
+import ast
 
 from copy import deepcopy
+from collections import defaultdict
 
-from .. import HOMEPATH, configure
+from .. import compat
+from .. import HOMEPATH, PACKAGEPATH
 from .. import log as logging
 from ..log import INFO, DEBUG, TRACE
 from ..building.datastruct import TOC
@@ -53,7 +56,7 @@
 from ..lib.modulegraph.find_modules import get_implies
 from ..lib.modulegraph.modulegraph import ModuleGraph
 from ..utils.hooks import collect_submodules, is_package
-from ..utils.misc import load_py_data_struct
+
 
 logger = logging.getLogger(__name__)
 
@@ -113,9 +116,6 @@ def __init__(self, pyi_homepath, user_hook_dirs=(), excludes=(), **kwargs):
         self._excludes = excludes
         self._reset(user_hook_dirs)
         self._analyze_base_modules()
-        self._available_rthooks = load_py_data_struct(
-            os.path.join(self._homepath,
-                         'PyInstaller', 'loader', 'rthooks.dat'))
 
     def _reset(self, user_hook_dirs):
         """
@@ -124,7 +124,10 @@ def _reset(self, user_hook_dirs):
         """
         self._top_script_node = None
         self._additional_files_cache = AdditionalFilesCache()
-        self._user_hook_dirs = user_hook_dirs
+        # Prepend PyInstaller hook dir to user hook dirs.
+        self._user_hook_dirs = (
+            [os.path.join(PACKAGEPATH, 'hooks')] + list(user_hook_dirs)
+        )
         # Hook-specific lookup tables.
         # These need to reset when reusing cached PyiModuleGraph to avoid
         # hooks to refer to files or data from another test-case.
@@ -133,6 +136,68 @@ def _reset(self, user_hook_dirs):
         self._hooks_pre_safe_import_module = self._cache_hooks('pre_safe_import_module')
         self._hooks_pre_find_module_path = self._cache_hooks('pre_find_module_path')
 
+        # Search for run-time hooks in all hook directories.
+        self._available_rthooks = defaultdict(list)
+        for uhd in self._user_hook_dirs:
+            uhd_path = os.path.abspath(os.path.join(uhd, 'rthooks.dat'))
+            try:
+                with compat.open_file(uhd_path, compat.text_read_mode,
+                                      encoding='utf-8') as f:
+                    rthooks = ast.literal_eval(f.read())
+            except FileNotFoundError:
+                # Ignore if this hook path doesn't have run-time hooks.
+                continue
+            except Exception as e:
+                logger.error('Unable to read run-time hooks from %r: %s' %
+                             (uhd_path, e))
+                continue
+
+            self._merge_rthooks(rthooks, uhd, uhd_path)
+
+        # Convert back to a standard dict.
+        self._available_rthooks = dict(self._available_rthooks)
+
+    def _merge_rthooks(self, rthooks, uhd, uhd_path):
+        """The expected data structure for a run-time hook file is a Python
+        dictionary of type ``Dict[str, List[str]]`` where the dictionary
+        keys are module names the the sequence strings are Python file names.
+
+        Check then merge this data structure, updating the file names to be
+        absolute.
+        """
+        # Check that the root element is a dict.
+        assert isinstance(rthooks, dict), (
+            'The root element in %s must be a dict.' % uhd_path)
+        for module_name, python_file_name_list in rthooks.items():
+            # Ensure the key is a string.
+            assert isinstance(module_name, compat.string_types), (
+                '%s must be a dict whose keys are strings; %s '
+                'is not a string.' % (uhd_path, module_name))
+            # Ensure the value is a list.
+            assert isinstance(python_file_name_list, list), (
+                'The value of %s key %s must be a list.' %
+                (uhd_path, module_name))
+            if module_name in self._available_rthooks:
+                logger.warning("Several run-time hooks defined for module %r."
+                               "Please take care they do not conflict.",
+                               module_name)
+            # Merge this with existing run-time hooks.
+            for python_file_name in python_file_name_list:
+                # Ensure each item in the list is a string.
+                assert isinstance(python_file_name, compat.string_types), (
+                    '%s key %s, item %r must be a string.' %
+                    (uhd_path, module_name, python_file_name))
+                # Transform it into an absolute path.
+                abs_path = os.path.join(uhd, 'rthooks', python_file_name)
+                # Make sure this file exists.
+                assert os.path.exists(abs_path), (
+                    'In %s, key %s, the file %r expected to be located at '
+                    '%r does not exist.' %
+                    (uhd_path, module_name, python_file_name, abs_path))
+                # Merge it.
+                self._available_rthooks[module_name].append(abs_path)
+
+
     @staticmethod
     def _findCaller(*args, **kwargs):
         # Used to add an additional stack-frame above logger.findCaller.
@@ -187,12 +252,9 @@ def _cache_hooks(self, hook_type):
             subpackage of the `PyInstaller.hooks` package containing such hooks
             (e.g., `post_create_package` for post-create package hooks).
         """
-        # Absolute path of this type hook package's directory.
-        system_hook_dir = configure.get_importhooks_dir(hook_type)
-
-        # Cache of such hooks.
+        # Cache of this type of hooks.
         # logger.debug("Caching system %s hook dir %r" % (hook_type, system_hook_dir))
-        hook_dirs = [system_hook_dir]
+        hook_dirs = []
         for user_hook_dir in self._user_hook_dirs:
             # Absolute path of the user-defined subdirectory of this hook type.
             # If this directory exists, add it to the list to be cached.
@@ -338,7 +400,8 @@ def _safe_import_module(self, module_basename, module_name, parent_package):
             # For the absolute path of each such hook...
             for hook in self._hooks_pre_safe_import_module[module_name]:
                 # Dynamically import this hook as a fabricated module.
-                logger.info('Processing pre-safe import module hook   %s', module_name)
+                logger.info('Processing pre-safe import module hook %s '
+                            'from %r.', module_name, hook.hook_filename)
                 hook_module_name = 'PyInstaller_hooks_pre_safe_import_module_' + module_name.replace('.', '_')
                 hook_module = importlib_load_source(hook_module_name,
                                                     hook.hook_filename)
@@ -385,7 +448,8 @@ def _find_module_path(self, fullname, module_name, search_dirs):
             # For the absolute path of each such hook...
             for hook in self._hooks_pre_find_module_path[fullname]:
                 # Dynamically import this hook as a fabricated module.
-                logger.info('Processing pre-find module path hook   %s', fullname)
+                logger.info('Processing pre-find module path hook %s from %r.',
+                            fullname, hook.hook_filename)
                 hook_fullname = 'PyInstaller_hooks_pre_find_module_path_' + fullname.replace('.', '_')
                 hook_module = importlib_load_source(hook_fullname,
                                                     hook.hook_filename)
@@ -606,10 +670,9 @@ def analyze_runtime_hooks(self, custom_runhooks):
             # Look if there is any run-time hook for given module.
             if mod_name in self._available_rthooks:
                 # There could be several run-time hooks for a module.
-                for hook in self._available_rthooks[mod_name]:
-                    logger.info("Including run-time hook %r", hook)
-                    path = os.path.join(self._homepath, 'PyInstaller', 'loader', 'rthooks', hook)
-                    rthooks_nodes.append(self.run_script(path))
+                for abs_path in self._available_rthooks[mod_name]:
+                    logger.info("Including run-time hook %r", abs_path)
+                    rthooks_nodes.append(self.run_script(abs_path))
 
         return rthooks_nodes
 

PyInstaller/depend/imphook.py

@@ -379,8 +379,9 @@ def _load_hook_module(self):
 
         # Load and execute the hook script. Even if mechanisms from the import
         # machinery are used, this does not import the hook as the module.
+        head, tail = os.path.split(self.hook_filename)
         logger.info(
-            'Loading module hook "%s"...', os.path.basename(self.hook_filename))
+            'Loading module hook %r from %r...', tail, head)
         self._hook_module = importlib_load_source(
             self.hook_module_name, self.hook_filename)