diff --git a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py index df7397865f..f11cd84b54 100644 --- a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py +++ b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py @@ -9,14 +9,208 @@ # SPDX-License-Identifier: Apache-2.0 #----------------------------------------------------------------------------- +import os +import sys +import pathlib -import pkg_resources as res +import pkg_resources from pyimod03_importers import FrozenImporter +SYS_PREFIX = pathlib.PurePath(sys._MEIPASS) -# To make pkg_resources work with froze moduels we need to set the 'Provider' + +# To make pkg_resources work with frozen modules we need to set the 'Provider' # class for FrozenImporter. This class decides where to look for resources # and other stuff. 'pkg_resources.NullProvider' is dedicated to PEP302 # import hooks like FrozenImporter is. It uses method __loader__.get_data() in # methods pkg_resources.resource_string() and pkg_resources.resource_stream() -res.register_loader_type(FrozenImporter, res.NullProvider) +# +# We provide PyiFrozenProvider, which subclasses the NullProvider and +# implements _has(), _isdir(), and _listdir() methods, which are needed +# for pkg_resources.resource_exists(), resource_isdir(), and resource_listdir() +# to work. We cannot use the DefaultProvider, because it provides +# filesystem-only implementations (and overrides _get() with a filesystem-only +# one), whereas our provider needs to also support embedded resources. +# +# The PyiFrozenProvider allows querying/listing both PYZ-embedded and +# on-filesystem resources in a frozen package. The results are typically +# combined for both types of resources (e.g., when listing a directory +# or checking whether a resource exists). When the order of precedence +# matters, the PYZ-embedded resources take precedence over the on-filesystem +# ones, to keep the behavior consistent with the actual file content +# retrieval via _get() method (which in turn uses FrozenImporter's get_data() +# method). For example, when checking whether a resource is a directory +# via _isdir(), a PYZ-embedded file will take precedence over a potential +# on-filesystem directory. Also, in contrast to unfrozen packages, the frozen +# ones do not contain source .py files, which are therefore absent from +# content listings. + + +class _TocFilesystem: + """A prefix tree implementation for embedded filesystem reconstruction.""" + + def __init__(self, toc_files, toc_dirs=[]): + # Reconstruct the fileystem hierarchy by building a prefix tree from + # the given file and directory paths + self._tree = dict() + + # Data files + for path in toc_files: + path = pathlib.PurePath(path) + current = self._tree + for component in path.parts[:-1]: + current = current.setdefault(component, {}) + current[path.parts[-1]] = '' + + # Extra directories + for path in toc_dirs: + path = pathlib.PurePath(path) + current = self._tree + for component in path.parts: + current = current.setdefault(component, {}) + + def _get_tree_node(self, path): + path = pathlib.PurePath(path) + current = self._tree + for component in path.parts: + if component not in current: + return None + current = current[component] + return current + + def path_exists(self, path): + node = self._get_tree_node(path) + return node is not None # File or directory + + def path_isdir(self, path): + node = self._get_tree_node(path) + if node is None: + return False # Non-existant + if isinstance(node, str): + return False # File + return True + + def path_listdir(self, path): + node = self._get_tree_node(path) + if not isinstance(node, dict): + return [] # Non-existant or file + return list(node.keys()) + + +# Cache for reconstructed embedded trees +_toc_tree_cache = {} + + +class PyiFrozenProvider(pkg_resources.NullProvider): + """Custom pkg_resourvces provider for FrozenImporter.""" + + def __init__(self, module): + super().__init__(module) + + # Get top-level path; if "module" corresponds to a package, + # we need the path to the package itself. If "module" is a + # submodule in a package, we need the path to the parent + # package. + self._pkg_path = pathlib.PurePath(module.__file__).parent + + # Defer initialization of PYZ-embedded resources tree to the + # first access + self._embedded_tree = None + + def _init_embedded_tree(self, rel_pkg_path, pkg_name): + # Collect relevant entries from TOC. We are interested in either + # files that are located in the package/module's directory (data + # files) or in packages that are prefixed with package/module's + # name (to reconstruct subpackage directories) + data_files = [] + package_dirs = [] + for entry in self.loader.toc: + entry_path = pathlib.PurePath(entry) + if rel_pkg_path in entry_path.parents: + # Data file path + data_files.append(entry_path) + elif entry.startswith(pkg_name) and self.loader.is_package(entry): + # Package or subpackage; convert the name to directory path + package_dir = pathlib.PurePath(*entry.split('.')) + package_dirs.append(package_dir) + + # Reconstruct the filesystem + return _TocFilesystem(data_files, package_dirs) + + @property + def embedded_tree(self): + if self._embedded_tree is None: + # Construct a path relative to _MEIPASS directory for + # searching the TOC + rel_pkg_path = self._pkg_path.relative_to(SYS_PREFIX) + + # Reconstruct package name prefix (use package path to + # obtain correct prefix in case of a module) + pkg_name = '.'.join(rel_pkg_path.parts) + + # Initialize and cache the tree, if necessary + if pkg_name not in _toc_tree_cache: + _toc_tree_cache[pkg_name] = \ + self._init_embedded_tree(rel_pkg_path, pkg_name) + self._embedded_tree = _toc_tree_cache[pkg_name] + return self._embedded_tree + + def _normalize_path(self, path): + # Avoid using Path.resolve(), because it resolves symlinks. This + # is undesirable, because the pure path in self._pkg_path does + # not have symlinks resolved, so comparison between the two + # would be faulty. So use os.path.abspath() instead to normalize + # the path + return pathlib.Path(os.path.abspath(path)) + + def _is_relative_to_package(self, path): + return path == self._pkg_path or self._pkg_path in path.parents + + def _has(self, path): + # Prevent access outside the package + path = self._normalize_path(path) + if not self._is_relative_to_package(path): + return False + + # Check the filesystem first to avoid unnecessarily computing + # the relative path... + if path.exists(): + return True + rel_path = path.relative_to(SYS_PREFIX) + return self.embedded_tree.path_exists(rel_path) + + def _isdir(self, path): + # Prevent access outside the package + path = self._normalize_path(path) + if not self._is_relative_to_package(path): + return False + + # Embedded resources have precedence over filesystem... + rel_path = path.relative_to(SYS_PREFIX) + node = self.embedded_tree._get_tree_node(rel_path) + if node is None: + return path.is_dir() # No match found; try the filesystem + else: + # str = file, dict = directory + return not isinstance(node, str) + + def _listdir(self, path): + # Prevent access outside the package + path = self._normalize_path(path) + if not self._is_relative_to_package(path): + return [] + + # Relative path for searching embedded resources + rel_path = path.relative_to(SYS_PREFIX) + # List content from embedded filesystem... + content = self.embedded_tree.path_listdir(rel_path) + # ... as well as the actual one + if path.is_dir(): + # Use os.listdir() to avoid having to convert Path objects + # to strings... Also make sure to de-duplicate the results + path = str(path) # not is_py36 + content = list(set(content + os.listdir(path))) + return content + + +pkg_resources.register_loader_type(FrozenImporter, PyiFrozenProvider) diff --git a/news/5284.hooks.rst b/news/5284.hooks.rst new file mode 100644 index 0000000000..a4660ca7e4 --- /dev/null +++ b/news/5284.hooks.rst @@ -0,0 +1,4 @@ +Add support for package content listing via ``pkg_resources``. The +implementation enables querying/listing resources in a frozen package +(both PYZ-embedded and on-filesystem, in that order of precedence) via +``pkg_resources.resource_exists()``, ``resource_isdir()``, and ``resource_listdir()``. diff --git a/tests/functional/modules/pyi_pkg_resources_provider/hooks/hook-pyi_pkgres_testpkg.py b/tests/functional/modules/pyi_pkg_resources_provider/hooks/hook-pyi_pkgres_testpkg.py new file mode 100644 index 0000000000..1e80c4f6c0 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/hooks/hook-pyi_pkgres_testpkg.py @@ -0,0 +1,14 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2020, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- + +from PyInstaller.utils.hooks import collect_data_files + +datas = collect_data_files('pyi_pkgres_testpkg', excludes=['**/__pycache__', ]) diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/__init__.py new file mode 100644 index 0000000000..b533e90ed1 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/__init__.py @@ -0,0 +1,2 @@ +from . import a, b # noqa: F401 +from . import subpkg1, subpkg2, subpkg3 # noqa: F401 diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/a.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/a.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/a.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/b.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/b.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/b.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/__init__.py new file mode 100644 index 0000000000..123eeb89b3 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/__init__.py @@ -0,0 +1 @@ +from . import c, d # noqa: F401 diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/c.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/c.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/c.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/d.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/d.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/d.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry1.txt b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry1.txt new file mode 100644 index 0000000000..b012949762 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry1.txt @@ -0,0 +1 @@ +Data entry #1 in subpkg1/data. diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry2.md b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry2.md new file mode 100644 index 0000000000..7639b63999 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry2.md @@ -0,0 +1 @@ +Data entry #2 in `subpkg1/data`. diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry3.rst b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry3.rst new file mode 100644 index 0000000000..9647f92b82 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry3.rst @@ -0,0 +1 @@ +Data entry #3 in ``subpkg1/data``. diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/extra/extra_entry1.json b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/extra/extra_entry1.json new file mode 100644 index 0000000000..b1b9b798a9 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/extra/extra_entry1.json @@ -0,0 +1,3 @@ +{ + "_comment": "Extra data entry #1 in subpkg1/data/extra." +} diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/__init__.py new file mode 100644 index 0000000000..387ca1ee68 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/__init__.py @@ -0,0 +1,2 @@ +from . import subsubpkg21 # noqa: F401 +from . import mod # noqa: F401 diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/mod.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/mod.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/mod.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/__init__.py new file mode 100644 index 0000000000..6cfedf83d9 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/__init__.py @@ -0,0 +1 @@ +from . import mod # noqa: F401 diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/mod.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/mod.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/mod.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/__init__.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/__init__.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/_datafile.json b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/_datafile.json new file mode 100644 index 0000000000..a037c28a8e --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/_datafile.json @@ -0,0 +1,3 @@ +{ + "_comment": "Data file in supbkg3." +} diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/setup.py b/tests/functional/modules/pyi_pkg_resources_provider/package/setup.py new file mode 100644 index 0000000000..b230365e53 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/setup.py @@ -0,0 +1,33 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2020, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- +# +# This assists in creating an ``.egg`` package for use with the +# ``test_pkg_resources_provider``. To build the package, execute +# ``python setup.py bdist_egg``. + +import setuptools + +setuptools.setup( + name='pyi_pkgres_testpkg', + version='1.0.0', + setup_requires="setuptools >= 40.0.0", + author='PyInstaller development team', + packages=setuptools.find_packages(), + package_data={ + "pyi_pkgres_testpkg": [ + "subpkg1/data/*.txt", + "subpkg1/data/*.md", + "subpkg1/data/*.rst", + "subpkg1/data/extra/*.json", + "subpkg3/*.json", + ], + } +) diff --git a/tests/functional/scripts/pyi_pkg_resources_provider.py b/tests/functional/scripts/pyi_pkg_resources_provider.py new file mode 100644 index 0000000000..7a39e8f9bb --- /dev/null +++ b/tests/functional/scripts/pyi_pkg_resources_provider.py @@ -0,0 +1,409 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2020, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- +# +# A test script for validation of pkg_resources provider implementation. +# +# The test package has the following structure: +# +# pyi_pkgres_testpkg/ +# ├── a.py +# ├── b.py +# ├── __init__.py +# ├── subpkg1 +# │   ├── c.py +# │   ├── data +# │   │   ├── entry1.txt +# │   │   ├── entry2.md +# │   │   ├── entry3.rst +# │   │   └── extra +# │   │   └── extra_entry1.json +# │   ├── d.py +# │   └── __init__.py +# ├── subpkg2 +# │   ├── __init__.py +# │   ├── mod.py +# │   └── subsubpkg21 +# │   ├── __init__.py +# │   └── mod.py +# └── subpkg3 +# ├── _datafile.json +# └── __init__.py +# +# When run as unfrozen script, this script can be used to check the +# behavior of "native" providers that come with pkg_resources, e.g., +# DefaultProvider (for regular packages) and ZipProvider (for eggs). +# +# When run as a frozen application, this script validates the behavior +# of the frozen provider implemented by PyInstaller. Due to transitivity +# of test results, this script running without errors both as a native +# script and as a frozen application serves as proof of conformance for +# the PyInstaller's provider. +# +# Wherever the behavior between the native providers is inconsistent, +# we allow the same leeway for the PyInstaller's frozen provider. + +import sys +from pkg_resources import resource_exists, resource_isdir, resource_listdir +from pkg_resources import get_provider, DefaultProvider, ZipProvider + +pkgname = 'pyi_pkgres_testpkg' + +# Identify provider type +provider = get_provider(pkgname) +is_default = isinstance(provider, DefaultProvider) +is_zip = isinstance(provider, ZipProvider) +is_frozen = getattr(sys, 'frozen', False) + +assert is_default or is_zip or is_frozen, "Unsupported provider type!" + + +######################################################################## +# Validate behavior of resource_exists() # +######################################################################## +# Package's directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider returns True +ret = resource_exists(pkgname, '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Package's directory, with empty path +assert resource_exists(pkgname, '') + +# Subpackage's directory (relative to main package): +assert resource_exists(pkgname, 'subpkg1') +assert resource_exists(pkgname, 'subpkg2') +assert resource_exists(pkgname, 'subpkg2/subsubpkg21') +assert resource_exists(pkgname, 'subpkg3') + +# Subpackage's directory (relative to subpackage itself): +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider returns True +ret = resource_exists(pkgname + '.subpkg1', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Subpackage's directory (relative to subpackage itself), with empty path: +assert resource_exists(pkgname + '.subpkg1', '') + +# Data directory in subpackage +assert resource_exists(pkgname, 'subpkg1/data') +assert resource_exists(pkgname + '.subpkg1', 'data') + +# Subdirectory in data directory +assert resource_exists(pkgname, 'subpkg1/data/extra') +assert resource_exists(pkgname + '.subpkg1', 'data/extra') + +# File in data directory +assert resource_exists(pkgname, 'subpkg1/data/entry1.txt') + +# Deeply nested data file +assert resource_exists(pkgname, 'subpkg1/data/extra/extra_entry1.json') + +# A non-existant file/directory - should return False +assert not resource_exists(pkgname, 'subpkg1/non-existant') + +# A source script file in package +# > PyiFrozenProvider returns False because frozen application does +# not contain source files +ret = resource_exists(pkgname, '__init__.py') +assert (not is_frozen and ret) or \ + (is_frozen and not ret) + +# Parent of pacakge's top-level directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +ret = resource_exists(pkgname, '..') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and not ret) + +# Parent of subpackage's directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +ret = resource_exists(pkgname + '.subpkg1', '..') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and not ret) + +# Submodule in main package +ret = resource_exists(pkgname + '.a', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Submodule in main package, with empty path +assert resource_exists(pkgname + '.a', '') + +# Submodule in subpackage +ret = resource_exists(pkgname + '.subpkg1.c', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Submodule in subpackage, with empty path +assert resource_exists(pkgname + '.subpkg1.c', '') + + +######################################################################## +# Validate behavior of resource_isdir() # +######################################################################## +# Package's directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider returns True +ret = resource_isdir(pkgname, '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Package's directory, with empty path +assert resource_isdir(pkgname, '') + +# Subpackage's directory (relative to main pacakge): +# * both DefaultProvider and ZipProvider return True +assert resource_isdir(pkgname, 'subpkg1') +assert resource_isdir(pkgname, 'subpkg2') +assert resource_isdir(pkgname, 'subpkg2/subsubpkg21') +assert resource_isdir(pkgname, 'subpkg3') + +# Subpackage's directory (relative to subpackage itself): +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider returns True +ret = resource_isdir(pkgname + '.subpkg1', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Subpackage's directory (relative to subpackage itself), with empty path: +assert resource_isdir(pkgname + '.subpkg1', '') + +# Data directory in subpackage +assert resource_isdir(pkgname, 'subpkg1/data') +assert resource_isdir(pkgname + '.subpkg1', 'data') + +# Subdirectory in data directory +assert resource_isdir(pkgname, 'subpkg1/data/extra') +assert resource_isdir(pkgname + '.subpkg1', 'data/extra') + +# File in data directory - should return False +assert not resource_isdir(pkgname, 'subpkg1/data/entry1.txt') + +# Deeply nested data file - should return False +assert not resource_isdir(pkgname, 'subpkg1/data/extra/extra_entry1.json') + +# A non-existant file-directory - should return False +assert not resource_isdir(pkgname, 'subpkg1/non-existant') + +# A source script file in package - should return False +# NOTE: PyFrozenProvider returns False because the file does not +# exist. +assert not resource_isdir(pkgname, '__init__.py') + +# Parent of package's top-level directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +ret = resource_isdir(pkgname, '..') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and not ret) + +# Parent of subpacakge's directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +ret = resource_isdir(pkgname + '.subpkg1', '..') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and not ret) + +# Submodule in main package +ret = resource_isdir(pkgname + '.a', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Submodule in main package, with empty path +assert resource_isdir(pkgname + '.a', '') + +# Submodule in subpackage +ret = resource_isdir(pkgname + '.subpkg1.c', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Submodule in subpackage, with empty path +assert resource_isdir(pkgname + '.subpkg1.c', '') + + +######################################################################## +# Validate behavior of resource_listdir() # +######################################################################## +# A helper for resource_listdir() tests. +def _listdir_test(pkgname, path, expected): + # For frozen application, remove .py files from expected results + if is_frozen: + expected = [x for x in expected if not x.endswith('.py')] + # List the content + content = resource_listdir(pkgname, path) + # Ignore pycache + if '__pycache__' in content: + content.remove('__pycache__') + assert sorted(content) == sorted(expected) + + +# List package's top-level directory +# * DefaultProvider lists the directory +# * ZipProvider returns empty list +# > PyiFrozenProvider lists the directory, but does not provide source +# .py files +if is_zip: + expected = [] +else: + expected = ['__init__.py', 'a.py', 'b.py', 'subpkg1', 'subpkg2', 'subpkg3'] +_listdir_test(pkgname, '.', expected) + +# List package's top-level directory, with empty path +# > PyiFrozenProvider lists the directory, but does not provide source +# .py files +expected = ['__init__.py', 'a.py', 'b.py', 'subpkg1', 'subpkg2', 'subpkg3'] +_listdir_test(pkgname, '', expected) + +# List subpackage's directory (relative to main package) +# > PyiFrozenProvider lists the directory, but does not provide source +# .py files +expected = ['__init__.py', 'c.py', 'd.py', 'data'] +_listdir_test(pkgname, 'subpkg1', expected) + +# List data directory in subpackage (relative to main package) +expected = ['entry1.txt', 'entry2.md', 'entry3.rst', 'extra'] +_listdir_test(pkgname, 'subpkg1/data', expected) + +# List data directory in subpackage (relative to subpackage itself) +expected = ['entry1.txt', 'entry2.md', 'entry3.rst', 'extra'] +_listdir_test(pkgname + '.subpkg1', 'data', expected) + +# List data in subdirectory of data directory in subpackage +expected = ['extra_entry1.json'] +_listdir_test(pkgname + '.subpkg1', 'data/extra', expected) + +# Attempt to list a file (existing resource but not a directory). +# * DefaultProvider raises NotADirectoryError +# * ZipProvider returns empty list +# > PyiFrozenProvider returns empty list +try: + content = resource_listdir(pkgname + '.subpkg1', 'data/entry1.txt') +except NotADirectoryError: + assert is_default +except Exception: + raise +else: + assert (is_zip or is_frozen) and content == [] + +# Attempt to list an non-existant directory in main package. +# * DefaultProvider raises FileNotFoundError +# * ZipProvider returns empty list +# > PyiFrozenProvider returns empty list +try: + content = resource_listdir(pkgname, 'non-existant') +except FileNotFoundError: + assert is_default +except Exception: + raise +else: + assert (is_zip or is_frozen) and content == [] + +# Attempt to list an non-existant directory in subpackage +# * DefaultProvider raises FileNotFoundError +# * ZipProvider returns empty list +# > PyiFrozenProvider returns empty list +try: + content = resource_listdir(pkgname + '.subpkg1', 'data/non-existant') +except FileNotFoundError: + assert is_default +except Exception: + raise +else: + assert (is_zip or is_frozen) and content == [] + +# Attempt to list pacakge's parent directory +# * DefaultProvider actually lists the parent directory +# * ZipProvider returns empty list +# > PyiFrozenProvider disallows jumping to parent, and returns empty list +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +content = resource_listdir(pkgname, '..') +assert (is_default and pkgname in content) or \ + (is_zip and content == []) or \ + (is_frozen and content == []) + +# Attempt to list subpackage's parent directory +# * DefaultProvider actually lists the parent directory +# * ZipProvider returns empty list +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +if is_default: + expected = ['__init__.py', 'a.py', 'b.py', 'subpkg1', 'subpkg2', 'subpkg3'] +else: + expected = [] +_listdir_test(pkgname + '.subpkg1', '..', expected) + +# Attempt to list directory of subpackage that has no data files or +# directories (relative to main package) +expected = ['__init__.py', 'mod.py', 'subsubpkg21'] +_listdir_test(pkgname, 'subpkg2', expected) + +# Attempt to list directory of subpackage that has no data files or +# directories (relative to subpackage itself) +expected = ['__init__.py', 'mod.py', 'subsubpkg21'] +_listdir_test(pkgname + '.subpkg2', '', expected) # empty path! + +# Attempt to list directory of subsubpackage that has no data +# files/directories (relative to main package) +expected = ['__init__.py', 'mod.py'] +_listdir_test(pkgname, 'subpkg2/subsubpkg21', expected) + +# Attempt to list directory of subsubpackage that has no data +# files/directories (relative to parent subpackage) +expected = ['__init__.py', 'mod.py'] +_listdir_test(pkgname + '.subpkg2', 'subsubpkg21', expected) + +# Attempt to list directory of subsubpackage that has no data +# files/directories (relative to subsubpackage itself) +expected = ['__init__.py', 'mod.py'] +_listdir_test(pkgname + '.subpkg2.subsubpkg21', '', expected) # empty path! + +# Attempt to list submodule in main package - should give the same results +# as listing the package itself +assert sorted(resource_listdir(pkgname + '.a', '')) == \ + sorted(resource_listdir(pkgname, '')) # empty path! + +# Attempt to list submodule in subpackage - should give the same results +# as listing the subpackage itself +assert sorted(resource_listdir(pkgname + '.subpkg1.c', '')) == \ + sorted(resource_listdir(pkgname + '.subpkg1', '')) # empty path! diff --git a/tests/functional/test_pkg_resources_provider.py b/tests/functional/test_pkg_resources_provider.py new file mode 100644 index 0000000000..6d4568dc22 --- /dev/null +++ b/tests/functional/test_pkg_resources_provider.py @@ -0,0 +1,111 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2005-2020, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- +# +# These tests run a test script (scripts/pyi_pkg_resources_provider.py) +# in unfrozen and frozen form, in combination with a custom test package +# (modules/pyi_pkg_resources_provider/package) in either source or +# zipped egg form. +# +# Running the unfrozen test script allows us to verify the behavior of +# DefaultProvider and ZipProvider from pkg_resources and thereby also +# validate the test script itself. Running the frozen test validates +# the behavior of the PyiFrozenProvider. +# +# For details on the structure of the test and the contents of the test +# package, see the top comment in the test script itself. + +# Library imports +# --------------- +import os +import shutil + +# Third-party imports +# ------------------- +import pytest + +# Local imports +# ------------- +from PyInstaller.utils.tests import importorskip +from PyInstaller.compat import exec_python, exec_python_rc + +# Directory with testing modules used in some tests. +_MODULES_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'modules' +) + + +def __exec_python_script(script_filename, pathex): + # Prepare the environment - default to 'os.environ'... + env = os.environ.copy() + # ... and prepend PYTHONPATH with pathex + if 'PYTHONPATH' in env: + pathex = os.pathsep.join([pathex, env['PYTHONPATH']]) + env['PYTHONPATH'] = pathex + # Run the test script + return exec_python_rc(script_filename, env=env) + + +def __get_test_package_path(package_type, tmpdir, monkeypatch): + # Same test package, in two different formats: source package or + # zipped egg (built on-the-fly) + src_path = os.path.join(_MODULES_DIR, + 'pyi_pkg_resources_provider', + 'package') + # Source package + if package_type == 'pkg': + return src_path + # Copy files to a tmpdir for building the egg. + dest_path = tmpdir.join('src') + shutil.copytree(src_path, dest_path.strpath) + monkeypatch.chdir(dest_path) + # Create an egg from the test package. For debug, show the output of + # the egg build. + print(exec_python('setup.py', 'bdist_egg')) + # Obtain the name of the egg, which depends on the Python version. + dist_path = dest_path.join('dist') + files = os.listdir(dist_path.strpath) + assert len(files) == 1 + egg_name = files[0] + assert egg_name.endswith('.egg') + # Return the full path to the egg file + return dist_path.join(egg_name).strpath + + +@importorskip('pkg_resources') +@pytest.mark.parametrize('package_type', ['pkg', 'egg']) +def test_pkg_resources_provider_source(package_type, tmpdir, script_dir, + monkeypatch): + # Run the test script unfrozen - to validate it is working and to + # verify the behavior of pkg_resources.DefaultProvider / ZipProvider. + pathex = __get_test_package_path(package_type, tmpdir, monkeypatch) + test_script = 'pyi_pkg_resources_provider.py' + test_script = os.path.join(str(script_dir), # not is_py36: str() + test_script) + ret = __exec_python_script(test_script, pathex=pathex) + assert ret == 0, "Test script failed!" + + +@importorskip('pkg_resources') +@pytest.mark.parametrize('package_type', ['pkg', 'egg']) +def test_pkg_resources_provider_frozen(pyi_builder, package_type, tmpdir, + script_dir, monkeypatch): + # Run the test script as a frozen program + pathex = __get_test_package_path(package_type, tmpdir, monkeypatch) + test_script = 'pyi_pkg_resources_provider.py' + hooks_dir = os.path.join(_MODULES_DIR, + 'pyi_pkg_resources_provider', + 'hooks') + pyi_builder.test_script(test_script, pyi_args=[ + '--paths', pathex, + '--hidden-import', 'pyi_pkgres_testpkg', + '--additional-hooks-dir', hooks_dir] + )