From 8dc60044b8582ea979b3702b6cc95a8622bb7e62 Mon Sep 17 00:00:00 2001 From: Rok Mandeljc Date: Thu, 22 Oct 2020 20:03:30 +0200 Subject: [PATCH 1/6] tests: add test for pkg_resources package content listing The test script tests for behavior of resource_exists(), resource_isdir(), and resource_listdir() functions from pkg_resources package (which in turn call the methods with same name in the provider class). The idea is to run the test script twice, once as unfrozen python script and once as a frozen program. In both cases, the test package is once present as a plain package directory and once as a zipped egg (generated on-the-fly from the source directory). This way, we test behavior of the original provider (DefaultProvider or ZipProvider) and the provider used within the frozen application (which we will need to implement to replace the currently used NullProvider). --- .../hooks/hook-pyi_pkgres_testpkg.py | 14 + .../package/pyi_pkgres_testpkg/__init__.py | 2 + .../package/pyi_pkgres_testpkg/a.py | 1 + .../package/pyi_pkgres_testpkg/b.py | 1 + .../pyi_pkgres_testpkg/subpkg1/__init__.py | 1 + .../package/pyi_pkgres_testpkg/subpkg1/c.py | 1 + .../package/pyi_pkgres_testpkg/subpkg1/d.py | 1 + .../subpkg1/data/entry1.txt | 1 + .../pyi_pkgres_testpkg/subpkg1/data/entry2.md | 1 + .../subpkg1/data/entry3.rst | 1 + .../subpkg1/data/extra/extra_entry1.json | 3 + .../pyi_pkgres_testpkg/subpkg2/__init__.py | 2 + .../package/pyi_pkgres_testpkg/subpkg2/mod.py | 1 + .../subpkg2/subsubpkg21/__init__.py | 1 + .../subpkg2/subsubpkg21/mod.py | 1 + .../pyi_pkgres_testpkg/subpkg3/__init__.py | 1 + .../pyi_pkgres_testpkg/subpkg3/_datafile.json | 3 + .../package/setup.py | 33 ++ .../scripts/pyi_pkg_resources_provider.py | 409 ++++++++++++++++++ .../functional/test_pkg_resources_provider.py | 111 +++++ 20 files changed, 589 insertions(+) create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/hooks/hook-pyi_pkgres_testpkg.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/__init__.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/a.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/b.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/__init__.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/c.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/d.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry1.txt create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry2.md create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry3.rst create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/extra/extra_entry1.json create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/__init__.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/mod.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/__init__.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/mod.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/__init__.py create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/_datafile.json create mode 100644 tests/functional/modules/pyi_pkg_resources_provider/package/setup.py create mode 100644 tests/functional/scripts/pyi_pkg_resources_provider.py create mode 100644 tests/functional/test_pkg_resources_provider.py diff --git a/tests/functional/modules/pyi_pkg_resources_provider/hooks/hook-pyi_pkgres_testpkg.py b/tests/functional/modules/pyi_pkg_resources_provider/hooks/hook-pyi_pkgres_testpkg.py new file mode 100644 index 0000000000..1e80c4f6c0 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/hooks/hook-pyi_pkgres_testpkg.py @@ -0,0 +1,14 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2020, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- + +from PyInstaller.utils.hooks import collect_data_files + +datas = collect_data_files('pyi_pkgres_testpkg', excludes=['**/__pycache__', ]) diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/__init__.py new file mode 100644 index 0000000000..b533e90ed1 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/__init__.py @@ -0,0 +1,2 @@ +from . import a, b # noqa: F401 +from . import subpkg1, subpkg2, subpkg3 # noqa: F401 diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/a.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/a.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/a.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/b.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/b.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/b.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/__init__.py new file mode 100644 index 0000000000..123eeb89b3 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/__init__.py @@ -0,0 +1 @@ +from . import c, d # noqa: F401 diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/c.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/c.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/c.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/d.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/d.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/d.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry1.txt b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry1.txt new file mode 100644 index 0000000000..b012949762 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry1.txt @@ -0,0 +1 @@ +Data entry #1 in subpkg1/data. diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry2.md b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry2.md new file mode 100644 index 0000000000..7639b63999 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry2.md @@ -0,0 +1 @@ +Data entry #2 in `subpkg1/data`. diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry3.rst b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry3.rst new file mode 100644 index 0000000000..9647f92b82 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/entry3.rst @@ -0,0 +1 @@ +Data entry #3 in ``subpkg1/data``. diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/extra/extra_entry1.json b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/extra/extra_entry1.json new file mode 100644 index 0000000000..b1b9b798a9 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg1/data/extra/extra_entry1.json @@ -0,0 +1,3 @@ +{ + "_comment": "Extra data entry #1 in subpkg1/data/extra." +} diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/__init__.py new file mode 100644 index 0000000000..387ca1ee68 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/__init__.py @@ -0,0 +1,2 @@ +from . import subsubpkg21 # noqa: F401 +from . import mod # noqa: F401 diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/mod.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/mod.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/mod.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/__init__.py new file mode 100644 index 0000000000..6cfedf83d9 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/__init__.py @@ -0,0 +1 @@ +from . import mod # noqa: F401 diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/mod.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/mod.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg2/subsubpkg21/mod.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/__init__.py b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/__init__.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/__init__.py @@ -0,0 +1 @@ +# diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/_datafile.json b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/_datafile.json new file mode 100644 index 0000000000..a037c28a8e --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/pyi_pkgres_testpkg/subpkg3/_datafile.json @@ -0,0 +1,3 @@ +{ + "_comment": "Data file in supbkg3." +} diff --git a/tests/functional/modules/pyi_pkg_resources_provider/package/setup.py b/tests/functional/modules/pyi_pkg_resources_provider/package/setup.py new file mode 100644 index 0000000000..b230365e53 --- /dev/null +++ b/tests/functional/modules/pyi_pkg_resources_provider/package/setup.py @@ -0,0 +1,33 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2020, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- +# +# This assists in creating an ``.egg`` package for use with the +# ``test_pkg_resources_provider``. To build the package, execute +# ``python setup.py bdist_egg``. + +import setuptools + +setuptools.setup( + name='pyi_pkgres_testpkg', + version='1.0.0', + setup_requires="setuptools >= 40.0.0", + author='PyInstaller development team', + packages=setuptools.find_packages(), + package_data={ + "pyi_pkgres_testpkg": [ + "subpkg1/data/*.txt", + "subpkg1/data/*.md", + "subpkg1/data/*.rst", + "subpkg1/data/extra/*.json", + "subpkg3/*.json", + ], + } +) diff --git a/tests/functional/scripts/pyi_pkg_resources_provider.py b/tests/functional/scripts/pyi_pkg_resources_provider.py new file mode 100644 index 0000000000..7a39e8f9bb --- /dev/null +++ b/tests/functional/scripts/pyi_pkg_resources_provider.py @@ -0,0 +1,409 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2020, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- +# +# A test script for validation of pkg_resources provider implementation. +# +# The test package has the following structure: +# +# pyi_pkgres_testpkg/ +# ├── a.py +# ├── b.py +# ├── __init__.py +# ├── subpkg1 +# │   ├── c.py +# │   ├── data +# │   │   ├── entry1.txt +# │   │   ├── entry2.md +# │   │   ├── entry3.rst +# │   │   └── extra +# │   │   └── extra_entry1.json +# │   ├── d.py +# │   └── __init__.py +# ├── subpkg2 +# │   ├── __init__.py +# │   ├── mod.py +# │   └── subsubpkg21 +# │   ├── __init__.py +# │   └── mod.py +# └── subpkg3 +# ├── _datafile.json +# └── __init__.py +# +# When run as unfrozen script, this script can be used to check the +# behavior of "native" providers that come with pkg_resources, e.g., +# DefaultProvider (for regular packages) and ZipProvider (for eggs). +# +# When run as a frozen application, this script validates the behavior +# of the frozen provider implemented by PyInstaller. Due to transitivity +# of test results, this script running without errors both as a native +# script and as a frozen application serves as proof of conformance for +# the PyInstaller's provider. +# +# Wherever the behavior between the native providers is inconsistent, +# we allow the same leeway for the PyInstaller's frozen provider. + +import sys +from pkg_resources import resource_exists, resource_isdir, resource_listdir +from pkg_resources import get_provider, DefaultProvider, ZipProvider + +pkgname = 'pyi_pkgres_testpkg' + +# Identify provider type +provider = get_provider(pkgname) +is_default = isinstance(provider, DefaultProvider) +is_zip = isinstance(provider, ZipProvider) +is_frozen = getattr(sys, 'frozen', False) + +assert is_default or is_zip or is_frozen, "Unsupported provider type!" + + +######################################################################## +# Validate behavior of resource_exists() # +######################################################################## +# Package's directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider returns True +ret = resource_exists(pkgname, '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Package's directory, with empty path +assert resource_exists(pkgname, '') + +# Subpackage's directory (relative to main package): +assert resource_exists(pkgname, 'subpkg1') +assert resource_exists(pkgname, 'subpkg2') +assert resource_exists(pkgname, 'subpkg2/subsubpkg21') +assert resource_exists(pkgname, 'subpkg3') + +# Subpackage's directory (relative to subpackage itself): +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider returns True +ret = resource_exists(pkgname + '.subpkg1', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Subpackage's directory (relative to subpackage itself), with empty path: +assert resource_exists(pkgname + '.subpkg1', '') + +# Data directory in subpackage +assert resource_exists(pkgname, 'subpkg1/data') +assert resource_exists(pkgname + '.subpkg1', 'data') + +# Subdirectory in data directory +assert resource_exists(pkgname, 'subpkg1/data/extra') +assert resource_exists(pkgname + '.subpkg1', 'data/extra') + +# File in data directory +assert resource_exists(pkgname, 'subpkg1/data/entry1.txt') + +# Deeply nested data file +assert resource_exists(pkgname, 'subpkg1/data/extra/extra_entry1.json') + +# A non-existant file/directory - should return False +assert not resource_exists(pkgname, 'subpkg1/non-existant') + +# A source script file in package +# > PyiFrozenProvider returns False because frozen application does +# not contain source files +ret = resource_exists(pkgname, '__init__.py') +assert (not is_frozen and ret) or \ + (is_frozen and not ret) + +# Parent of pacakge's top-level directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +ret = resource_exists(pkgname, '..') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and not ret) + +# Parent of subpackage's directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +ret = resource_exists(pkgname + '.subpkg1', '..') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and not ret) + +# Submodule in main package +ret = resource_exists(pkgname + '.a', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Submodule in main package, with empty path +assert resource_exists(pkgname + '.a', '') + +# Submodule in subpackage +ret = resource_exists(pkgname + '.subpkg1.c', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Submodule in subpackage, with empty path +assert resource_exists(pkgname + '.subpkg1.c', '') + + +######################################################################## +# Validate behavior of resource_isdir() # +######################################################################## +# Package's directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider returns True +ret = resource_isdir(pkgname, '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Package's directory, with empty path +assert resource_isdir(pkgname, '') + +# Subpackage's directory (relative to main pacakge): +# * both DefaultProvider and ZipProvider return True +assert resource_isdir(pkgname, 'subpkg1') +assert resource_isdir(pkgname, 'subpkg2') +assert resource_isdir(pkgname, 'subpkg2/subsubpkg21') +assert resource_isdir(pkgname, 'subpkg3') + +# Subpackage's directory (relative to subpackage itself): +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider returns True +ret = resource_isdir(pkgname + '.subpkg1', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Subpackage's directory (relative to subpackage itself), with empty path: +assert resource_isdir(pkgname + '.subpkg1', '') + +# Data directory in subpackage +assert resource_isdir(pkgname, 'subpkg1/data') +assert resource_isdir(pkgname + '.subpkg1', 'data') + +# Subdirectory in data directory +assert resource_isdir(pkgname, 'subpkg1/data/extra') +assert resource_isdir(pkgname + '.subpkg1', 'data/extra') + +# File in data directory - should return False +assert not resource_isdir(pkgname, 'subpkg1/data/entry1.txt') + +# Deeply nested data file - should return False +assert not resource_isdir(pkgname, 'subpkg1/data/extra/extra_entry1.json') + +# A non-existant file-directory - should return False +assert not resource_isdir(pkgname, 'subpkg1/non-existant') + +# A source script file in package - should return False +# NOTE: PyFrozenProvider returns False because the file does not +# exist. +assert not resource_isdir(pkgname, '__init__.py') + +# Parent of package's top-level directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +ret = resource_isdir(pkgname, '..') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and not ret) + +# Parent of subpacakge's directory +# * DefaultProvider returns True +# * ZipProvider returns False +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +ret = resource_isdir(pkgname + '.subpkg1', '..') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and not ret) + +# Submodule in main package +ret = resource_isdir(pkgname + '.a', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Submodule in main package, with empty path +assert resource_isdir(pkgname + '.a', '') + +# Submodule in subpackage +ret = resource_isdir(pkgname + '.subpkg1.c', '.') +assert (is_default and ret) or \ + (is_zip and not ret) or \ + (is_frozen and ret) + +# Submodule in subpackage, with empty path +assert resource_isdir(pkgname + '.subpkg1.c', '') + + +######################################################################## +# Validate behavior of resource_listdir() # +######################################################################## +# A helper for resource_listdir() tests. +def _listdir_test(pkgname, path, expected): + # For frozen application, remove .py files from expected results + if is_frozen: + expected = [x for x in expected if not x.endswith('.py')] + # List the content + content = resource_listdir(pkgname, path) + # Ignore pycache + if '__pycache__' in content: + content.remove('__pycache__') + assert sorted(content) == sorted(expected) + + +# List package's top-level directory +# * DefaultProvider lists the directory +# * ZipProvider returns empty list +# > PyiFrozenProvider lists the directory, but does not provide source +# .py files +if is_zip: + expected = [] +else: + expected = ['__init__.py', 'a.py', 'b.py', 'subpkg1', 'subpkg2', 'subpkg3'] +_listdir_test(pkgname, '.', expected) + +# List package's top-level directory, with empty path +# > PyiFrozenProvider lists the directory, but does not provide source +# .py files +expected = ['__init__.py', 'a.py', 'b.py', 'subpkg1', 'subpkg2', 'subpkg3'] +_listdir_test(pkgname, '', expected) + +# List subpackage's directory (relative to main package) +# > PyiFrozenProvider lists the directory, but does not provide source +# .py files +expected = ['__init__.py', 'c.py', 'd.py', 'data'] +_listdir_test(pkgname, 'subpkg1', expected) + +# List data directory in subpackage (relative to main package) +expected = ['entry1.txt', 'entry2.md', 'entry3.rst', 'extra'] +_listdir_test(pkgname, 'subpkg1/data', expected) + +# List data directory in subpackage (relative to subpackage itself) +expected = ['entry1.txt', 'entry2.md', 'entry3.rst', 'extra'] +_listdir_test(pkgname + '.subpkg1', 'data', expected) + +# List data in subdirectory of data directory in subpackage +expected = ['extra_entry1.json'] +_listdir_test(pkgname + '.subpkg1', 'data/extra', expected) + +# Attempt to list a file (existing resource but not a directory). +# * DefaultProvider raises NotADirectoryError +# * ZipProvider returns empty list +# > PyiFrozenProvider returns empty list +try: + content = resource_listdir(pkgname + '.subpkg1', 'data/entry1.txt') +except NotADirectoryError: + assert is_default +except Exception: + raise +else: + assert (is_zip or is_frozen) and content == [] + +# Attempt to list an non-existant directory in main package. +# * DefaultProvider raises FileNotFoundError +# * ZipProvider returns empty list +# > PyiFrozenProvider returns empty list +try: + content = resource_listdir(pkgname, 'non-existant') +except FileNotFoundError: + assert is_default +except Exception: + raise +else: + assert (is_zip or is_frozen) and content == [] + +# Attempt to list an non-existant directory in subpackage +# * DefaultProvider raises FileNotFoundError +# * ZipProvider returns empty list +# > PyiFrozenProvider returns empty list +try: + content = resource_listdir(pkgname + '.subpkg1', 'data/non-existant') +except FileNotFoundError: + assert is_default +except Exception: + raise +else: + assert (is_zip or is_frozen) and content == [] + +# Attempt to list pacakge's parent directory +# * DefaultProvider actually lists the parent directory +# * ZipProvider returns empty list +# > PyiFrozenProvider disallows jumping to parent, and returns empty list +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +content = resource_listdir(pkgname, '..') +assert (is_default and pkgname in content) or \ + (is_zip and content == []) or \ + (is_frozen and content == []) + +# Attempt to list subpackage's parent directory +# * DefaultProvider actually lists the parent directory +# * ZipProvider returns empty list +# > PyiFrozenProvider disallows jumping to parent, and returns False +# NOTE: using .. in path is deprecated (since setuptools 40.8.0) and +# will raise exception in a future release +if is_default: + expected = ['__init__.py', 'a.py', 'b.py', 'subpkg1', 'subpkg2', 'subpkg3'] +else: + expected = [] +_listdir_test(pkgname + '.subpkg1', '..', expected) + +# Attempt to list directory of subpackage that has no data files or +# directories (relative to main package) +expected = ['__init__.py', 'mod.py', 'subsubpkg21'] +_listdir_test(pkgname, 'subpkg2', expected) + +# Attempt to list directory of subpackage that has no data files or +# directories (relative to subpackage itself) +expected = ['__init__.py', 'mod.py', 'subsubpkg21'] +_listdir_test(pkgname + '.subpkg2', '', expected) # empty path! + +# Attempt to list directory of subsubpackage that has no data +# files/directories (relative to main package) +expected = ['__init__.py', 'mod.py'] +_listdir_test(pkgname, 'subpkg2/subsubpkg21', expected) + +# Attempt to list directory of subsubpackage that has no data +# files/directories (relative to parent subpackage) +expected = ['__init__.py', 'mod.py'] +_listdir_test(pkgname + '.subpkg2', 'subsubpkg21', expected) + +# Attempt to list directory of subsubpackage that has no data +# files/directories (relative to subsubpackage itself) +expected = ['__init__.py', 'mod.py'] +_listdir_test(pkgname + '.subpkg2.subsubpkg21', '', expected) # empty path! + +# Attempt to list submodule in main package - should give the same results +# as listing the package itself +assert sorted(resource_listdir(pkgname + '.a', '')) == \ + sorted(resource_listdir(pkgname, '')) # empty path! + +# Attempt to list submodule in subpackage - should give the same results +# as listing the subpackage itself +assert sorted(resource_listdir(pkgname + '.subpkg1.c', '')) == \ + sorted(resource_listdir(pkgname + '.subpkg1', '')) # empty path! diff --git a/tests/functional/test_pkg_resources_provider.py b/tests/functional/test_pkg_resources_provider.py new file mode 100644 index 0000000000..6d4568dc22 --- /dev/null +++ b/tests/functional/test_pkg_resources_provider.py @@ -0,0 +1,111 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2005-2020, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- +# +# These tests run a test script (scripts/pyi_pkg_resources_provider.py) +# in unfrozen and frozen form, in combination with a custom test package +# (modules/pyi_pkg_resources_provider/package) in either source or +# zipped egg form. +# +# Running the unfrozen test script allows us to verify the behavior of +# DefaultProvider and ZipProvider from pkg_resources and thereby also +# validate the test script itself. Running the frozen test validates +# the behavior of the PyiFrozenProvider. +# +# For details on the structure of the test and the contents of the test +# package, see the top comment in the test script itself. + +# Library imports +# --------------- +import os +import shutil + +# Third-party imports +# ------------------- +import pytest + +# Local imports +# ------------- +from PyInstaller.utils.tests import importorskip +from PyInstaller.compat import exec_python, exec_python_rc + +# Directory with testing modules used in some tests. +_MODULES_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'modules' +) + + +def __exec_python_script(script_filename, pathex): + # Prepare the environment - default to 'os.environ'... + env = os.environ.copy() + # ... and prepend PYTHONPATH with pathex + if 'PYTHONPATH' in env: + pathex = os.pathsep.join([pathex, env['PYTHONPATH']]) + env['PYTHONPATH'] = pathex + # Run the test script + return exec_python_rc(script_filename, env=env) + + +def __get_test_package_path(package_type, tmpdir, monkeypatch): + # Same test package, in two different formats: source package or + # zipped egg (built on-the-fly) + src_path = os.path.join(_MODULES_DIR, + 'pyi_pkg_resources_provider', + 'package') + # Source package + if package_type == 'pkg': + return src_path + # Copy files to a tmpdir for building the egg. + dest_path = tmpdir.join('src') + shutil.copytree(src_path, dest_path.strpath) + monkeypatch.chdir(dest_path) + # Create an egg from the test package. For debug, show the output of + # the egg build. + print(exec_python('setup.py', 'bdist_egg')) + # Obtain the name of the egg, which depends on the Python version. + dist_path = dest_path.join('dist') + files = os.listdir(dist_path.strpath) + assert len(files) == 1 + egg_name = files[0] + assert egg_name.endswith('.egg') + # Return the full path to the egg file + return dist_path.join(egg_name).strpath + + +@importorskip('pkg_resources') +@pytest.mark.parametrize('package_type', ['pkg', 'egg']) +def test_pkg_resources_provider_source(package_type, tmpdir, script_dir, + monkeypatch): + # Run the test script unfrozen - to validate it is working and to + # verify the behavior of pkg_resources.DefaultProvider / ZipProvider. + pathex = __get_test_package_path(package_type, tmpdir, monkeypatch) + test_script = 'pyi_pkg_resources_provider.py' + test_script = os.path.join(str(script_dir), # not is_py36: str() + test_script) + ret = __exec_python_script(test_script, pathex=pathex) + assert ret == 0, "Test script failed!" + + +@importorskip('pkg_resources') +@pytest.mark.parametrize('package_type', ['pkg', 'egg']) +def test_pkg_resources_provider_frozen(pyi_builder, package_type, tmpdir, + script_dir, monkeypatch): + # Run the test script as a frozen program + pathex = __get_test_package_path(package_type, tmpdir, monkeypatch) + test_script = 'pyi_pkg_resources_provider.py' + hooks_dir = os.path.join(_MODULES_DIR, + 'pyi_pkg_resources_provider', + 'hooks') + pyi_builder.test_script(test_script, pyi_args=[ + '--paths', pathex, + '--hidden-import', 'pyi_pkgres_testpkg', + '--additional-hooks-dir', hooks_dir] + ) From 97126fbc15faece018bc0639f1ebe71b388b29b9 Mon Sep 17 00:00:00 2001 From: Rok Mandeljc Date: Fri, 23 Oct 2020 11:29:01 +0200 Subject: [PATCH 2/6] hooks: pkg_resources: implement custom pkg_resources provider Implement PyiFrozenProvider that subclasses NullProvider from pkg_resources and provides _has(), _isdir(), and _listdir() methods. The implementation of these methods supports both PYZ-embedded and on-filesystem resources, in that order of precedence. --- PyInstaller/hooks/rthooks/pyi_rth_pkgres.py | 164 +++++++++++++++++++- news/5284.hooks.rst | 4 + 2 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 news/5284.hooks.rst diff --git a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py index df7397865f..8f829fb8e1 100644 --- a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py +++ b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py @@ -9,14 +9,172 @@ # SPDX-License-Identifier: Apache-2.0 #----------------------------------------------------------------------------- +import os +import sys +import pathlib -import pkg_resources as res +import pkg_resources from pyimod03_importers import FrozenImporter +SYS_PREFIX = pathlib.PurePath(sys._MEIPASS) -# To make pkg_resources work with froze moduels we need to set the 'Provider' + +# To make pkg_resources work with frozen modules we need to set the 'Provider' # class for FrozenImporter. This class decides where to look for resources # and other stuff. 'pkg_resources.NullProvider' is dedicated to PEP302 # import hooks like FrozenImporter is. It uses method __loader__.get_data() in # methods pkg_resources.resource_string() and pkg_resources.resource_stream() -res.register_loader_type(FrozenImporter, res.NullProvider) +# +# We subclass the NullProvider and implement _has(), _isdir(), and _listdir(), +# which are needed for pkg_resources.resource_exists(), resource_isdir(), +# and resource_listdir() to work. We cannot use the DefaultProvider, +# because it provides filesystem-only implementations (and overrides _get() +# with a filesystem-only one), whereas our provider needs to also support +# embedded resources. + + +class _TocFilesystem: + def __init__(self, toc_files, toc_dirs=[]): + # Reconstruct the fileystem hierarchy by building a prefix tree from + # the given file and directory paths + self._tree = dict() + + # Data files + for path in toc_files: + path = pathlib.PurePath(path) + current = self._tree + for component in path.parts[:-1]: + current = current.setdefault(component, {}) + current[path.parts[-1]] = '' + + # Extra directories + for path in toc_dirs: + path = pathlib.PurePath(path) + current = self._tree + for component in path.parts: + current = current.setdefault(component, {}) + + def _get_tree_node(self, path): + path = pathlib.PurePath(path) + current = self._tree + for component in path.parts: + if component not in current: + return None + current = current[component] + return current + + def path_exists(self, path): + node = self._get_tree_node(path) + return node is not None # File or directory + + def path_isdir(self, path): + node = self._get_tree_node(path) + if node is None: + return False # Non-existant + if isinstance(node, str): + return False # File + return True + + def path_listdir(self, path): + node = self._get_tree_node(path) + if not isinstance(node, dict): + return [] # Non-existant or file + return list(node.keys()) + + +class PyiFrozenProvider(pkg_resources.NullProvider): + def __init__(self, module): + super().__init__(module) + + # Get top-level path; if "module" corresponds to a package, + # we need the path to the package itself. If "module" is a + # submodule in a package, we need the path to the parent + # package. + self._pkg_path = pathlib.PurePath(module.__file__).parent + + # Construct a path relative to _MEIPASS directory for searching + # the TOC + rel_pkg_path = self._pkg_path.relative_to(SYS_PREFIX) + + # Reconstruct package name prefix (use package path to obtain + # correct prefix in case of a module) + pkg_name = '.'.join(rel_pkg_path.parts) + + # Collect relevant entries from TOC. We are interested in either + # files that are located in the package/module's directory (data + # files) or in packages that are prefixed with package/module's + # name (to reconstruct subpackage directories) + data_files = [] + package_dirs = [] + for entry in self.loader.toc: + entry_path = pathlib.PurePath(entry) + if rel_pkg_path in entry_path.parents: + # Data file path + data_files.append(entry_path) + elif entry.startswith(pkg_name) and self.loader.is_package(entry): + # Package or subpackage; convert the name to directory path + package_dir = pathlib.PurePath(*entry.split('.')) + package_dirs.append(package_dir) + + # Reconstruct the filesystem + self.embedded_tree = _TocFilesystem(data_files, package_dirs) + + def _normalize_path(self, path): + # Avoid using Path.resolve(), because it resolves symlinks. This + # is undesirable, because the pure path in self._pkg_path does + # not have symlinks resolved, so comparison between the two + # would be faulty. So use os.path.abspath() instead to normalize + # the path + return pathlib.Path(os.path.abspath(path)) + + def _is_relative_to_package(self, path): + return path == self._pkg_path or self._pkg_path in path.parents + + def _has(self, path): + # Prevent access outside the package + path = self._normalize_path(path) + if not self._is_relative_to_package(path): + return False + + # Check the filesystem first to avoid unnecessarily computing + # the relative path... + if path.exists(): + return True + rel_path = path.relative_to(SYS_PREFIX) + return self.embedded_tree.path_exists(rel_path) + + def _isdir(self, path): + # Prevent access outside the package + path = self._normalize_path(path) + if not self._is_relative_to_package(path): + return False + + # Embedded resources have precedence over filesystem... + rel_path = path.relative_to(SYS_PREFIX) + node = self.embedded_tree._get_tree_node(rel_path) + if node is None: + return path.is_dir() # No match found; try the filesystem + else: + # str = file, dict = directory + return not isinstance(node, str) + + def _listdir(self, path): + # Prevent access outside the package + path = self._normalize_path(path) + if not self._is_relative_to_package(path): + return [] + + # Relative path for searching embedded resources + rel_path = path.relative_to(SYS_PREFIX) + # List content from embedded filesystem... + content = self.embedded_tree.path_listdir(rel_path) + # ... as well as the actual one + if path.is_dir(): + # Use os.listdir() to avoid having to convert Path objects + # to strings... + path = str(path) # not is_py36 + content += os.listdir(path) + return content + + +pkg_resources.register_loader_type(FrozenImporter, PyiFrozenProvider) diff --git a/news/5284.hooks.rst b/news/5284.hooks.rst new file mode 100644 index 0000000000..a4660ca7e4 --- /dev/null +++ b/news/5284.hooks.rst @@ -0,0 +1,4 @@ +Add support for package content listing via ``pkg_resources``. The +implementation enables querying/listing resources in a frozen package +(both PYZ-embedded and on-filesystem, in that order of precedence) via +``pkg_resources.resource_exists()``, ``resource_isdir()``, and ``resource_listdir()``. From a42efe6d3078e937f7725659d1fa403fa023a0ae Mon Sep 17 00:00:00 2001 From: Rok Mandeljc Date: Fri, 6 Nov 2020 20:31:36 +0100 Subject: [PATCH 3/6] hooks: pkg_resources: prevent duplication of directories in _listdir() Because a directory may exist as both an embedded and on-filesystem resource, we need to de-duplicate the results when listing the filesystem in addition to embedded tree. --- PyInstaller/hooks/rthooks/pyi_rth_pkgres.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py index 8f829fb8e1..0912fa4529 100644 --- a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py +++ b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py @@ -171,9 +171,9 @@ def _listdir(self, path): # ... as well as the actual one if path.is_dir(): # Use os.listdir() to avoid having to convert Path objects - # to strings... + # to strings... Also make sure to de-duplicate the results path = str(path) # not is_py36 - content += os.listdir(path) + content = list(set(content + os.listdir(path))) return content From a2687b961a7248ea974f4e8e4702c6747885359b Mon Sep 17 00:00:00 2001 From: Rok Mandeljc Date: Fri, 6 Nov 2020 20:53:50 +0100 Subject: [PATCH 4/6] hooks: pkg_resources: defer embedded resource tree initialization to first use --- PyInstaller/hooks/rthooks/pyi_rth_pkgres.py | 60 ++++++++++++--------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py index 0912fa4529..36c1f302e6 100644 --- a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py +++ b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py @@ -92,32 +92,40 @@ def __init__(self, module): # package. self._pkg_path = pathlib.PurePath(module.__file__).parent - # Construct a path relative to _MEIPASS directory for searching - # the TOC - rel_pkg_path = self._pkg_path.relative_to(SYS_PREFIX) - - # Reconstruct package name prefix (use package path to obtain - # correct prefix in case of a module) - pkg_name = '.'.join(rel_pkg_path.parts) - - # Collect relevant entries from TOC. We are interested in either - # files that are located in the package/module's directory (data - # files) or in packages that are prefixed with package/module's - # name (to reconstruct subpackage directories) - data_files = [] - package_dirs = [] - for entry in self.loader.toc: - entry_path = pathlib.PurePath(entry) - if rel_pkg_path in entry_path.parents: - # Data file path - data_files.append(entry_path) - elif entry.startswith(pkg_name) and self.loader.is_package(entry): - # Package or subpackage; convert the name to directory path - package_dir = pathlib.PurePath(*entry.split('.')) - package_dirs.append(package_dir) - - # Reconstruct the filesystem - self.embedded_tree = _TocFilesystem(data_files, package_dirs) + # Defer initialization of PYZ-embedded resources tree to the + # first access + self._embedded_tree = None + + @property + def embedded_tree(self): + if self._embedded_tree is None: + # Construct a path relative to _MEIPASS directory for searching + # the TOC + rel_pkg_path = self._pkg_path.relative_to(SYS_PREFIX) + + # Reconstruct package name prefix (use package path to obtain + # correct prefix in case of a module) + pkg_name = '.'.join(rel_pkg_path.parts) + + # Collect relevant entries from TOC. We are interested in either + # files that are located in the package/module's directory (data + # files) or in packages that are prefixed with package/module's + # name (to reconstruct subpackage directories) + data_files = [] + package_dirs = [] + for entry in self.loader.toc: + entry_path = pathlib.PurePath(entry) + if rel_pkg_path in entry_path.parents: + # Data file path + data_files.append(entry_path) + elif entry.startswith(pkg_name) and self.loader.is_package(entry): + # Package or subpackage; convert the name to directory path + package_dir = pathlib.PurePath(*entry.split('.')) + package_dirs.append(package_dir) + + # Reconstruct the filesystem + self._embedded_tree = _TocFilesystem(data_files, package_dirs) + return self._embedded_tree def _normalize_path(self, path): # Avoid using Path.resolve(), because it resolves symlinks. This From 26178e075d229e61b17b4ce4bb370253e28b83e7 Mon Sep 17 00:00:00 2001 From: Rok Mandeljc Date: Fri, 6 Nov 2020 21:17:22 +0100 Subject: [PATCH 5/6] hooks: pkg_resources: implement caching for embedded-resource trees --- PyInstaller/hooks/rthooks/pyi_rth_pkgres.py | 55 ++++++++++++--------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py index 36c1f302e6..f1c60a966e 100644 --- a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py +++ b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py @@ -82,6 +82,10 @@ def path_listdir(self, path): return list(node.keys()) +# Cache for reconstructed embedded trees +_toc_tree_cache = {} + + class PyiFrozenProvider(pkg_resources.NullProvider): def __init__(self, module): super().__init__(module) @@ -96,35 +100,42 @@ def __init__(self, module): # first access self._embedded_tree = None + def _init_embedded_tree(self, rel_pkg_path, pkg_name): + # Collect relevant entries from TOC. We are interested in either + # files that are located in the package/module's directory (data + # files) or in packages that are prefixed with package/module's + # name (to reconstruct subpackage directories) + data_files = [] + package_dirs = [] + for entry in self.loader.toc: + entry_path = pathlib.PurePath(entry) + if rel_pkg_path in entry_path.parents: + # Data file path + data_files.append(entry_path) + elif entry.startswith(pkg_name) and self.loader.is_package(entry): + # Package or subpackage; convert the name to directory path + package_dir = pathlib.PurePath(*entry.split('.')) + package_dirs.append(package_dir) + + # Reconstruct the filesystem + return _TocFilesystem(data_files, package_dirs) + @property def embedded_tree(self): if self._embedded_tree is None: - # Construct a path relative to _MEIPASS directory for searching - # the TOC + # Construct a path relative to _MEIPASS directory for + # searching the TOC rel_pkg_path = self._pkg_path.relative_to(SYS_PREFIX) - # Reconstruct package name prefix (use package path to obtain - # correct prefix in case of a module) + # Reconstruct package name prefix (use package path to + # obtain correct prefix in case of a module) pkg_name = '.'.join(rel_pkg_path.parts) - # Collect relevant entries from TOC. We are interested in either - # files that are located in the package/module's directory (data - # files) or in packages that are prefixed with package/module's - # name (to reconstruct subpackage directories) - data_files = [] - package_dirs = [] - for entry in self.loader.toc: - entry_path = pathlib.PurePath(entry) - if rel_pkg_path in entry_path.parents: - # Data file path - data_files.append(entry_path) - elif entry.startswith(pkg_name) and self.loader.is_package(entry): - # Package or subpackage; convert the name to directory path - package_dir = pathlib.PurePath(*entry.split('.')) - package_dirs.append(package_dir) - - # Reconstruct the filesystem - self._embedded_tree = _TocFilesystem(data_files, package_dirs) + # Initialize and cache the tree, if necessary + if pkg_name not in _toc_tree_cache: + _toc_tree_cache[pkg_name] = \ + self._init_embedded_tree(rel_pkg_path, pkg_name) + self._embedded_tree = _toc_tree_cache[pkg_name] return self._embedded_tree def _normalize_path(self, path): From a6d0075e9ad502d12bc8abbadc374e6854489759 Mon Sep 17 00:00:00 2001 From: Rok Mandeljc Date: Sat, 7 Nov 2020 13:40:47 +0100 Subject: [PATCH 6/6] hooks: pkg_resources: improve documentation Add a block describing basic behavior of PyiFrozenProvider, w.r.t. to PYZ-embedded and on-filesystem resources. --- PyInstaller/hooks/rthooks/pyi_rth_pkgres.py | 29 ++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py index f1c60a966e..f11cd84b54 100644 --- a/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py +++ b/PyInstaller/hooks/rthooks/pyi_rth_pkgres.py @@ -25,15 +25,30 @@ # import hooks like FrozenImporter is. It uses method __loader__.get_data() in # methods pkg_resources.resource_string() and pkg_resources.resource_stream() # -# We subclass the NullProvider and implement _has(), _isdir(), and _listdir(), -# which are needed for pkg_resources.resource_exists(), resource_isdir(), -# and resource_listdir() to work. We cannot use the DefaultProvider, -# because it provides filesystem-only implementations (and overrides _get() -# with a filesystem-only one), whereas our provider needs to also support -# embedded resources. +# We provide PyiFrozenProvider, which subclasses the NullProvider and +# implements _has(), _isdir(), and _listdir() methods, which are needed +# for pkg_resources.resource_exists(), resource_isdir(), and resource_listdir() +# to work. We cannot use the DefaultProvider, because it provides +# filesystem-only implementations (and overrides _get() with a filesystem-only +# one), whereas our provider needs to also support embedded resources. +# +# The PyiFrozenProvider allows querying/listing both PYZ-embedded and +# on-filesystem resources in a frozen package. The results are typically +# combined for both types of resources (e.g., when listing a directory +# or checking whether a resource exists). When the order of precedence +# matters, the PYZ-embedded resources take precedence over the on-filesystem +# ones, to keep the behavior consistent with the actual file content +# retrieval via _get() method (which in turn uses FrozenImporter's get_data() +# method). For example, when checking whether a resource is a directory +# via _isdir(), a PYZ-embedded file will take precedence over a potential +# on-filesystem directory. Also, in contrast to unfrozen packages, the frozen +# ones do not contain source .py files, which are therefore absent from +# content listings. class _TocFilesystem: + """A prefix tree implementation for embedded filesystem reconstruction.""" + def __init__(self, toc_files, toc_dirs=[]): # Reconstruct the fileystem hierarchy by building a prefix tree from # the given file and directory paths @@ -87,6 +102,8 @@ def path_listdir(self, path): class PyiFrozenProvider(pkg_resources.NullProvider): + """Custom pkg_resourvces provider for FrozenImporter.""" + def __init__(self, module): super().__init__(module)