From c0c8bd17885f9e13982bee52a753e0e03d202790 Mon Sep 17 00:00:00 2001 From: Rok Mandeljc Date: Fri, 5 Mar 2021 22:52:20 +0100 Subject: [PATCH 1/2] tests: extend the multipackage tests with nested resources Extend the multipackage test to verify references to nested resources (data files and extensions) within the shared package in the multipackage mode. For nested extension, the psutil package is used (as it contains one). Refactor the test scripts to use a common test function from a package to avoid duplicating test code. The extended tests shows that multipackage currently does not properly handle resources that are collected in subdirectories of _MEIPASS instead of _MEIPASS itself. --- .../extra-hooks/hook-multipackage_test_pkg.py | 13 ++++++ .../multipackage-scripts/multipackage1_B.py | 9 +--- .../multipackage-scripts/multipackage2_B.py | 9 +--- .../multipackage-scripts/multipackage3_B.py | 9 +--- .../multipackage-scripts/multipackage4_B.py | 9 +--- .../multipackage-scripts/multipackage5_B.py | 9 +--- .../multipackage-scripts/multipackage5_C.py | 9 +--- .../multipackage_test_pkg/__init__.py | 44 +++++++++++++++++++ .../multipackage_test_pkg/data/secret.txt | 1 + .../test_multipackage1.py | 9 +--- .../test_multipackage2.py | 9 +--- .../test_multipackage3.py | 9 +--- .../test_multipackage4.py | 9 +--- .../test_multipackage5.py | 9 +--- .../functional/specs/test_multipackage1.spec | 4 +- .../functional/specs/test_multipackage2.spec | 2 + .../functional/specs/test_multipackage3.spec | 4 +- .../functional/specs/test_multipackage4.spec | 2 + .../functional/specs/test_multipackage5.spec | 5 ++- tests/functional/test_multipackage.py | 3 ++ 20 files changed, 97 insertions(+), 80 deletions(-) create mode 100644 tests/functional/specs/multipackage-scripts/extra-hooks/hook-multipackage_test_pkg.py create mode 100644 tests/functional/specs/multipackage-scripts/multipackage_test_pkg/__init__.py create mode 100644 tests/functional/specs/multipackage-scripts/multipackage_test_pkg/data/secret.txt diff --git a/tests/functional/specs/multipackage-scripts/extra-hooks/hook-multipackage_test_pkg.py b/tests/functional/specs/multipackage-scripts/extra-hooks/hook-multipackage_test_pkg.py new file mode 100644 index 0000000000..baf3b88dd7 --- /dev/null +++ b/tests/functional/specs/multipackage-scripts/extra-hooks/hook-multipackage_test_pkg.py @@ -0,0 +1,13 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2021, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- + +from PyInstaller.utils.hooks import collect_data_files +datas = collect_data_files('multipackage_test_pkg') diff --git a/tests/functional/specs/multipackage-scripts/multipackage1_B.py b/tests/functional/specs/multipackage-scripts/multipackage1_B.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/multipackage1_B.py +++ b/tests/functional/specs/multipackage-scripts/multipackage1_B.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/multipackage2_B.py b/tests/functional/specs/multipackage-scripts/multipackage2_B.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/multipackage2_B.py +++ b/tests/functional/specs/multipackage-scripts/multipackage2_B.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/multipackage3_B.py b/tests/functional/specs/multipackage-scripts/multipackage3_B.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/multipackage3_B.py +++ b/tests/functional/specs/multipackage-scripts/multipackage3_B.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/multipackage4_B.py b/tests/functional/specs/multipackage-scripts/multipackage4_B.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/multipackage4_B.py +++ b/tests/functional/specs/multipackage-scripts/multipackage4_B.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/multipackage5_B.py b/tests/functional/specs/multipackage-scripts/multipackage5_B.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/multipackage5_B.py +++ b/tests/functional/specs/multipackage-scripts/multipackage5_B.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/multipackage5_C.py b/tests/functional/specs/multipackage-scripts/multipackage5_C.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/multipackage5_C.py +++ b/tests/functional/specs/multipackage-scripts/multipackage5_C.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/multipackage_test_pkg/__init__.py b/tests/functional/specs/multipackage-scripts/multipackage_test_pkg/__init__.py new file mode 100644 index 0000000000..c67871d865 --- /dev/null +++ b/tests/functional/specs/multipackage-scripts/multipackage_test_pkg/__init__.py @@ -0,0 +1,44 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2021, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- + +def _test_basic_imports(): + # import a very simple and rarely used pure-python lib ... + import getopt # noqa: F401 + # ... and a module importing a shared lib + import ssl # noqa: F401 + + print('Hello World!') + + +def _test_nested_data_file(): + # try reading secret from a file in sub-directory + import os + + secret_file = os.path.join(__path__[0], 'data', 'secret.txt') + print("Reading secret from %s..." % (secret_file)) + with open(secret_file, 'r') as fp: + secret = fp.read().strip() + print("Secret: %s" % (secret)) + + assert secret == 'Secret1234' + + +def _test_nested_extensions(): + # import psutil, which contains an extension in its package directory + import psutil # noqa: F401 + print("Successfully imported psutil!") + + +def test_function(): + # Main test function + _test_basic_imports() + _test_nested_data_file() + _test_nested_extensions() diff --git a/tests/functional/specs/multipackage-scripts/multipackage_test_pkg/data/secret.txt b/tests/functional/specs/multipackage-scripts/multipackage_test_pkg/data/secret.txt new file mode 100644 index 0000000000..2132f20506 --- /dev/null +++ b/tests/functional/specs/multipackage-scripts/multipackage_test_pkg/data/secret.txt @@ -0,0 +1 @@ +Secret1234 diff --git a/tests/functional/specs/multipackage-scripts/test_multipackage1.py b/tests/functional/specs/multipackage-scripts/test_multipackage1.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/test_multipackage1.py +++ b/tests/functional/specs/multipackage-scripts/test_multipackage1.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/test_multipackage2.py b/tests/functional/specs/multipackage-scripts/test_multipackage2.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/test_multipackage2.py +++ b/tests/functional/specs/multipackage-scripts/test_multipackage2.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/test_multipackage3.py b/tests/functional/specs/multipackage-scripts/test_multipackage3.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/test_multipackage3.py +++ b/tests/functional/specs/multipackage-scripts/test_multipackage3.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/test_multipackage4.py b/tests/functional/specs/multipackage-scripts/test_multipackage4.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/test_multipackage4.py +++ b/tests/functional/specs/multipackage-scripts/test_multipackage4.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/multipackage-scripts/test_multipackage5.py b/tests/functional/specs/multipackage-scripts/test_multipackage5.py index 7510a36d47..6f3d750110 100644 --- a/tests/functional/specs/multipackage-scripts/test_multipackage5.py +++ b/tests/functional/specs/multipackage-scripts/test_multipackage5.py @@ -9,10 +9,5 @@ # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) #----------------------------------------------------------------------------- - -# import a very simple and rarely used pure-python lib ... -import getopt -# ... and a module importing a shared lib -import ssl - -print('Hello World!') +import multipackage_test_pkg +multipackage_test_pkg.test_function() diff --git a/tests/functional/specs/test_multipackage1.spec b/tests/functional/specs/test_multipackage1.spec index f123e50809..764e42f5b0 100644 --- a/tests/functional/specs/test_multipackage1.spec +++ b/tests/functional/specs/test_multipackage1.spec @@ -19,8 +19,10 @@ __testname__ = 'test_multipackage1' __testdep__ = 'multipackage1_B' a = Analysis([os.path.join(SCRIPT_DIR, __testname__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) b = Analysis([os.path.join(SCRIPT_DIR, __testdep__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) MERGE((b, __testdep__, __testdep__), (a, __testname__, __testname__)) @@ -37,7 +39,7 @@ exe = EXE(pyz, strip=False, upx=False, console=1 ) - + pyzB = PYZ(b.pure) exeB = EXE(pyzB, b.scripts, diff --git a/tests/functional/specs/test_multipackage2.spec b/tests/functional/specs/test_multipackage2.spec index fc154de010..5baa0c80be 100644 --- a/tests/functional/specs/test_multipackage2.spec +++ b/tests/functional/specs/test_multipackage2.spec @@ -20,8 +20,10 @@ __testname__ = 'test_multipackage2' __testdep__ = 'multipackage2_B' a = Analysis([os.path.join(SCRIPT_DIR, __testname__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) b = Analysis([os.path.join(SCRIPT_DIR, __testdep__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) MERGE((b, __testdep__, os.path.join(__testdep__, __testdep__)), diff --git a/tests/functional/specs/test_multipackage3.spec b/tests/functional/specs/test_multipackage3.spec index 879f466a85..5d15ac72fd 100644 --- a/tests/functional/specs/test_multipackage3.spec +++ b/tests/functional/specs/test_multipackage3.spec @@ -20,8 +20,10 @@ __testname__ = 'test_multipackage3' __testdep__ = 'multipackage3_B' a = Analysis([os.path.join(SCRIPT_DIR, __testname__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) b = Analysis([os.path.join(SCRIPT_DIR, __testdep__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) MERGE((b, __testdep__, os.path.join(__testdep__)), @@ -46,7 +48,7 @@ coll = COLLECT( exe, strip=False, upx=True, name=os.path.join('dist', __testname__ )) - + pyzB = PYZ(b.pure) exeB = EXE(pyzB, b.scripts, diff --git a/tests/functional/specs/test_multipackage4.spec b/tests/functional/specs/test_multipackage4.spec index 1d9bbbdb45..7f911e2d9f 100644 --- a/tests/functional/specs/test_multipackage4.spec +++ b/tests/functional/specs/test_multipackage4.spec @@ -20,8 +20,10 @@ __testname__ = 'test_multipackage4' __testdep__ = 'multipackage4_B' a = Analysis([os.path.join(SCRIPT_DIR, __testname__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) b = Analysis([os.path.join(SCRIPT_DIR, __testdep__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) MERGE((b, __testdep__, os.path.join(__testdep__, __testdep__)), diff --git a/tests/functional/specs/test_multipackage5.spec b/tests/functional/specs/test_multipackage5.spec index e9a061516d..0327694575 100644 --- a/tests/functional/specs/test_multipackage5.spec +++ b/tests/functional/specs/test_multipackage5.spec @@ -22,10 +22,13 @@ __testdep__ = 'multipackage5_B' __testdep2__ = 'multipackage5_C' a = Analysis([os.path.join(SCRIPT_DIR, __testname__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) b = Analysis([os.path.join(SCRIPT_DIR, __testdep__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) c = Analysis([os.path.join(SCRIPT_DIR, __testdep2__ + '.py')], + hookspath=[os.path.join(SPECPATH, SCRIPT_DIR, 'extra-hooks')], pathex=['.']) @@ -72,7 +75,7 @@ coll = COLLECT( exeB, strip=False, upx=True, name=os.path.join('dist', __testdep__)) - + pyzC = PYZ(c.pure) exeC = EXE(pyzC, c.scripts, diff --git a/tests/functional/test_multipackage.py b/tests/functional/test_multipackage.py index 817e4d785f..98ed5dbf87 100644 --- a/tests/functional/test_multipackage.py +++ b/tests/functional/test_multipackage.py @@ -1,6 +1,9 @@ import pytest +from PyInstaller.utils.tests import importorskip + +@importorskip('psutil') # Used as test for nested extension @pytest.mark.parametrize( "spec_file", ( From 9956aa0e9a4f9dcca0b916e758511ed0378bb50c Mon Sep 17 00:00:00 2001 From: Rok Mandeljc Date: Fri, 5 Mar 2021 23:14:16 +0100 Subject: [PATCH 2/2] building: fix the references to nested resources in multipackage Fix MERGE() to properly set references to nested resources, i.e., with their full shared-package-relative path instead of just basename (e.g., path/to/shared/pkg:rel/path/to/file as opposed to just path/to/shared/pkg:file). The latter breaks any resources (data files, extensions/shared libraries, etc.) that are located in sub-directories of the shared package. --- PyInstaller/building/api.py | 22 +++++++++++++++++++++- news/5606.bugfix.rst | 2 ++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 news/5606.bugfix.rst diff --git a/PyInstaller/building/api.py b/PyInstaller/building/api.py index 02d5c30a9f..2b00dfaf59 100644 --- a/PyInstaller/building/api.py +++ b/PyInstaller/building/api.py @@ -824,8 +824,28 @@ def _set_dependencies(self, analysis, path): else: dep_path = self._get_relative_path(path, self._dependencies[tpl[1]]) logger.debug("Referencing %s to be a dependecy for %s, located in %s" % (tpl[1], path, dep_path)) + # Determine the path relative to dep_path (i.e, within + # the target directory) from the 'name' component + # of the TOC tuple. If entry is EXTENSION, then the + # relative path needs to be reconstructed from the + # name components. + if tpl[2] == 'EXTENSION': + ext_components = tpl[0].split('.')[:-1] + if ext_components: + rel_path = os.path.join(*ext_components) + else: + rel_path = '' + else: + rel_path = os.path.dirname(tpl[0]) + # Take filename from 'path' (second component of + # TOC tuple); this way, we don't need to worry about + # suffix of extensions. + filename = os.path.basename(tpl[1]) + # Construct the full file path relative to dep_path... + filename = os.path.join(rel_path, filename) + # ...and use it in new DEPENDENCY entry analysis.dependencies.append( - (":".join((dep_path, os.path.basename(tpl[1]))), + (":".join((dep_path, filename)), tpl[1], "DEPENDENCY")) toc[i] = (None, None, None) diff --git a/news/5606.bugfix.rst b/news/5606.bugfix.rst new file mode 100644 index 0000000000..d6e59790e0 --- /dev/null +++ b/news/5606.bugfix.rst @@ -0,0 +1,2 @@ +Fix ``MERGE()`` to properly set references to nested resources with their +full shared-package-relative path instead of just basename.