From 8e38faf93acaf979a4d311c2830208017a6b4df5 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Thu, 28 Jan 2021 19:18:40 -0300 Subject: [PATCH 01/13] Make ament_python_install_package() install a flat Python .egg Signed-off-by: Michel Hidalgo --- .../cmake/ament_python_install_package.cmake | 87 ++++++++++++++----- 1 file changed, 66 insertions(+), 21 deletions(-) diff --git a/ament_cmake_python/cmake/ament_python_install_package.cmake b/ament_cmake_python/cmake/ament_python_install_package.cmake index bba01ee7..36839064 100644 --- a/ament_cmake_python/cmake/ament_python_install_package.cmake +++ b/ament_cmake_python/cmake/ament_python_install_package.cmake @@ -13,14 +13,18 @@ # limitations under the License. # -# Install a Python package (and its recursive subpackages). +# Install a Python package (and its recursive subpackages) as a flat Python .egg # # :param package_name: the Python package name # :type package_name: string # :param PACKAGE_DIR: the path to the Python package directory (default: # folder relative to the CMAKE_CURRENT_LIST_DIR) # :type PACKAGE_DIR: string -# :param SKIP_COMPILE: if set do not compile the installed package +# :param VERSION: the Python package version (default: package.xml version) +# :param VERSION: string +# :param SETUP_CFG: the path to a setup.cfg file, if provided +# :param SETUP_CFG: string +# :param SKIP_COMPILE: if set do not byte-compile the installed package # :type SKIP_COMPILE: option # macro(ament_python_install_package) @@ -29,7 +33,7 @@ macro(ament_python_install_package) endmacro() function(_ament_cmake_python_install_package package_name) - cmake_parse_arguments(ARG "SKIP_COMPILE" "PACKAGE_DIR" "" ${ARGN}) + cmake_parse_arguments(ARG "SKIP_COMPILE" "PACKAGE_DIR;VERSION;SETUP_CFG" "" ${ARGN}) if(ARG_UNPARSED_ARGUMENTS) message(FATAL_ERROR "ament_python_install_package() called with unused " "arguments: ${ARG_UNPARSED_ARGUMENTS}") @@ -42,34 +46,75 @@ function(_ament_cmake_python_install_package package_name) set(ARG_PACKAGE_DIR "${CMAKE_CURRENT_LIST_DIR}/${ARG_PACKAGE_DIR}") endif() + if(NOT ARG_VERSION) + # Use package.xml version + if(NOT _AMENT_PACKAGE_NAME) + ament_package_xml() + endif() + set(ARG_VERSION "${${PROJECT_NAME}_VERSION}") + endif() + if(NOT EXISTS "${ARG_PACKAGE_DIR}/__init__.py") message(FATAL_ERROR "ament_python_install_package() the Python package " "folder '${ARG_PACKAGE_DIR}' doesn't contain an '__init__.py' file") endif() - _ament_cmake_python_register_environment_hook() + set(build_dir "${CMAKE_CURRENT_BINARY_DIR}/ament_cmake_python/${package_name}") - if(NOT PYTHON_INSTALL_DIR) - message(FATAL_ERROR "ament_python_install_package() variable " - "'PYTHON_INSTALL_DIR' must not be empty") - endif() - install( - DIRECTORY "${ARG_PACKAGE_DIR}/" - DESTINATION "${PYTHON_INSTALL_DIR}/${package_name}" - PATTERN "*.pyc" EXCLUDE - PATTERN "__pycache__" EXCLUDE + string(CONFIGURE "\ +from setuptools import setup + +setup( + name='${package_name}', + version='${ARG_VERSION}', + packages=['${package_name}'], + package_dir={'${package_name}': '${ARG_PACKAGE_DIR}'}, +) +" setup_py_content) + + file(GENERATE + OUTPUT "${build_dir}/setup.py" + CONTENT "${setup_py_content}" ) - if(NOT ARG_SKIP_COMPILE) - # compile Python files - install(CODE - "execute_process( - COMMAND - \"${PYTHON_EXECUTABLE}\" \"-m\" \"compileall\" - \"${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_DIR}/${package_name}\" - )" + + if(ARG_SETUP_CFG) + if(NOT IS_ABSOLUTE ${ARG_SETUP_CFG}) + set(ARG_SETUP_CFG "${CMAKE_CURRENT_LIST_DIR}/${ARG_SETUP_CFG}") + endif() + add_custom_command( + OUTPUT "${build_dir}/setup.cfg" + COMMAND ${CMAKE_COMMAND} -E copy ${ARG_SETUP_CFG} ${build_dir}/setup.cfg + MAIN_DEPENDENCY ${ARG_SETUP_CFG} ) endif() + if(NOT ARG_SKIP_COMPILE) + set(extra_install_args "--compile") + else() + set(extra_install_args "--no-compile") + endif() + + # Install as flat Python .egg to mimic https://github.com/colcon/colcon-core + # handling of pure Python packages. + + # NOTE(hidmic): Allow setup.py install to build, as there is no way to + # determine the Python package's source dependencies for proper build + # invalidation. + install(CODE + "message(STATUS \"Installing: ${package_name} as flat Python .egg \" + \"to ${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_DIR}\") + execute_process( + COMMAND + \"${PYTHON_EXECUTABLE}\" setup.py install + --single-version-externally-managed + --prefix ${CMAKE_INSTALL_PREFIX} + --record install.log + ${extra_install_args} + WORKING_DIRECTORY \"${build_dir}\" + OUTPUT_QUIET + )" + ) + if(package_name IN_LIST AMENT_CMAKE_PYTHON_INSTALL_INSTALLED_NAMES) message(FATAL_ERROR "ament_python_install_package() a Python module file or package with " From 4c518eab4c3751d6188b950ace262dc012ce2de8 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Fri, 29 Jan 2021 19:28:34 -0300 Subject: [PATCH 02/13] Address peer review comments. Signed-off-by: Michel Hidalgo --- .../cmake/ament_python_install_package.cmake | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ament_cmake_python/cmake/ament_python_install_package.cmake b/ament_cmake_python/cmake/ament_python_install_package.cmake index 36839064..783f664d 100644 --- a/ament_cmake_python/cmake/ament_python_install_package.cmake +++ b/ament_cmake_python/cmake/ament_python_install_package.cmake @@ -13,7 +13,7 @@ # limitations under the License. # -# Install a Python package (and its recursive subpackages) as a flat Python .egg +# Install a Python package (and its recursive subpackages) as a flat Python egg # # :param package_name: the Python package name # :type package_name: string @@ -22,7 +22,8 @@ # :type PACKAGE_DIR: string # :param VERSION: the Python package version (default: package.xml version) # :param VERSION: string -# :param SETUP_CFG: the path to a setup.cfg file, if provided +# :param SETUP_CFG: the path to a setup.cfg file (default: +# setup.cfg file at CMAKE_CURRENT_LIST_DIR root, if any) # :param SETUP_CFG: string # :param SKIP_COMPILE: if set do not byte-compile the installed package # :type SKIP_COMPILE: option @@ -59,6 +60,14 @@ function(_ament_cmake_python_install_package package_name) "folder '${ARG_PACKAGE_DIR}' doesn't contain an '__init__.py' file") endif() + if(NOT ARG_SETUP_CFG) + if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/setup.cfg") + set(ARG_SETUP_CFG "${CMAKE_CURRENT_LIST_DIR}/setup.cfg") + endif() + elseif(NOT IS_ABSOLUTE "${ARG_SETUP_CFG}") + set(ARG_SETUP_CFG "${CMAKE_CURRENT_LIST_DIR}/${ARG_SETUP_CFG}") + endif() + set(build_dir "${CMAKE_CURRENT_BINARY_DIR}/ament_cmake_python/${package_name}") string(CONFIGURE "\ @@ -78,14 +87,14 @@ setup( ) if(ARG_SETUP_CFG) - if(NOT IS_ABSOLUTE ${ARG_SETUP_CFG}) - set(ARG_SETUP_CFG "${CMAKE_CURRENT_LIST_DIR}/${ARG_SETUP_CFG}") - endif() add_custom_command( OUTPUT "${build_dir}/setup.cfg" COMMAND ${CMAKE_COMMAND} -E copy ${ARG_SETUP_CFG} ${build_dir}/setup.cfg MAIN_DEPENDENCY ${ARG_SETUP_CFG} ) + add_custom_target(${package_name}_setup ALL + DEPENDS "${build_dir}/setup.cfg" + ) endif() if(NOT ARG_SKIP_COMPILE) @@ -101,7 +110,7 @@ setup( # determine the Python package's source dependencies for proper build # invalidation. install(CODE - "message(STATUS \"Installing: ${package_name} as flat Python .egg \" + "message(STATUS \"Installing: ${package_name} as flat Python egg \" \"to ${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_DIR}\") execute_process( COMMAND From 1589dfb53fc675f48ecbd198ea29e9c0716eb07d Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Tue, 2 Feb 2021 16:34:05 -0300 Subject: [PATCH 03/13] Include subpackages and data files. Signed-off-by: Michel Hidalgo --- .../cmake/ament_python_install_package.cmake | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ament_cmake_python/cmake/ament_python_install_package.cmake b/ament_cmake_python/cmake/ament_python_install_package.cmake index 783f664d..2b7609c6 100644 --- a/ament_cmake_python/cmake/ament_python_install_package.cmake +++ b/ament_cmake_python/cmake/ament_python_install_package.cmake @@ -69,15 +69,20 @@ function(_ament_cmake_python_install_package package_name) endif() set(build_dir "${CMAKE_CURRENT_BINARY_DIR}/ament_cmake_python/${package_name}") + file(RELATIVE_PATH package_dir "${build_dir}" "${ARG_PACKAGE_DIR}") string(CONFIGURE "\ +from setuptools import find_packages from setuptools import setup setup( name='${package_name}', version='${ARG_VERSION}', - packages=['${package_name}'], - package_dir={'${package_name}': '${ARG_PACKAGE_DIR}'}, + packages=find_packages( + where='${package_dir}/..', + include=('${package_name}*')), + package_dir={'${package_name}': '${package_dir}'}, + package_data={'': ['*.*']} ) " setup_py_content) From b3d783d11b89bac52c10ed30618589fb138236bf Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Tue, 2 Feb 2021 18:49:59 -0300 Subject: [PATCH 04/13] Fix package glob expressions Signed-off-by: Michel Hidalgo --- ament_cmake_python/cmake/ament_python_install_package.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ament_cmake_python/cmake/ament_python_install_package.cmake b/ament_cmake_python/cmake/ament_python_install_package.cmake index 2b7609c6..2fa4ce6d 100644 --- a/ament_cmake_python/cmake/ament_python_install_package.cmake +++ b/ament_cmake_python/cmake/ament_python_install_package.cmake @@ -80,7 +80,7 @@ setup( version='${ARG_VERSION}', packages=find_packages( where='${package_dir}/..', - include=('${package_name}*')), + include=('${package_name}', '${package_name}.*')), package_dir={'${package_name}': '${package_dir}'}, package_data={'': ['*.*']} ) From 57af8f56ff36bb216ed0a09f71c6d9b5e56284e1 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Mon, 8 Feb 2021 16:28:07 -0300 Subject: [PATCH 05/13] Always use relative paths Signed-off-by: Michel Hidalgo --- .../cmake/ament_python_install_package.cmake | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ament_cmake_python/cmake/ament_python_install_package.cmake b/ament_cmake_python/cmake/ament_python_install_package.cmake index 2fa4ce6d..f2dc0648 100644 --- a/ament_cmake_python/cmake/ament_python_install_package.cmake +++ b/ament_cmake_python/cmake/ament_python_install_package.cmake @@ -69,7 +69,7 @@ function(_ament_cmake_python_install_package package_name) endif() set(build_dir "${CMAKE_CURRENT_BINARY_DIR}/ament_cmake_python/${package_name}") - file(RELATIVE_PATH package_dir "${build_dir}" "${ARG_PACKAGE_DIR}") + file(RELATIVE_PATH source_dir "${build_dir}" "${ARG_PACKAGE_DIR}") string(CONFIGURE "\ from setuptools import find_packages @@ -79,9 +79,9 @@ setup( name='${package_name}', version='${ARG_VERSION}', packages=find_packages( - where='${package_dir}/..', + where='${source_dir}/..', include=('${package_name}', '${package_name}.*')), - package_dir={'${package_name}': '${package_dir}'}, + package_dir={'${package_name}': '${source_dir}'}, package_data={'': ['*.*']} ) " setup_py_content) @@ -110,6 +110,7 @@ setup( # Install as flat Python .egg to mimic https://github.com/colcon/colcon-core # handling of pure Python packages. + file(RELATIVE_PATH install_dir "${build_dir}" "${CMAKE_INSTALL_PREFIX}") # NOTE(hidmic): Allow setup.py install to build, as there is no way to # determine the Python package's source dependencies for proper build @@ -121,7 +122,7 @@ setup( COMMAND \"${PYTHON_EXECUTABLE}\" setup.py install --single-version-externally-managed - --prefix ${CMAKE_INSTALL_PREFIX} + --prefix \"${install_dir}\" --record install.log ${extra_install_args} WORKING_DIRECTORY \"${build_dir}\" From a55116e9057192187eb4c5b540a01de341d42a50 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Mon, 8 Feb 2021 19:31:32 -0300 Subject: [PATCH 06/13] Normalize setuptools.find_packages() path It fails on Windows otherwise Signed-off-by: Michel Hidalgo --- ament_cmake_python/cmake/ament_python_install_package.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ament_cmake_python/cmake/ament_python_install_package.cmake b/ament_cmake_python/cmake/ament_python_install_package.cmake index f2dc0648..22b53bfc 100644 --- a/ament_cmake_python/cmake/ament_python_install_package.cmake +++ b/ament_cmake_python/cmake/ament_python_install_package.cmake @@ -72,6 +72,7 @@ function(_ament_cmake_python_install_package package_name) file(RELATIVE_PATH source_dir "${build_dir}" "${ARG_PACKAGE_DIR}") string(CONFIGURE "\ +import os from setuptools import find_packages from setuptools import setup @@ -79,7 +80,7 @@ setup( name='${package_name}', version='${ARG_VERSION}', packages=find_packages( - where='${source_dir}/..', + where=os.path.normpath('${source_dir}/..'), include=('${package_name}', '${package_name}.*')), package_dir={'${package_name}': '${source_dir}'}, package_data={'': ['*.*']} From 7d8cc5c599c10cabe36651fbe082b11995c48f43 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Mon, 15 Feb 2021 10:39:11 -0300 Subject: [PATCH 07/13] Add a few missing quotes Signed-off-by: Michel Hidalgo --- ament_cmake_python/cmake/ament_python_install_package.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ament_cmake_python/cmake/ament_python_install_package.cmake b/ament_cmake_python/cmake/ament_python_install_package.cmake index 22b53bfc..a8f2f64e 100644 --- a/ament_cmake_python/cmake/ament_python_install_package.cmake +++ b/ament_cmake_python/cmake/ament_python_install_package.cmake @@ -95,8 +95,8 @@ setup( if(ARG_SETUP_CFG) add_custom_command( OUTPUT "${build_dir}/setup.cfg" - COMMAND ${CMAKE_COMMAND} -E copy ${ARG_SETUP_CFG} ${build_dir}/setup.cfg - MAIN_DEPENDENCY ${ARG_SETUP_CFG} + COMMAND ${CMAKE_COMMAND} -E copy "${ARG_SETUP_CFG}" "${build_dir}/setup.cfg" + MAIN_DEPENDENCY "${ARG_SETUP_CFG}" ) add_custom_target(${package_name}_setup ALL DEPENDS "${build_dir}/setup.cfg" From 7876939bd429a38f60d1b7339ef1285990349dfd Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Mon, 15 Feb 2021 13:50:10 -0300 Subject: [PATCH 08/13] Rework how package_data is collected. - Handle files with no extension - Handle arbitrarily nested subdirectories Signed-off-by: Michel Hidalgo --- ament_cmake_python/CMakeLists.txt | 4 + .../ament_cmake_python/__init__.py | 104 ++++++++++++++++++ .../cmake/ament_python_install_package.cmake | 41 +++++-- 3 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 ament_cmake_python/ament_cmake_python/__init__.py diff --git a/ament_cmake_python/CMakeLists.txt b/ament_cmake_python/CMakeLists.txt index 7e760c58..ea292833 100644 --- a/ament_cmake_python/CMakeLists.txt +++ b/ament_cmake_python/CMakeLists.txt @@ -4,6 +4,10 @@ project(ament_cmake_python NONE) find_package(ament_cmake_core REQUIRED) +set(ament_cmake_python_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +include("ament_cmake_python-extras.cmake") +ament_python_install_package(${PROJECT_NAME} NO_DATA) + ament_package( CONFIG_EXTRAS "ament_cmake_python-extras.cmake" ) diff --git a/ament_cmake_python/ament_cmake_python/__init__.py b/ament_cmake_python/ament_cmake_python/__init__.py new file mode 100644 index 00000000..11c2e22d --- /dev/null +++ b/ament_cmake_python/ament_cmake_python/__init__.py @@ -0,0 +1,104 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import pathlib +from urllib.parse import urlparse + +from setuptools import find_packages + + +def fuzzy_lookup(key, mapping): + """Lookup key in a mapping where keys may contain wildcards ('*').""" + for pattern, value in mapping.items(): + if fnmatch.fnmatch(key, pattern): + yield value + + +def find_packages_data(where, exclude=(), include=()): + """ + Find data in Python packages found within directory 'where'. + + Similar to `setuptools.find_packages`. + + :param where: a cross-platform (i.e. URL-style) path + :param exclude: a dictionary that maps from package names to + lists of glob patterns to be excluded from the search. Wildcards + ('*') may be used in package names. A collection of package names + may be provided instead of a dictionary, in which case whole packages + will be excluded. + :param include: a dictionary that maps from package names to + lists of glob patterns to be included in the search. Wildcards + ('*') may be used in package names. A collection of package names + may be provided instead of a dictionary, in which case whole packages + will be included but excluding Python sources and byte-compiled code. + :returns: a dictionary suitable to be used as 'package_data' when calling + `setuptools.setup` + """ + packages = find_packages( + where=where, include=set(include), + # Defer whole package exclusion (may be partial) + exclude=set(exclude) - set(include)) + + where = pathlib.Path(urlparse(where).path) + if not include: + # Use what find_packages() found + include = packages + if not isinstance(exclude, dict): + # Exclude whole packages + exclude = {name: ['**/*'] for name in exclude} + if not isinstance(include, dict): + # Include whole packages + include = {name: ['**/*'] for name in include} + # But + for name in include: + # Exclude Python sources and byte-compiled code + if name not in exclude: + exclude[name] = [] + exclude[name].extend([ + '**/*.py', '**/*.pyc', + '**/__pycache__/**/*' + ]) + + packages_data = {} + processed_data = set() + # Bottom-up search for packages' data + for name in sorted(packages, reverse=True): + rootpath = where / name.replace('.', '/') + + # Exclude nested packages' content too + excluded_data = set(processed_data) + for patterns in fuzzy_lookup(name, exclude): + excluded_data.update( + path for pattern in patterns + for path in rootpath.glob(pattern) + ) + + included_data = set() + for patterns in fuzzy_lookup(name, include): + included_data.update( + path for pattern in patterns + for path in rootpath.glob(pattern) + if path not in excluded_data + ) + + packages_data[name] = [ + str(path.relative_to(rootpath)) + for path in included_data + ] + + # Keep track of packages processed + processed_data.update(rootpath.glob('**/*')) + + return packages_data diff --git a/ament_cmake_python/cmake/ament_python_install_package.cmake b/ament_cmake_python/cmake/ament_python_install_package.cmake index a8f2f64e..b7d0bd2c 100644 --- a/ament_cmake_python/cmake/ament_python_install_package.cmake +++ b/ament_cmake_python/cmake/ament_python_install_package.cmake @@ -27,6 +27,8 @@ # :param SETUP_CFG: string # :param SKIP_COMPILE: if set do not byte-compile the installed package # :type SKIP_COMPILE: option +# :param NO_DATA: if set do not install any package data +# :type NO_DATA: option # macro(ament_python_install_package) _ament_cmake_python_register_environment_hook() @@ -34,7 +36,7 @@ macro(ament_python_install_package) endmacro() function(_ament_cmake_python_install_package package_name) - cmake_parse_arguments(ARG "SKIP_COMPILE" "PACKAGE_DIR;VERSION;SETUP_CFG" "" ${ARGN}) + cmake_parse_arguments(ARG "SKIP_COMPILE;NO_DATA" "PACKAGE_DIR;VERSION;SETUP_CFG" "" ${ARGN}) if(ARG_UNPARSED_ARGUMENTS) message(FATAL_ERROR "ament_python_install_package() called with unused " "arguments: ${ARG_UNPARSED_ARGUMENTS}") @@ -71,21 +73,42 @@ function(_ament_cmake_python_install_package package_name) set(build_dir "${CMAKE_CURRENT_BINARY_DIR}/ament_cmake_python/${package_name}") file(RELATIVE_PATH source_dir "${build_dir}" "${ARG_PACKAGE_DIR}") - string(CONFIGURE "\ + if(ARG_NO_DATA) + string(CONFIGURE "\ import os from setuptools import find_packages from setuptools import setup setup( - name='${package_name}', - version='${ARG_VERSION}', - packages=find_packages( - where=os.path.normpath('${source_dir}/..'), - include=('${package_name}', '${package_name}.*')), - package_dir={'${package_name}': '${source_dir}'}, - package_data={'': ['*.*']} + name='${package_name}', + version='${ARG_VERSION}', + packages=find_packages( + where=os.path.normpath('${source_dir}/..'), + include=('${package_name}', '${package_name}.*')), + package_dir={'${package_name}': '${source_dir}'}, ) " setup_py_content) + else() + string(CONFIGURE "\ +import os +from setuptools import find_packages +from setuptools import setup + +from ament_cmake_python import find_packages_data + +setup( + name='${package_name}', + version='${ARG_VERSION}', + packages=find_packages( + where=os.path.normpath('${source_dir}/..'), + include=('${package_name}', '${package_name}.*')), + package_dir={'${package_name}': '${source_dir}'}, + package_data=find_packages_data( + where=os.path.normpath('${source_dir}/..'), + include=('${package_name}', '${package_name}.*')) +) +" setup_py_content) + endif() file(GENERATE OUTPUT "${build_dir}/setup.py" From a2f1349dd42c3630116beb1c39da60e4998aab86 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Mon, 15 Feb 2021 15:19:43 -0300 Subject: [PATCH 09/13] Add tests for find_packages_data() Signed-off-by: Michel Hidalgo --- ament_cmake_python/CMakeLists.txt | 14 ++++ .../ament_cmake_python/__init__.py | 22 +++--- ament_cmake_python/test/baz/__init__.py | 0 ament_cmake_python/test/baz/data | 0 ament_cmake_python/test/baz/data.bin | 0 ament_cmake_python/test/foo/__init__.py | 0 ament_cmake_python/test/foo/bar/__init__.py | 0 ament_cmake_python/test/foo/bar/data.txt | 0 .../test/foo/bar/resources/buzz.txt | 0 .../test/foo/bar/resources/fizz.txt | 0 ament_cmake_python/test/foo/data | 0 ament_cmake_python/test/foo/data.txt | 0 .../test/test_find_packages_data.py | 75 +++++++++++++++++++ 13 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 ament_cmake_python/test/baz/__init__.py create mode 100644 ament_cmake_python/test/baz/data create mode 100644 ament_cmake_python/test/baz/data.bin create mode 100644 ament_cmake_python/test/foo/__init__.py create mode 100644 ament_cmake_python/test/foo/bar/__init__.py create mode 100644 ament_cmake_python/test/foo/bar/data.txt create mode 100644 ament_cmake_python/test/foo/bar/resources/buzz.txt create mode 100644 ament_cmake_python/test/foo/bar/resources/fizz.txt create mode 100644 ament_cmake_python/test/foo/data create mode 100644 ament_cmake_python/test/foo/data.txt create mode 100644 ament_cmake_python/test/test_find_packages_data.py diff --git a/ament_cmake_python/CMakeLists.txt b/ament_cmake_python/CMakeLists.txt index ea292833..1b86d3ae 100644 --- a/ament_cmake_python/CMakeLists.txt +++ b/ament_cmake_python/CMakeLists.txt @@ -8,6 +8,20 @@ set(ament_cmake_python_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include("ament_cmake_python-extras.cmake") ament_python_install_package(${PROJECT_NAME} NO_DATA) +include(CTest) + +if(BUILD_TESTING) + add_test( + NAME test_find_packages_data + COMMAND "${PYTHON_EXECUTABLE}" test_find_packages_data.py + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test" + ) + set_tests_properties(test_find_packages_data PROPERTIES + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 30 + ) +endif() + ament_package( CONFIG_EXTRAS "ament_cmake_python-extras.cmake" ) diff --git a/ament_cmake_python/ament_cmake_python/__init__.py b/ament_cmake_python/ament_cmake_python/__init__.py index 11c2e22d..0905b305 100644 --- a/ament_cmake_python/ament_cmake_python/__init__.py +++ b/ament_cmake_python/ament_cmake_python/__init__.py @@ -26,7 +26,7 @@ def fuzzy_lookup(key, mapping): yield value -def find_packages_data(where, exclude=(), include=()): +def find_packages_data(where='.', exclude=(), include=('*',)): """ Find data in Python packages found within directory 'where'. @@ -48,13 +48,9 @@ def find_packages_data(where, exclude=(), include=()): """ packages = find_packages( where=where, include=set(include), - # Defer whole package exclusion (may be partial) - exclude=set(exclude) - set(include)) - + # Defer package exclusion (may be partial) + ) where = pathlib.Path(urlparse(where).path) - if not include: - # Use what find_packages() found - include = packages if not isinstance(exclude, dict): # Exclude whole packages exclude = {name: ['**/*'] for name in exclude} @@ -90,15 +86,17 @@ def find_packages_data(where, exclude=(), include=()): included_data.update( path for pattern in patterns for path in rootpath.glob(pattern) - if path not in excluded_data + if not path.is_dir() and path not in excluded_data ) - packages_data[name] = [ - str(path.relative_to(rootpath)) - for path in included_data - ] + if included_data: + packages_data[name] = [ + str(path.relative_to(rootpath)) + for path in included_data + ] # Keep track of packages processed processed_data.update(rootpath.glob('**/*')) + processed_data.add(rootpath) return packages_data diff --git a/ament_cmake_python/test/baz/__init__.py b/ament_cmake_python/test/baz/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/baz/data b/ament_cmake_python/test/baz/data new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/baz/data.bin b/ament_cmake_python/test/baz/data.bin new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/foo/__init__.py b/ament_cmake_python/test/foo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/foo/bar/__init__.py b/ament_cmake_python/test/foo/bar/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/foo/bar/data.txt b/ament_cmake_python/test/foo/bar/data.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/foo/bar/resources/buzz.txt b/ament_cmake_python/test/foo/bar/resources/buzz.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/foo/bar/resources/fizz.txt b/ament_cmake_python/test/foo/bar/resources/fizz.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/foo/data b/ament_cmake_python/test/foo/data new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/foo/data.txt b/ament_cmake_python/test/foo/data.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/test_find_packages_data.py b/ament_cmake_python/test/test_find_packages_data.py new file mode 100644 index 00000000..97751d53 --- /dev/null +++ b/ament_cmake_python/test/test_find_packages_data.py @@ -0,0 +1,75 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from ament_cmake_python import find_packages_data + + +class TestFindPackagesData(unittest.TestCase): + + def test_all_packages_data_is_found(self): + data = find_packages_data() + assert set(data) == {'foo', 'foo.bar', 'baz'} + assert set(data['foo']) == {'data', 'data.txt'} + assert set(data['foo.bar']) == { + 'data.txt', + 'resources/fizz.txt', + 'resources/buzz.txt' + } + assert set(data['baz']) == {'data.bin', 'data'} + + def test_whole_package_data_is_included(self): + data = find_packages_data( + include=('foo', 'foo.*')) + assert set(data) == {'foo', 'foo.bar'} + assert set(data['foo']) == {'data', 'data.txt'} + assert set(data['foo.bar']) == { + 'data.txt', + 'resources/fizz.txt', + 'resources/buzz.txt' + } + + def test_whole_package_data_is_excluded(self): + data = find_packages_data( + include=('foo', 'foo.*'), + exclude=('foo.bar',)) + assert set(data) == {'foo'} + assert set(data['foo']) == {'data', 'data.txt'} + + def test_partial_package_data_is_excluded(self): + data = find_packages_data( + include=('foo', 'foo.*'), + exclude={'foo.bar': ['resources/*']}) + assert set(data) == {'foo', 'foo.bar'} + assert set(data['foo']) == {'data', 'data.txt'} + assert set(data['foo.bar']) == {'data.txt'} + + def test_partial_package_data_is_included(self): + data = find_packages_data( + include={ + 'foo': ['*.txt'], + 'foo.*': ['resources/*.txt'] + }, + ) + assert set(data) == {'foo', 'foo.bar'} + assert set(data['foo']) == {'data.txt'} + assert set(data['foo.bar']) == { + 'resources/fizz.txt', + 'resources/buzz.txt' + } + + +if __name__ == '__main__': + unittest.main() From d6d13a23e8f61a4bcb4739f5c85568b3d495d7cf Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Wed, 17 Feb 2021 10:57:45 -0300 Subject: [PATCH 10/13] Fix find_packages_data() tests on Windows. Signed-off-by: Michel Hidalgo --- ament_cmake_python/test/test_find_packages_data.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ament_cmake_python/test/test_find_packages_data.py b/ament_cmake_python/test/test_find_packages_data.py index 97751d53..969c4be6 100644 --- a/ament_cmake_python/test/test_find_packages_data.py +++ b/ament_cmake_python/test/test_find_packages_data.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import unittest from ament_cmake_python import find_packages_data @@ -25,8 +26,8 @@ def test_all_packages_data_is_found(self): assert set(data['foo']) == {'data', 'data.txt'} assert set(data['foo.bar']) == { 'data.txt', - 'resources/fizz.txt', - 'resources/buzz.txt' + os.path.join('resources', 'fizz.txt'), + os.path.join('resources', 'buzz.txt') } assert set(data['baz']) == {'data.bin', 'data'} @@ -37,8 +38,8 @@ def test_whole_package_data_is_included(self): assert set(data['foo']) == {'data', 'data.txt'} assert set(data['foo.bar']) == { 'data.txt', - 'resources/fizz.txt', - 'resources/buzz.txt' + os.path.join('resources', 'fizz.txt'), + os.path.join('resources', 'buzz.txt') } def test_whole_package_data_is_excluded(self): @@ -66,8 +67,8 @@ def test_partial_package_data_is_included(self): assert set(data) == {'foo', 'foo.bar'} assert set(data['foo']) == {'data.txt'} assert set(data['foo.bar']) == { - 'resources/fizz.txt', - 'resources/buzz.txt' + os.path.join('resources', 'fizz.txt'), + os.path.join('resources', 'buzz.txt') } From e7be7d40620e59d94e5c81c23f4d41953672a90e Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Wed, 17 Feb 2021 17:01:21 -0300 Subject: [PATCH 11/13] Test byte-compiled Python sources are excluded. Signed-off-by: Michel Hidalgo --- ament_cmake_python/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ament_cmake_python/CMakeLists.txt b/ament_cmake_python/CMakeLists.txt index 1b86d3ae..308249cc 100644 --- a/ament_cmake_python/CMakeLists.txt +++ b/ament_cmake_python/CMakeLists.txt @@ -11,6 +11,11 @@ ament_python_install_package(${PROJECT_NAME} NO_DATA) include(CTest) if(BUILD_TESTING) + execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -m py_compile + foo/__init__.py foo/bar/__init__.py baz/__init__.py + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test" + ) add_test( NAME test_find_packages_data COMMAND "${PYTHON_EXECUTABLE}" test_find_packages_data.py From 4db9bb83c09d3912815077567b46f6091e8abbeb Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Wed, 17 Feb 2021 17:02:15 -0300 Subject: [PATCH 12/13] Ignore byte-compiled Python files Signed-off-by: Michel Hidalgo --- ament_cmake_python/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 ament_cmake_python/.gitignore diff --git a/ament_cmake_python/.gitignore b/ament_cmake_python/.gitignore new file mode 100644 index 00000000..c678a5e1 --- /dev/null +++ b/ament_cmake_python/.gitignore @@ -0,0 +1,3 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] From 54aeedbd1ac198989de7c641b47e0d54dc0c6e86 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Wed, 17 Feb 2021 17:16:08 -0300 Subject: [PATCH 13/13] Add test for nested Python package in nested directory Signed-off-by: Michel Hidalgo --- ament_cmake_python/CMakeLists.txt | 8 +++++--- ament_cmake_python/test/{ => data}/baz/__init__.py | 0 ament_cmake_python/test/{ => data}/baz/data | 0 ament_cmake_python/test/{ => data}/baz/data.bin | 0 ament_cmake_python/test/{ => data}/foo/__init__.py | 0 ament_cmake_python/test/{ => data}/foo/bar/__init__.py | 0 ament_cmake_python/test/{ => data}/foo/bar/data.txt | 0 .../test/{ => data}/foo/bar/resources/buzz.txt | 0 .../test/{ => data}/foo/bar/resources/fizz.txt | 0 ament_cmake_python/test/{ => data}/foo/data | 0 ament_cmake_python/test/{ => data}/foo/data.txt | 0 ament_cmake_python/test/data/nested/pkgs/fizz/__init__.py | 0 .../test/data/nested/pkgs/fizz/buzz/__init__.py | 0 .../test/data/nested/pkgs/fizz/buzz/data.txt | 0 .../test/data/nested/pkgs/fizz/data/buzz.bin | 0 ament_cmake_python/test/test_find_packages_data.py | 8 ++++++++ 16 files changed, 13 insertions(+), 3 deletions(-) rename ament_cmake_python/test/{ => data}/baz/__init__.py (100%) rename ament_cmake_python/test/{ => data}/baz/data (100%) rename ament_cmake_python/test/{ => data}/baz/data.bin (100%) rename ament_cmake_python/test/{ => data}/foo/__init__.py (100%) rename ament_cmake_python/test/{ => data}/foo/bar/__init__.py (100%) rename ament_cmake_python/test/{ => data}/foo/bar/data.txt (100%) rename ament_cmake_python/test/{ => data}/foo/bar/resources/buzz.txt (100%) rename ament_cmake_python/test/{ => data}/foo/bar/resources/fizz.txt (100%) rename ament_cmake_python/test/{ => data}/foo/data (100%) rename ament_cmake_python/test/{ => data}/foo/data.txt (100%) create mode 100644 ament_cmake_python/test/data/nested/pkgs/fizz/__init__.py create mode 100644 ament_cmake_python/test/data/nested/pkgs/fizz/buzz/__init__.py create mode 100644 ament_cmake_python/test/data/nested/pkgs/fizz/buzz/data.txt create mode 100644 ament_cmake_python/test/data/nested/pkgs/fizz/data/buzz.bin diff --git a/ament_cmake_python/CMakeLists.txt b/ament_cmake_python/CMakeLists.txt index 308249cc..d29d7674 100644 --- a/ament_cmake_python/CMakeLists.txt +++ b/ament_cmake_python/CMakeLists.txt @@ -14,12 +14,14 @@ if(BUILD_TESTING) execute_process( COMMAND "${PYTHON_EXECUTABLE}" -m py_compile foo/__init__.py foo/bar/__init__.py baz/__init__.py - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/data" ) add_test( NAME test_find_packages_data - COMMAND "${PYTHON_EXECUTABLE}" test_find_packages_data.py - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test" + COMMAND + "${PYTHON_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/test/test_find_packages_data.py" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/data" ) set_tests_properties(test_find_packages_data PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}" diff --git a/ament_cmake_python/test/baz/__init__.py b/ament_cmake_python/test/data/baz/__init__.py similarity index 100% rename from ament_cmake_python/test/baz/__init__.py rename to ament_cmake_python/test/data/baz/__init__.py diff --git a/ament_cmake_python/test/baz/data b/ament_cmake_python/test/data/baz/data similarity index 100% rename from ament_cmake_python/test/baz/data rename to ament_cmake_python/test/data/baz/data diff --git a/ament_cmake_python/test/baz/data.bin b/ament_cmake_python/test/data/baz/data.bin similarity index 100% rename from ament_cmake_python/test/baz/data.bin rename to ament_cmake_python/test/data/baz/data.bin diff --git a/ament_cmake_python/test/foo/__init__.py b/ament_cmake_python/test/data/foo/__init__.py similarity index 100% rename from ament_cmake_python/test/foo/__init__.py rename to ament_cmake_python/test/data/foo/__init__.py diff --git a/ament_cmake_python/test/foo/bar/__init__.py b/ament_cmake_python/test/data/foo/bar/__init__.py similarity index 100% rename from ament_cmake_python/test/foo/bar/__init__.py rename to ament_cmake_python/test/data/foo/bar/__init__.py diff --git a/ament_cmake_python/test/foo/bar/data.txt b/ament_cmake_python/test/data/foo/bar/data.txt similarity index 100% rename from ament_cmake_python/test/foo/bar/data.txt rename to ament_cmake_python/test/data/foo/bar/data.txt diff --git a/ament_cmake_python/test/foo/bar/resources/buzz.txt b/ament_cmake_python/test/data/foo/bar/resources/buzz.txt similarity index 100% rename from ament_cmake_python/test/foo/bar/resources/buzz.txt rename to ament_cmake_python/test/data/foo/bar/resources/buzz.txt diff --git a/ament_cmake_python/test/foo/bar/resources/fizz.txt b/ament_cmake_python/test/data/foo/bar/resources/fizz.txt similarity index 100% rename from ament_cmake_python/test/foo/bar/resources/fizz.txt rename to ament_cmake_python/test/data/foo/bar/resources/fizz.txt diff --git a/ament_cmake_python/test/foo/data b/ament_cmake_python/test/data/foo/data similarity index 100% rename from ament_cmake_python/test/foo/data rename to ament_cmake_python/test/data/foo/data diff --git a/ament_cmake_python/test/foo/data.txt b/ament_cmake_python/test/data/foo/data.txt similarity index 100% rename from ament_cmake_python/test/foo/data.txt rename to ament_cmake_python/test/data/foo/data.txt diff --git a/ament_cmake_python/test/data/nested/pkgs/fizz/__init__.py b/ament_cmake_python/test/data/nested/pkgs/fizz/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/nested/pkgs/fizz/buzz/__init__.py b/ament_cmake_python/test/data/nested/pkgs/fizz/buzz/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/nested/pkgs/fizz/buzz/data.txt b/ament_cmake_python/test/data/nested/pkgs/fizz/buzz/data.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/nested/pkgs/fizz/data/buzz.bin b/ament_cmake_python/test/data/nested/pkgs/fizz/data/buzz.bin new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/test_find_packages_data.py b/ament_cmake_python/test/test_find_packages_data.py index 969c4be6..1a32b7fb 100644 --- a/ament_cmake_python/test/test_find_packages_data.py +++ b/ament_cmake_python/test/test_find_packages_data.py @@ -71,6 +71,14 @@ def test_partial_package_data_is_included(self): os.path.join('resources', 'buzz.txt') } + def test_nested_packages_data_is_found(self): + data = find_packages_data(where='nested/pkgs') + assert set(data) == {'fizz', 'fizz.buzz'} + assert set(data['fizz']) == { + os.path.join('data', 'buzz.bin') + } + assert set(data['fizz.buzz']) == {'data.txt'} + if __name__ == '__main__': unittest.main()