diff --git a/ament_cmake_python/.gitignore b/ament_cmake_python/.gitignore new file mode 100644 index 00000000..c678a5e1 --- /dev/null +++ b/ament_cmake_python/.gitignore @@ -0,0 +1,3 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] diff --git a/ament_cmake_python/CMakeLists.txt b/ament_cmake_python/CMakeLists.txt index 7e760c58..d29d7674 100644 --- a/ament_cmake_python/CMakeLists.txt +++ b/ament_cmake_python/CMakeLists.txt @@ -4,6 +4,31 @@ project(ament_cmake_python NONE) find_package(ament_cmake_core REQUIRED) +set(ament_cmake_python_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +include("ament_cmake_python-extras.cmake") +ament_python_install_package(${PROJECT_NAME} NO_DATA) + +include(CTest) + +if(BUILD_TESTING) + execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -m py_compile + foo/__init__.py foo/bar/__init__.py baz/__init__.py + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/data" + ) + add_test( + NAME test_find_packages_data + COMMAND + "${PYTHON_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/test/test_find_packages_data.py" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/data" + ) + set_tests_properties(test_find_packages_data PROPERTIES + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}" + TIMEOUT 30 + ) +endif() + ament_package( CONFIG_EXTRAS "ament_cmake_python-extras.cmake" ) diff --git a/ament_cmake_python/ament_cmake_python/__init__.py b/ament_cmake_python/ament_cmake_python/__init__.py new file mode 100644 index 00000000..0905b305 --- /dev/null +++ b/ament_cmake_python/ament_cmake_python/__init__.py @@ -0,0 +1,102 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import pathlib +from urllib.parse import urlparse + +from setuptools import find_packages + + +def fuzzy_lookup(key, mapping): + """Lookup key in a mapping where keys may contain wildcards ('*').""" + for pattern, value in mapping.items(): + if fnmatch.fnmatch(key, pattern): + yield value + + +def find_packages_data(where='.', exclude=(), include=('*',)): + """ + Find data in Python packages found within directory 'where'. + + Similar to `setuptools.find_packages`. + + :param where: a cross-platform (i.e. URL-style) path + :param exclude: a dictionary that maps from package names to + lists of glob patterns to be excluded from the search. Wildcards + ('*') may be used in package names. A collection of package names + may be provided instead of a dictionary, in which case whole packages + will be excluded. + :param include: a dictionary that maps from package names to + lists of glob patterns to be included in the search. Wildcards + ('*') may be used in package names. A collection of package names + may be provided instead of a dictionary, in which case whole packages + will be included but excluding Python sources and byte-compiled code. + :returns: a dictionary suitable to be used as 'package_data' when calling + `setuptools.setup` + """ + packages = find_packages( + where=where, include=set(include), + # Defer package exclusion (may be partial) + ) + where = pathlib.Path(urlparse(where).path) + if not isinstance(exclude, dict): + # Exclude whole packages + exclude = {name: ['**/*'] for name in exclude} + if not isinstance(include, dict): + # Include whole packages + include = {name: ['**/*'] for name in include} + # But + for name in include: + # Exclude Python sources and byte-compiled code + if name not in exclude: + exclude[name] = [] + exclude[name].extend([ + '**/*.py', '**/*.pyc', + '**/__pycache__/**/*' + ]) + + packages_data = {} + processed_data = set() + # Bottom-up search for packages' data + for name in sorted(packages, reverse=True): + rootpath = where / name.replace('.', '/') + + # Exclude nested packages' content too + excluded_data = set(processed_data) + for patterns in fuzzy_lookup(name, exclude): + excluded_data.update( + path for pattern in patterns + for path in rootpath.glob(pattern) + ) + + included_data = set() + for patterns in fuzzy_lookup(name, include): + included_data.update( + path for pattern in patterns + for path in rootpath.glob(pattern) + if not path.is_dir() and path not in excluded_data + ) + + if included_data: + packages_data[name] = [ + str(path.relative_to(rootpath)) + for path in included_data + ] + + # Keep track of packages processed + processed_data.update(rootpath.glob('**/*')) + processed_data.add(rootpath) + + return packages_data diff --git a/ament_cmake_python/cmake/ament_python_install_package.cmake b/ament_cmake_python/cmake/ament_python_install_package.cmake index bba01ee7..b7d0bd2c 100644 --- a/ament_cmake_python/cmake/ament_python_install_package.cmake +++ b/ament_cmake_python/cmake/ament_python_install_package.cmake @@ -13,15 +13,22 @@ # limitations under the License. # -# Install a Python package (and its recursive subpackages). +# Install a Python package (and its recursive subpackages) as a flat Python egg # # :param package_name: the Python package name # :type package_name: string # :param PACKAGE_DIR: the path to the Python package directory (default: # folder relative to the CMAKE_CURRENT_LIST_DIR) # :type PACKAGE_DIR: string -# :param SKIP_COMPILE: if set do not compile the installed package +# :param VERSION: the Python package version (default: package.xml version) +# :param VERSION: string +# :param SETUP_CFG: the path to a setup.cfg file (default: +# setup.cfg file at CMAKE_CURRENT_LIST_DIR root, if any) +# :param SETUP_CFG: string +# :param SKIP_COMPILE: if set do not byte-compile the installed package # :type SKIP_COMPILE: option +# :param NO_DATA: if set do not install any package data +# :type NO_DATA: option # macro(ament_python_install_package) _ament_cmake_python_register_environment_hook() @@ -29,7 +36,7 @@ macro(ament_python_install_package) endmacro() function(_ament_cmake_python_install_package package_name) - cmake_parse_arguments(ARG "SKIP_COMPILE" "PACKAGE_DIR" "" ${ARGN}) + cmake_parse_arguments(ARG "SKIP_COMPILE;NO_DATA" "PACKAGE_DIR;VERSION;SETUP_CFG" "" ${ARGN}) if(ARG_UNPARSED_ARGUMENTS) message(FATAL_ERROR "ament_python_install_package() called with unused " "arguments: ${ARG_UNPARSED_ARGUMENTS}") @@ -42,34 +49,111 @@ function(_ament_cmake_python_install_package package_name) set(ARG_PACKAGE_DIR "${CMAKE_CURRENT_LIST_DIR}/${ARG_PACKAGE_DIR}") endif() + if(NOT ARG_VERSION) + # Use package.xml version + if(NOT _AMENT_PACKAGE_NAME) + ament_package_xml() + endif() + set(ARG_VERSION "${${PROJECT_NAME}_VERSION}") + endif() + if(NOT EXISTS "${ARG_PACKAGE_DIR}/__init__.py") message(FATAL_ERROR "ament_python_install_package() the Python package " "folder '${ARG_PACKAGE_DIR}' doesn't contain an '__init__.py' file") endif() - _ament_cmake_python_register_environment_hook() + if(NOT ARG_SETUP_CFG) + if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/setup.cfg") + set(ARG_SETUP_CFG "${CMAKE_CURRENT_LIST_DIR}/setup.cfg") + endif() + elseif(NOT IS_ABSOLUTE "${ARG_SETUP_CFG}") + set(ARG_SETUP_CFG "${CMAKE_CURRENT_LIST_DIR}/${ARG_SETUP_CFG}") + endif() + + set(build_dir "${CMAKE_CURRENT_BINARY_DIR}/ament_cmake_python/${package_name}") + file(RELATIVE_PATH source_dir "${build_dir}" "${ARG_PACKAGE_DIR}") - if(NOT PYTHON_INSTALL_DIR) - message(FATAL_ERROR "ament_python_install_package() variable " - "'PYTHON_INSTALL_DIR' must not be empty") + if(ARG_NO_DATA) + string(CONFIGURE "\ +import os +from setuptools import find_packages +from setuptools import setup + +setup( + name='${package_name}', + version='${ARG_VERSION}', + packages=find_packages( + where=os.path.normpath('${source_dir}/..'), + include=('${package_name}', '${package_name}.*')), + package_dir={'${package_name}': '${source_dir}'}, +) +" setup_py_content) + else() + string(CONFIGURE "\ +import os +from setuptools import find_packages +from setuptools import setup + +from ament_cmake_python import find_packages_data + +setup( + name='${package_name}', + version='${ARG_VERSION}', + packages=find_packages( + where=os.path.normpath('${source_dir}/..'), + include=('${package_name}', '${package_name}.*')), + package_dir={'${package_name}': '${source_dir}'}, + package_data=find_packages_data( + where=os.path.normpath('${source_dir}/..'), + include=('${package_name}', '${package_name}.*')) +) +" setup_py_content) endif() - install( - DIRECTORY "${ARG_PACKAGE_DIR}/" - DESTINATION "${PYTHON_INSTALL_DIR}/${package_name}" - PATTERN "*.pyc" EXCLUDE - PATTERN "__pycache__" EXCLUDE + + file(GENERATE + OUTPUT "${build_dir}/setup.py" + CONTENT "${setup_py_content}" ) - if(NOT ARG_SKIP_COMPILE) - # compile Python files - install(CODE - "execute_process( - COMMAND - \"${PYTHON_EXECUTABLE}\" \"-m\" \"compileall\" - \"${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_DIR}/${package_name}\" - )" + + if(ARG_SETUP_CFG) + add_custom_command( + OUTPUT "${build_dir}/setup.cfg" + COMMAND ${CMAKE_COMMAND} -E copy "${ARG_SETUP_CFG}" "${build_dir}/setup.cfg" + MAIN_DEPENDENCY "${ARG_SETUP_CFG}" + ) + add_custom_target(${package_name}_setup ALL + DEPENDS "${build_dir}/setup.cfg" ) endif() + if(NOT ARG_SKIP_COMPILE) + set(extra_install_args "--compile") + else() + set(extra_install_args "--no-compile") + endif() + + # Install as flat Python .egg to mimic https://github.com/colcon/colcon-core + # handling of pure Python packages. + file(RELATIVE_PATH install_dir "${build_dir}" "${CMAKE_INSTALL_PREFIX}") + + # NOTE(hidmic): Allow setup.py install to build, as there is no way to + # determine the Python package's source dependencies for proper build + # invalidation. + install(CODE + "message(STATUS \"Installing: ${package_name} as flat Python egg \" + \"to ${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_DIR}\") + execute_process( + COMMAND + \"${PYTHON_EXECUTABLE}\" setup.py install + --single-version-externally-managed + --prefix \"${install_dir}\" + --record install.log + ${extra_install_args} + WORKING_DIRECTORY \"${build_dir}\" + OUTPUT_QUIET + )" + ) + if(package_name IN_LIST AMENT_CMAKE_PYTHON_INSTALL_INSTALLED_NAMES) message(FATAL_ERROR "ament_python_install_package() a Python module file or package with " diff --git a/ament_cmake_python/test/data/baz/__init__.py b/ament_cmake_python/test/data/baz/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/baz/data b/ament_cmake_python/test/data/baz/data new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/baz/data.bin b/ament_cmake_python/test/data/baz/data.bin new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/foo/__init__.py b/ament_cmake_python/test/data/foo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/foo/bar/__init__.py b/ament_cmake_python/test/data/foo/bar/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/foo/bar/data.txt b/ament_cmake_python/test/data/foo/bar/data.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/foo/bar/resources/buzz.txt b/ament_cmake_python/test/data/foo/bar/resources/buzz.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/foo/bar/resources/fizz.txt b/ament_cmake_python/test/data/foo/bar/resources/fizz.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/foo/data b/ament_cmake_python/test/data/foo/data new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/foo/data.txt b/ament_cmake_python/test/data/foo/data.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/nested/pkgs/fizz/__init__.py b/ament_cmake_python/test/data/nested/pkgs/fizz/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/nested/pkgs/fizz/buzz/__init__.py b/ament_cmake_python/test/data/nested/pkgs/fizz/buzz/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/nested/pkgs/fizz/buzz/data.txt b/ament_cmake_python/test/data/nested/pkgs/fizz/buzz/data.txt new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/data/nested/pkgs/fizz/data/buzz.bin b/ament_cmake_python/test/data/nested/pkgs/fizz/data/buzz.bin new file mode 100644 index 00000000..e69de29b diff --git a/ament_cmake_python/test/test_find_packages_data.py b/ament_cmake_python/test/test_find_packages_data.py new file mode 100644 index 00000000..1a32b7fb --- /dev/null +++ b/ament_cmake_python/test/test_find_packages_data.py @@ -0,0 +1,84 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest + +from ament_cmake_python import find_packages_data + + +class TestFindPackagesData(unittest.TestCase): + + def test_all_packages_data_is_found(self): + data = find_packages_data() + assert set(data) == {'foo', 'foo.bar', 'baz'} + assert set(data['foo']) == {'data', 'data.txt'} + assert set(data['foo.bar']) == { + 'data.txt', + os.path.join('resources', 'fizz.txt'), + os.path.join('resources', 'buzz.txt') + } + assert set(data['baz']) == {'data.bin', 'data'} + + def test_whole_package_data_is_included(self): + data = find_packages_data( + include=('foo', 'foo.*')) + assert set(data) == {'foo', 'foo.bar'} + assert set(data['foo']) == {'data', 'data.txt'} + assert set(data['foo.bar']) == { + 'data.txt', + os.path.join('resources', 'fizz.txt'), + os.path.join('resources', 'buzz.txt') + } + + def test_whole_package_data_is_excluded(self): + data = find_packages_data( + include=('foo', 'foo.*'), + exclude=('foo.bar',)) + assert set(data) == {'foo'} + assert set(data['foo']) == {'data', 'data.txt'} + + def test_partial_package_data_is_excluded(self): + data = find_packages_data( + include=('foo', 'foo.*'), + exclude={'foo.bar': ['resources/*']}) + assert set(data) == {'foo', 'foo.bar'} + assert set(data['foo']) == {'data', 'data.txt'} + assert set(data['foo.bar']) == {'data.txt'} + + def test_partial_package_data_is_included(self): + data = find_packages_data( + include={ + 'foo': ['*.txt'], + 'foo.*': ['resources/*.txt'] + }, + ) + assert set(data) == {'foo', 'foo.bar'} + assert set(data['foo']) == {'data.txt'} + assert set(data['foo.bar']) == { + os.path.join('resources', 'fizz.txt'), + os.path.join('resources', 'buzz.txt') + } + + def test_nested_packages_data_is_found(self): + data = find_packages_data(where='nested/pkgs') + assert set(data) == {'fizz', 'fizz.buzz'} + assert set(data['fizz']) == { + os.path.join('data', 'buzz.bin') + } + assert set(data['fizz.buzz']) == {'data.txt'} + + +if __name__ == '__main__': + unittest.main()