Make ament_python_install_package() install a flat Python egg

ament_cmake_python/.gitignore

@@ -0,0 +1,3 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]

ament_cmake_python/CMakeLists.txt

@@ -4,6 +4,31 @@ project(ament_cmake_python NONE)
 
 find_package(ament_cmake_core REQUIRED)
 
+set(ament_cmake_python_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+include("ament_cmake_python-extras.cmake")
+ament_python_install_package(${PROJECT_NAME} NO_DATA)
+
+include(CTest)
+
+if(BUILD_TESTING)
+  execute_process(
+    COMMAND "${PYTHON_EXECUTABLE}" -m py_compile
+      foo/__init__.py foo/bar/__init__.py baz/__init__.py
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/data"
+  )
+  add_test(
+    NAME test_find_packages_data
+    COMMAND
+      "${PYTHON_EXECUTABLE}"
+      "${CMAKE_CURRENT_SOURCE_DIR}/test/test_find_packages_data.py"
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/data"
+  )
+  set_tests_properties(test_find_packages_data PROPERTIES
+    ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}"
+    TIMEOUT 30
+  )
+endif()
+
 ament_package(
   CONFIG_EXTRAS "ament_cmake_python-extras.cmake"
 )

ament_cmake_python/ament_cmake_python/__init__.py

@@ -0,0 +1,102 @@
+# Copyright 2021 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fnmatch
+import pathlib
+from urllib.parse import urlparse
+
+from setuptools import find_packages
+
+
+def fuzzy_lookup(key, mapping):
+    """Lookup key in a mapping where keys may contain wildcards ('*')."""
+    for pattern, value in mapping.items():
+        if fnmatch.fnmatch(key, pattern):
+            yield value
+
+
+def find_packages_data(where='.', exclude=(), include=('*',)):
+    """
+    Find data in Python packages found within directory 'where'.
+
+    Similar to `setuptools.find_packages`.
+
+    :param where: a cross-platform (i.e. URL-style) path
+    :param exclude: a dictionary that maps from package names to
+      lists of glob patterns to be excluded from the search. Wildcards
+      ('*') may be used in package names. A collection of package names
+      may be provided instead of a dictionary, in which case whole packages
+      will be excluded.
+    :param include: a dictionary that maps from package names to
+      lists of glob patterns to be included in the search. Wildcards
+      ('*') may be used in package names. A collection of package names
+      may be provided instead of a dictionary, in which case whole packages
+      will be included but excluding Python sources and byte-compiled code.
+    :returns: a dictionary suitable to be used as 'package_data' when calling
+      `setuptools.setup`
+    """
+    packages = find_packages(
+        where=where, include=set(include),
+        # Defer package exclusion (may be partial)
+    )
+    where = pathlib.Path(urlparse(where).path)
+    if not isinstance(exclude, dict):
+        # Exclude whole packages
+        exclude = {name: ['**/*'] for name in exclude}
+    if not isinstance(include, dict):
+        # Include whole packages
+        include = {name: ['**/*'] for name in include}
+        # But
+        for name in include:
+            # Exclude Python sources and byte-compiled code
+            if name not in exclude:
+                exclude[name] = []
+            exclude[name].extend([
+                '**/*.py', '**/*.pyc',
+                '**/__pycache__/**/*'
+            ])
+
+    packages_data = {}
+    processed_data = set()
+    # Bottom-up search for packages' data
+    for name in sorted(packages, reverse=True):
+        rootpath = where / name.replace('.', '/')
+
+        # Exclude nested packages' content too
+        excluded_data = set(processed_data)
+        for patterns in fuzzy_lookup(name, exclude):
+            excluded_data.update(
+                path for pattern in patterns
+                for path in rootpath.glob(pattern)
+            )
+
+        included_data = set()
+        for patterns in fuzzy_lookup(name, include):
+            included_data.update(
+                path for pattern in patterns
+                for path in rootpath.glob(pattern)
+                if not path.is_dir() and path not in excluded_data
+            )
+
+        if included_data:
+            packages_data[name] = [
+                str(path.relative_to(rootpath))
+                for path in included_data
+            ]
+
+        # Keep track of packages processed
+        processed_data.update(rootpath.glob('**/*'))
+        processed_data.add(rootpath)
+
+    return packages_data

ament_cmake_python/cmake/ament_python_install_package.cmake

@@ -13,23 +13,30 @@
 # limitations under the License.
 
 #
-# Install a Python package (and its recursive subpackages).
+# Install a Python package (and its recursive subpackages) as a flat Python egg
 #
 # :param package_name: the Python package name
 # :type package_name: string
 # :param PACKAGE_DIR: the path to the Python package directory (default:
 #   <package_name> folder relative to the CMAKE_CURRENT_LIST_DIR)
 # :type PACKAGE_DIR: string
-# :param SKIP_COMPILE: if set do not compile the installed package
+# :param VERSION: the Python package version (default: package.xml version)
+# :param VERSION: string
+# :param SETUP_CFG: the path to a setup.cfg file (default:
+#   setup.cfg file at CMAKE_CURRENT_LIST_DIR root, if any)
+# :param SETUP_CFG: string
+# :param SKIP_COMPILE: if set do not byte-compile the installed package
 # :type SKIP_COMPILE: option
+# :param NO_DATA: if set do not install any package data
+# :type NO_DATA: option
 #
 macro(ament_python_install_package)
   _ament_cmake_python_register_environment_hook()
   _ament_cmake_python_install_package(${ARGN})
 endmacro()
 
 function(_ament_cmake_python_install_package package_name)
-  cmake_parse_arguments(ARG "SKIP_COMPILE" "PACKAGE_DIR" "" ${ARGN})
+  cmake_parse_arguments(ARG "SKIP_COMPILE;NO_DATA" "PACKAGE_DIR;VERSION;SETUP_CFG" "" ${ARGN})
   if(ARG_UNPARSED_ARGUMENTS)
     message(FATAL_ERROR "ament_python_install_package() called with unused "
       "arguments: ${ARG_UNPARSED_ARGUMENTS}")
@@ -42,34 +49,111 @@ function(_ament_cmake_python_install_package package_name)
     set(ARG_PACKAGE_DIR "${CMAKE_CURRENT_LIST_DIR}/${ARG_PACKAGE_DIR}")
   endif()
 
+  if(NOT ARG_VERSION)
+    # Use package.xml version
+    if(NOT _AMENT_PACKAGE_NAME)
+      ament_package_xml()
+    endif()
+    set(ARG_VERSION "${${PROJECT_NAME}_VERSION}")
+  endif()
+
   if(NOT EXISTS "${ARG_PACKAGE_DIR}/__init__.py")
     message(FATAL_ERROR "ament_python_install_package() the Python package "
       "folder '${ARG_PACKAGE_DIR}' doesn't contain an '__init__.py' file")
   endif()
 
-  _ament_cmake_python_register_environment_hook()
+  if(NOT ARG_SETUP_CFG)
+    if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/setup.cfg")
+      set(ARG_SETUP_CFG "${CMAKE_CURRENT_LIST_DIR}/setup.cfg")
+    endif()
+  elseif(NOT IS_ABSOLUTE "${ARG_SETUP_CFG}")
+    set(ARG_SETUP_CFG "${CMAKE_CURRENT_LIST_DIR}/${ARG_SETUP_CFG}")
+  endif()
+
+  set(build_dir "${CMAKE_CURRENT_BINARY_DIR}/ament_cmake_python/${package_name}")
+  file(RELATIVE_PATH source_dir "${build_dir}" "${ARG_PACKAGE_DIR}")
 
-  if(NOT PYTHON_INSTALL_DIR)
-    message(FATAL_ERROR "ament_python_install_package() variable "
-      "'PYTHON_INSTALL_DIR' must not be empty")
+  if(ARG_NO_DATA)
+    string(CONFIGURE "\
+import os
+from setuptools import find_packages
+from setuptools import setup
+
+setup(
+    name='${package_name}',
+    version='${ARG_VERSION}',
+    packages=find_packages(
+        where=os.path.normpath('${source_dir}/..'),
+        include=('${package_name}', '${package_name}.*')),
+    package_dir={'${package_name}': '${source_dir}'},
+)
+" setup_py_content)
+  else()
+    string(CONFIGURE "\
+import os
+from setuptools import find_packages
+from setuptools import setup
+
+from ament_cmake_python import find_packages_data
+
+setup(
+    name='${package_name}',
+    version='${ARG_VERSION}',
+    packages=find_packages(
+        where=os.path.normpath('${source_dir}/..'),
+        include=('${package_name}', '${package_name}.*')),
+    package_dir={'${package_name}': '${source_dir}'},
+    package_data=find_packages_data(
+        where=os.path.normpath('${source_dir}/..'),
+        include=('${package_name}', '${package_name}.*'))
+)
+" setup_py_content)
   endif()
-  install(
-    DIRECTORY "${ARG_PACKAGE_DIR}/"
-    DESTINATION "${PYTHON_INSTALL_DIR}/${package_name}"
-    PATTERN "*.pyc" EXCLUDE
-    PATTERN "__pycache__" EXCLUDE
+
+  file(GENERATE
+    OUTPUT "${build_dir}/setup.py"
+    CONTENT "${setup_py_content}"
   )
-  if(NOT ARG_SKIP_COMPILE)
-    # compile Python files
-    install(CODE
-      "execute_process(
-        COMMAND
-        \"${PYTHON_EXECUTABLE}\" \"-m\" \"compileall\"
-        \"${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_DIR}/${package_name}\"
-      )"
+
+  if(ARG_SETUP_CFG)
+    add_custom_command(
+      OUTPUT "${build_dir}/setup.cfg"
+      COMMAND ${CMAKE_COMMAND} -E copy "${ARG_SETUP_CFG}" "${build_dir}/setup.cfg"
+      MAIN_DEPENDENCY "${ARG_SETUP_CFG}"
+    )
+    add_custom_target(${package_name}_setup ALL
+      DEPENDS "${build_dir}/setup.cfg"
     )
   endif()
 
+  if(NOT ARG_SKIP_COMPILE)
+    set(extra_install_args "--compile")
+  else()
+    set(extra_install_args "--no-compile")
+  endif()
+
+  # Install as flat Python .egg to mimic https://github.com/colcon/colcon-core
+  # handling of pure Python packages.
+  file(RELATIVE_PATH install_dir "${build_dir}" "${CMAKE_INSTALL_PREFIX}")
+
+  # NOTE(hidmic): Allow setup.py install to build, as there is no way to
+  # determine the Python package's source dependencies for proper build
+  # invalidation.
+  install(CODE
+    "message(STATUS \"Installing: ${package_name} as flat Python egg \"
+                    \"to ${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_DIR}\")
+     execute_process(
+        COMMAND
+        \"${PYTHON_EXECUTABLE}\" setup.py install
+           --single-version-externally-managed
+           --prefix \"${install_dir}\"
+           --record install.log
+           ${extra_install_args}
+        WORKING_DIRECTORY \"${build_dir}\"
+        OUTPUT_QUIET
+     )"
+  )
+
   if(package_name IN_LIST AMENT_CMAKE_PYTHON_INSTALL_INSTALLED_NAMES)
     message(FATAL_ERROR
       "ament_python_install_package() a Python module file or package with "

ament_cmake_python/test/test_find_packages_data.py

@@ -0,0 +1,84 @@
+# Copyright 2021 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+from ament_cmake_python import find_packages_data
+
+
+class TestFindPackagesData(unittest.TestCase):
+
+    def test_all_packages_data_is_found(self):
+        data = find_packages_data()
+        assert set(data) == {'foo', 'foo.bar', 'baz'}
+        assert set(data['foo']) == {'data', 'data.txt'}
+        assert set(data['foo.bar']) == {
+            'data.txt',
+            os.path.join('resources', 'fizz.txt'),
+            os.path.join('resources', 'buzz.txt')
+        }
+        assert set(data['baz']) == {'data.bin', 'data'}
+
+    def test_whole_package_data_is_included(self):
+        data = find_packages_data(
+            include=('foo', 'foo.*'))
+        assert set(data) == {'foo', 'foo.bar'}
+        assert set(data['foo']) == {'data', 'data.txt'}
+        assert set(data['foo.bar']) == {
+            'data.txt',
+            os.path.join('resources', 'fizz.txt'),
+            os.path.join('resources', 'buzz.txt')
+        }
+
+    def test_whole_package_data_is_excluded(self):
+        data = find_packages_data(
+            include=('foo', 'foo.*'),
+            exclude=('foo.bar',))
+        assert set(data) == {'foo'}
+        assert set(data['foo']) == {'data', 'data.txt'}
+
+    def test_partial_package_data_is_excluded(self):
+        data = find_packages_data(
+            include=('foo', 'foo.*'),
+            exclude={'foo.bar': ['resources/*']})
+        assert set(data) == {'foo', 'foo.bar'}
+        assert set(data['foo']) == {'data', 'data.txt'}
+        assert set(data['foo.bar']) == {'data.txt'}
+
+    def test_partial_package_data_is_included(self):
+        data = find_packages_data(
+            include={
+                'foo': ['*.txt'],
+                'foo.*': ['resources/*.txt']
+            },
+        )
+        assert set(data) == {'foo', 'foo.bar'}
+        assert set(data['foo']) == {'data.txt'}
+        assert set(data['foo.bar']) == {
+            os.path.join('resources', 'fizz.txt'),
+            os.path.join('resources', 'buzz.txt')
+        }
+
+    def test_nested_packages_data_is_found(self):
+        data = find_packages_data(where='nested/pkgs')
+        assert set(data) == {'fizz', 'fizz.buzz'}
+        assert set(data['fizz']) == {
+            os.path.join('data', 'buzz.bin')
+        }
+        assert set(data['fizz.buzz']) == {'data.txt'}
+
+
+if __name__ == '__main__':
+    unittest.main()