From f2da32f6e9c7cc7d9185adb071bee01daa5cfcb1 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:08:45 +0100 Subject: [PATCH 1/7] Rely on get_outputs() and get_output_mapping() in editable_wheel --- setuptools/command/editable_wheel.py | 374 ++++++++++++++++------ setuptools/tests/test_editable_install.py | 30 +- 2 files changed, 292 insertions(+), 112 deletions(-) diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 2776577f44..b02486d0fc 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -7,24 +7,39 @@ One of the mechanisms briefly mentioned in PEP 660 to implement editable installs is to create a separated directory inside ``build`` and use a .pth file to point to that directory. In the context of this file such directory is referred as - *auxiliary build directory* or ``auxiliary_build_dir``. + *auxiliary build directory* or ``auxiliary_dir``. """ +import logging import os import re import shutil import sys -import logging import warnings +from contextlib import suppress from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, Iterable, Iterator, List, Mapping, Union, Tuple, TypeVar - -from setuptools import Command, namespaces +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, + TypeVar, + Union +) + +from setuptools import Command, errors, namespaces from setuptools.discovery import find_package_path from setuptools.dist import Distribution +if TYPE_CHECKING: + from wheel.wheelfile import WheelFile # noqa + _Path = Union[str, Path] _P = TypeVar("_P", bound=_Path) _logger = logging.getLogger(__name__) @@ -64,9 +79,9 @@ def finalize_options(self): self.project_dir = dist.src_root or os.curdir self.package_dir = dist.package_dir or {} self.dist_dir = Path(self.dist_dir or os.path.join(self.project_dir, "dist")) - self.dist_dir.mkdir(exist_ok=True) def run(self): + self.dist_dir.mkdir(exist_ok=True) self._ensure_dist_info() # Add missing dist_info files @@ -96,6 +111,140 @@ def _install_namespaces(self, installation_dir, pth_prefix): installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, src_root) installer.install_namespaces() + def _find_egg_info_dir(self) -> Optional[str]: + parent_dir = Path(self.dist_info_dir).parent if self.dist_info_dir else Path() + candidates = map(str, parent_dir.glob("*.egg-info")) + return next(candidates, None) + + def _configure_build( + self, name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path + ): + """Configure commands to behave in the following ways: + + - Build commands can write to ``build_lib`` if they really want to... + (but this folder is expected to be ignored and modules are expected to live + in the project directory...) + - Binary extensions should be built in-place (editable_mode = True) + - Data/header/script files are not part of the "editable" specification + so they are written directly to the unpacked_wheel directory. + """ + # Non-editable files (data, headers, scripts) are written directly to the + # unpacked_wheel + + dist = self.distribution + wheel = str(unpacked_wheel) + build_lib = str(build_lib) + data = str(Path(unpacked_wheel, f"{name}.data", "data")) + headers = str(Path(unpacked_wheel, f"{name}.data", "include")) + scripts = str(Path(unpacked_wheel, f"{name}.data", "scripts")) + + # egg-info may be generated again to create a manifest (used for package data) + egg_info = dist.reinitialize_command("egg_info", reinit_subcommands=True) + egg_info.egg_base = str(tmp_dir) + egg_info.ignore_egg_info_in_manifest = True + + build = dist.reinitialize_command("build", reinit_subcommands=True) + install = dist.reinitialize_command("install", reinit_subcommands=True) + + build.build_platlib = build.build_purelib = build.build_lib = build_lib + install.install_purelib = install.install_platlib = install.install_lib = wheel + install.install_scripts = build.build_scripts = scripts + install.install_headers = headers + install.install_data = data + + install_scripts = dist.get_command_obj("install_scripts") + install_scripts.no_ep = True + + build.build_temp = str(tmp_dir) + + build_py = dist.get_command_obj("build_py") + build_py.compile = False + build_py.existing_egg_info_dir = self._find_egg_info_dir() + + self._set_editable_mode() + + build.ensure_finalized() + install.ensure_finalized() + + def _set_editable_mode(self): + """Set the ``editable_mode`` flag in the build sub-commands""" + dist = self.distribution + build = dist.get_command_obj("build") + for cmd_name in build.get_sub_commands(): + cmd = dist.get_command_obj(cmd_name) + if hasattr(cmd, "editable_mode"): + cmd.editable_mode = True + + def _find_existing_source(self, file: Path) -> Optional[Path]: + """Given a file path relative to ``build_lib`` try to guess + what would be its original source file. + """ + dist = self.distribution + package = str(file.parent).replace(os.sep, ".") + package_path = find_package_path(package, dist.package_dir, self.project_dir) + candidate = Path(package_path, file.name) + return candidate if candidate.exists() else None + + def _collect_reminiscent_outputs( + self, build_lib: _Path + ) -> Tuple[List[str], Dict[str, str]]: + """People have been overwriting setuptools for a long time, and not everyone + might be respecting the new ``get_output_mapping`` APIs, so we have to do our + best to handle this scenario. + """ + files: List[str] = [] + mapping: Dict[str, str] = {} + + for dirpath, _dirnames, filenames in os.walk(build_lib): + for name in filenames: + file = Path(dirpath, name) + source = self._find_existing_source(file.relative_to(build_lib)) + if source: + mapping[str(file)] = str(source) + else: + files.append(str(file)) + + return files, mapping + + def _combine_outputs(self, *outputs: List[str]) -> List[str]: + return sorted({os.path.normpath(f) for f in chain.from_iterable(outputs)}) + + def _combine_output_mapping(self, *mappings: Dict[str, str]) -> Dict[str, str]: + mapping = ( + (os.path.normpath(k), os.path.normpath(v)) + for k, v in chain.from_iterable(m.items() for m in mappings) + ) + return dict(sorted(mapping)) + + def _collect_build_outputs(self) -> Tuple[List[str], Dict[str, str]]: + files: List[str] = [] + mapping: Dict[str, str] = {} + build = self.get_finalized_command("build") + + for cmd_name in build.get_sub_commands(): + cmd = self.get_finalized_command(cmd_name) + if hasattr(cmd, "get_outputs"): + files.extend(cmd.get_outputs() or []) + if hasattr(cmd, "get_output_mapping"): + mapping.update(cmd.get_output_mapping() or {}) + rfiles, rmapping = self._collect_reminiscent_outputs(build.build_lib) + + return ( + self._combine_outputs(files, rfiles), + self._combine_output_mapping(mapping, rmapping), + ) + + def _run_build_commands( + self, dist_name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path + ) -> Tuple[List[str], Dict[str, str]]: + self._configure_build(dist_name, unpacked_wheel, build_lib, tmp_dir) + self.run_command("build") + files, mapping = self._collect_build_outputs() + self._run_install("headers") + self._run_install("scripts") + self._run_install("data") + return files, mapping + def _create_wheel_file(self, bdist_wheel): from wheel.wheelfile import WheelFile @@ -110,22 +259,19 @@ def _create_wheel_file(self, bdist_wheel): # Currently the wheel API receives a directory and dump all its contents # inside of a wheel. So let's use a temporary directory. - unpacked_tmp = TemporaryDirectory(suffix=archive_name) + unpacked_wheel = TemporaryDirectory(suffix=archive_name) + build_lib = TemporaryDirectory(suffix=".build-lib") build_tmp = TemporaryDirectory(suffix=".build-temp") - with unpacked_tmp as unpacked, build_tmp as tmp: + with unpacked_wheel as unpacked, build_lib as lib, build_tmp as tmp: unpacked_dist_info = Path(unpacked, Path(self.dist_info_dir).name) shutil.copytree(self.dist_info_dir, unpacked_dist_info) self._install_namespaces(unpacked, dist_info.name) - - # Add non-editable files to the wheel - _configure_build(dist_name, self.distribution, unpacked, tmp) - self._run_install("headers") - self._run_install("scripts") - self._run_install("data") - - self._populate_wheel(dist_info.name, tag, unpacked, tmp) + files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp) with WheelFile(wheel_path, "w") as wf: + self._populate_wheel( + wf, dist_info.name, tag, unpacked, lib, tmp, files, mapping + ) wf.write_files(unpacked) return wheel_path @@ -136,13 +282,25 @@ def _run_install(self, category: str): _logger.info(f"Installing {category} as non editable") self.run_command(f"install_{category}") - def _populate_wheel(self, name: str, tag: str, unpacked_dir: Path, tmp: _Path): + def _populate_wheel( + self, + wheel: "WheelFile", + name: str, + tag: str, + unpacked_dir: Path, + build_lib: _Path, + tmp: _Path, + outputs: List[str], + output_mapping: Dict[str, str], + ): """Decides which strategy to use to implement an editable installation.""" build_name = f"__editable__.{name}-{tag}" project_dir = Path(self.project_dir) if self.strict or os.getenv("SETUPTOOLS_EDITABLE", None) == "strict": - return self._populate_link_tree(name, build_name, unpacked_dir, tmp) + return self._populate_link_tree( + name, build_name, wheel, build_lib, outputs, output_mapping + ) # Build extensions in-place self.reinitialize_command("build_ext", inplace=1) @@ -152,44 +310,57 @@ def _populate_wheel(self, name: str, tag: str, unpacked_dir: Path, tmp: _Path): has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) if set(self.package_dir) == {""} and has_simple_layout: # src-layout(ish) is relatively safe for a simple pth file - return self._populate_static_pth(name, project_dir, unpacked_dir) + return self._populate_static_pth(name, project_dir, wheel) # Use a MetaPathFinder to avoid adding accidental top-level packages/modules - self._populate_finder(name, unpacked_dir) + self._populate_finder(name, wheel) def _populate_link_tree( - self, name: str, build_name: str, unpacked_dir: Path, tmp: _Path + self, + name: str, + build_name: str, + wheel: "WheelFile", + build_lib: _Path, + outputs: List[str], + output_mapping: Dict[str, str], ): """Populate wheel using the "strict" ``link tree`` strategy.""" msg = "Strict editable install will be performed using a link tree.\n" _logger.warning(msg + _STRICT_WARNING) - auxiliary_build_dir = _empty_dir(Path(self.project_dir, "build", build_name)) - populate = _LinkTree(self.distribution, name, auxiliary_build_dir, tmp) - populate(unpacked_dir) + auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) + populate = _LinkTree( + self.distribution, + name, + auxiliary_dir, + build_lib, + outputs, + output_mapping, + ) + populate(wheel) msg = f"""\n Strict editable installation performed using the auxiliary directory: - {auxiliary_build_dir} + {auxiliary_dir} Please be careful to not remove this directory, otherwise you might not be able to import/use your package. """ warnings.warn(msg, InformationOnly) - def _populate_static_pth(self, name: str, project_dir: Path, unpacked_dir: Path): + def _populate_static_pth(self, name: str, project_dir: Path, wheel: "WheelFile"): """Populate wheel using the "lax" ``.pth`` file strategy, for ``src-layout``.""" src_dir = self.package_dir[""] msg = f"Editable install will be performed using .pth file to {src_dir}.\n" _logger.warning(msg + _LAX_WARNING) populate = _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) - populate(unpacked_dir) + populate(wheel) - def _populate_finder(self, name: str, unpacked_dir: Path): + def _populate_finder(self, name: str, wheel: "WheelFile"): """Populate wheel using the "lax" MetaPathFinder strategy.""" msg = "Editable install will be performed using a meta path finder.\n" _logger.warning(msg + _LAX_WARNING) populate = _TopLevelFinder(self.distribution, name) - populate(unpacked_dir) + populate(wheel) class _StaticPth: @@ -198,53 +369,69 @@ def __init__(self, dist: Distribution, name: str, path_entries: List[Path]): self.name = name self.path_entries = path_entries - def __call__(self, unpacked_wheel_dir: Path): - pth = Path(unpacked_wheel_dir, f"__editable__.{self.name}.pth") + def __call__(self, wheel: "WheelFile"): entries = "\n".join((str(p.resolve()) for p in self.path_entries)) - pth.write_text(f"{entries}\n", encoding="utf-8") + contents = bytes(f"{entries}\n", "utf-8") + wheel.writestr(f"__editable__.{self.name}.pth", contents) class _LinkTree(_StaticPth): """ - Creates a ``.pth`` file that points to a link tree in the ``auxiliary_build_dir``. + Creates a ``.pth`` file that points to a link tree in the ``auxiliary_dir``. This strategy will only link files (not dirs), so it can be implemented in any OS, even if that means using hardlinks instead of symlinks. - By collocating ``auxiliary_build_dir`` and the original source code, limitations + By collocating ``auxiliary_dir`` and the original source code, limitations with hardlinks should be avoided. """ def __init__( - self, dist: Distribution, name: str, auxiliary_build_dir: Path, tmp: _Path + self, dist: Distribution, + name: str, + auxiliary_dir: _Path, + build_lib: _Path, + outputs: List[str], + output_mapping: Dict[str, str], ): - super().__init__(dist, name, [auxiliary_build_dir]) - self.auxiliary_build_dir = auxiliary_build_dir - self.tmp = tmp + self.auxiliary_dir = Path(auxiliary_dir) + self.build_lib = Path(build_lib).resolve() + self.outputs = outputs + self.output_mapping = output_mapping + self._file = dist.get_command_obj("build_py").copy_file + super().__init__(dist, name, [self.auxiliary_dir]) + + def __call__(self, wheel: "WheelFile"): + self._create_links() + super().__call__(wheel) + + def _normalize_output(self, file: str) -> Optional[str]: + # Files relative to build_lib will be normalized to None + with suppress(ValueError): + path = Path(file).resolve().relative_to(self.build_lib) + return str(path).replace(os.sep, '/') + return None - def _build_py(self): - if not self.dist.has_pure_modules(): - return + def _create_file(self, relative_output: str, src_file: str, link=None): + dest = self.auxiliary_dir / relative_output + if not dest.parent.is_dir(): + dest.parent.mkdir(parents=True) + self._file(src_file, dest, link=link) - build_py = self.dist.get_command_obj("build_py") - build_py.ensure_finalized() - # Force build_py to use links instead of copying files - build_py.use_links = "sym" if _can_symlink_files() else "hard" - build_py.run() + def _create_links(self): + link_type = "sym" if _can_symlink_files() else "hard" + mappings = { + self._normalize_output(k): v + for k, v in self.output_mapping.items() + } + mappings.pop(None, None) # remove files that are not relative to build_lib - def _build_ext(self): - if not self.dist.has_ext_modules(): - return + for output in self.outputs: + relative = self._normalize_output(output) + if relative and relative not in mappings: + self._create_file(relative, output) - build_ext = self.dist.get_command_obj("build_ext") - build_ext.ensure_finalized() - # Extensions are not editable, so we just have to build them in the right dir - build_ext.run() - - def __call__(self, unpacked_wheel_dir: Path): - _configure_build(self.name, self.dist, self.auxiliary_build_dir, self.tmp) - self._build_py() - self._build_ext() - super().__call__(unpacked_wheel_dir) + for relative, src in mappings.items(): + self._create_file(relative, src, link=link_type) class _TopLevelFinder: @@ -252,7 +439,7 @@ def __init__(self, dist: Distribution, name: str): self.dist = dist self.name = name - def __call__(self, unpacked_wheel_dir: Path): + def __call__(self, wheel: "WheelFile"): src_root = self.dist.src_root or os.curdir top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) package_dir = self.dist.package_dir or {} @@ -265,51 +452,30 @@ def __call__(self, unpacked_wheel_dir: Path): name = f"__editable__.{self.name}.finder" finder = _make_identifier(name) - content = _finder_template(name, roots, namespaces_) - Path(unpacked_wheel_dir, f"{finder}.py").write_text(content, encoding="utf-8") - - pth = f"__editable__.{self.name}.pth" - content = f"import {finder}; {finder}.install()" - Path(unpacked_wheel_dir, pth).write_text(content, encoding="utf-8") + content = bytes(_finder_template(name, roots, namespaces_), "utf-8") + wheel.writestr(f"{finder}.py", content) + content = bytes(f"import {finder}; {finder}.install()", "utf-8") + wheel.writestr(f"__editable__.{self.name}.pth", content) -def _configure_build(name: str, dist: Distribution, target_dir: _Path, tmp_dir: _Path): - target = str(target_dir) - data = str(Path(target_dir, f"{name}.data", "data")) - headers = str(Path(target_dir, f"{name}.data", "include")) - scripts = str(Path(target_dir, f"{name}.data", "scripts")) - # egg-info will be generated again to create a manifest (used for package data) - egg_info = dist.reinitialize_command("egg_info", reinit_subcommands=True) - egg_info.egg_base = str(tmp_dir) - egg_info.ignore_egg_info_in_manifest = True - - build = dist.reinitialize_command("build", reinit_subcommands=True) - install = dist.reinitialize_command("install", reinit_subcommands=True) - - build.build_platlib = build.build_purelib = build.build_lib = target - install.install_purelib = install.install_platlib = install.install_lib = target - install.install_scripts = build.build_scripts = scripts - install.install_headers = headers - install.install_data = data - - build.build_temp = str(tmp_dir) - - build_py = dist.get_command_obj("build_py") - build_py.compile = False - - build.ensure_finalized() - install.ensure_finalized() - - -def _can_symlink_files(): - try: - with TemporaryDirectory() as tmp: - path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt") - path1.write_text("file1", encoding="utf-8") +def _can_symlink_files() -> bool: + with TemporaryDirectory() as tmp: + path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt") + path1.write_text("file1", encoding="utf-8") + with suppress(AttributeError, NotImplementedError, OSError): os.symlink(path1, path2) - return path2.is_symlink() and path2.read_text(encoding="utf-8") == "file1" - except (AttributeError, NotImplementedError, OSError): + if path2.is_symlink() and path2.read_text(encoding="utf-8") == "file1": + return True + + try: + os.link(path1, path2) # Ensure hard links can be created + except Exception as ex: + msg = ( + "File system does not seem to support either symlinks or hard links. " + "Strict editable installs require one of them to be supported." + ) + raise LinksNotSupported(msg) from ex return False @@ -336,6 +502,8 @@ def _simple_layout( False >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a.a1.a2": "_a2"}, ".") False + >>> _simple_layout(['a', 'a.b'], {"": "src", "a.b": "_ab"}, "/tmp/myproj") + False """ layout = { pkg: find_package_path(pkg, package_dir, project_dir) @@ -601,3 +769,7 @@ class InformationOnly(UserWarning): The only thing that might work is a warning, although it is not the most appropriate tool for the job... """ + + +class LinksNotSupported(errors.FileError): + """File system does not seem to support either symlinks or hard links.""" diff --git a/setuptools/tests/test_editable_install.py b/setuptools/tests/test_editable_install.py index 6c951c7940..faf614fd81 100644 --- a/setuptools/tests/test_editable_install.py +++ b/setuptools/tests/test_editable_install.py @@ -7,6 +7,7 @@ from importlib import import_module from pathlib import Path from textwrap import dedent +from unittest.mock import Mock from uuid import uuid4 import jaraco.envs @@ -589,24 +590,31 @@ def test_generated_tree(self, tmp_path): dist = Distribution({"script_name": "%PEP 517%"}) dist.parse_config_files() + wheel = Mock() + aux = tmp_path / ".aux" build = tmp_path / ".build" - tmp = tmp_path / ".tmp" - tmp.mkdir() - unpacked = tmp_path / ".unpacked" - unpacked.mkdir() + aux.mkdir() + build.mkdir() - make_tree = _LinkTree(dist, name, build, tmp) - make_tree(unpacked) + build_py = dist.get_command_obj("build_py") + build_py.editable_mode = True + build_py.build_lib = str(build) + build_py.ensure_finalized() + outputs = build_py.get_outputs() + output_mapping = build_py.get_output_mapping() - mod1 = next(build.glob("**/mod1.py")) + make_tree = _LinkTree(dist, name, aux, build, outputs, output_mapping) + make_tree(wheel) + + mod1 = next(aux.glob("**/mod1.py")) expected = tmp_path / "src/mypkg/mod1.py" assert_link_to(mod1, expected) - assert next(build.glob("**/subpackage"), None) is None - assert next(build.glob("**/mod2.py"), None) is None - assert next(build.glob("**/resource_file.txt"), None) is None + assert next(aux.glob("**/subpackage"), None) is None + assert next(aux.glob("**/mod2.py"), None) is None + assert next(aux.glob("**/resource_file.txt"), None) is None - assert next(build.glob("**/resource.not_in_manifest"), None) is None + assert next(aux.glob("**/resource.not_in_manifest"), None) is None def test_strict_install(self, tmp_path, venv, monkeypatch): monkeypatch.setenv("SETUPTOOLS_EDITABLE", "strict") From 7a9bc7601925b2e5743c8d821d2d1ef82277c22a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:12:45 +0100 Subject: [PATCH 2/7] Remove unnecessary complexity --- setuptools/command/editable_wheel.py | 47 +--------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index b02486d0fc..c1726efb71 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -175,47 +175,6 @@ def _set_editable_mode(self): if hasattr(cmd, "editable_mode"): cmd.editable_mode = True - def _find_existing_source(self, file: Path) -> Optional[Path]: - """Given a file path relative to ``build_lib`` try to guess - what would be its original source file. - """ - dist = self.distribution - package = str(file.parent).replace(os.sep, ".") - package_path = find_package_path(package, dist.package_dir, self.project_dir) - candidate = Path(package_path, file.name) - return candidate if candidate.exists() else None - - def _collect_reminiscent_outputs( - self, build_lib: _Path - ) -> Tuple[List[str], Dict[str, str]]: - """People have been overwriting setuptools for a long time, and not everyone - might be respecting the new ``get_output_mapping`` APIs, so we have to do our - best to handle this scenario. - """ - files: List[str] = [] - mapping: Dict[str, str] = {} - - for dirpath, _dirnames, filenames in os.walk(build_lib): - for name in filenames: - file = Path(dirpath, name) - source = self._find_existing_source(file.relative_to(build_lib)) - if source: - mapping[str(file)] = str(source) - else: - files.append(str(file)) - - return files, mapping - - def _combine_outputs(self, *outputs: List[str]) -> List[str]: - return sorted({os.path.normpath(f) for f in chain.from_iterable(outputs)}) - - def _combine_output_mapping(self, *mappings: Dict[str, str]) -> Dict[str, str]: - mapping = ( - (os.path.normpath(k), os.path.normpath(v)) - for k, v in chain.from_iterable(m.items() for m in mappings) - ) - return dict(sorted(mapping)) - def _collect_build_outputs(self) -> Tuple[List[str], Dict[str, str]]: files: List[str] = [] mapping: Dict[str, str] = {} @@ -227,12 +186,8 @@ def _collect_build_outputs(self) -> Tuple[List[str], Dict[str, str]]: files.extend(cmd.get_outputs() or []) if hasattr(cmd, "get_output_mapping"): mapping.update(cmd.get_output_mapping() or {}) - rfiles, rmapping = self._collect_reminiscent_outputs(build.build_lib) - return ( - self._combine_outputs(files, rfiles), - self._combine_output_mapping(mapping, rmapping), - ) + return files, mapping def _run_build_commands( self, dist_name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path From 5b7498ba18599b0ecfea2a6541ec4a64ae8c2bae Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:30:30 +0100 Subject: [PATCH 3/7] editable_wheel: Move warnings/logging inside the strategy classes --- setuptools/command/editable_wheel.py | 58 +++++++++++++++++++--------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index c1726efb71..6cfdefddc6 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -280,8 +280,6 @@ def _populate_link_tree( output_mapping: Dict[str, str], ): """Populate wheel using the "strict" ``link tree`` strategy.""" - msg = "Strict editable install will be performed using a link tree.\n" - _logger.warning(msg + _STRICT_WARNING) auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) populate = _LinkTree( self.distribution, @@ -291,31 +289,21 @@ def _populate_link_tree( outputs, output_mapping, ) - populate(wheel) - - msg = f"""\n - Strict editable installation performed using the auxiliary directory: - {auxiliary_dir} - - Please be careful to not remove this directory, otherwise you might not be able - to import/use your package. - """ - warnings.warn(msg, InformationOnly) + with populate: + populate(wheel) def _populate_static_pth(self, name: str, project_dir: Path, wheel: "WheelFile"): """Populate wheel using the "lax" ``.pth`` file strategy, for ``src-layout``.""" src_dir = self.package_dir[""] - msg = f"Editable install will be performed using .pth file to {src_dir}.\n" - _logger.warning(msg + _LAX_WARNING) populate = _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) - populate(wheel) + with populate: + populate(wheel) def _populate_finder(self, name: str, wheel: "WheelFile"): """Populate wheel using the "lax" MetaPathFinder strategy.""" - msg = "Editable install will be performed using a meta path finder.\n" - _logger.warning(msg + _LAX_WARNING) populate = _TopLevelFinder(self.distribution, name) - populate(wheel) + with populate: + populate(wheel) class _StaticPth: @@ -329,6 +317,17 @@ def __call__(self, wheel: "WheelFile"): contents = bytes(f"{entries}\n", "utf-8") wheel.writestr(f"__editable__.{self.name}.pth", contents) + def __enter__(self): + msg = f""" + Editable install will be performed using .pth file to extend `sys.path` with: + {self.path_entries!r} + """ + _logger.warning(msg + _LAX_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + class _LinkTree(_StaticPth): """ @@ -388,6 +387,21 @@ def _create_links(self): for relative, src in mappings.items(): self._create_file(relative, src, link=link_type) + def __enter__(self): + msg = "Strict editable install will be performed using a link tree.\n" + _logger.warning(msg + _STRICT_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + msg = f"""\n + Strict editable installation performed using the auxiliary directory: + {self.auxiliary_dir} + + Please be careful to not remove this directory, otherwise you might not be able + to import/use your package. + """ + warnings.warn(msg, InformationOnly) + class _TopLevelFinder: def __init__(self, dist: Distribution, name: str): @@ -413,6 +427,14 @@ def __call__(self, wheel: "WheelFile"): content = bytes(f"import {finder}; {finder}.install()", "utf-8") wheel.writestr(f"__editable__.{self.name}.pth", content) + def __enter__(self): + msg = "Editable install will be performed using a meta path finder.\n" + _logger.warning(msg + _LAX_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + def _can_symlink_files() -> bool: with TemporaryDirectory() as tmp: From 5f231b9fc86fa9ae9a0ed81f441b9113f416c5b1 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:39:30 +0100 Subject: [PATCH 4/7] editable_wheel: simplify strategy instantiation --- setuptools/command/editable_wheel.py | 68 ++++++---------------------- 1 file changed, 15 insertions(+), 53 deletions(-) diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 6cfdefddc6..c6c08688e4 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -212,8 +212,6 @@ def _create_wheel_file(self, bdist_wheel): if wheel_path.exists(): wheel_path.unlink() - # Currently the wheel API receives a directory and dump all its contents - # inside of a wheel. So let's use a temporary directory. unpacked_wheel = TemporaryDirectory(suffix=archive_name) build_lib = TemporaryDirectory(suffix=".build-lib") build_tmp = TemporaryDirectory(suffix=".build-temp") @@ -223,10 +221,9 @@ def _create_wheel_file(self, bdist_wheel): shutil.copytree(self.dist_info_dir, unpacked_dist_info) self._install_namespaces(unpacked, dist_info.name) files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp) - with WheelFile(wheel_path, "w") as wf: - self._populate_wheel( - wf, dist_info.name, tag, unpacked, lib, tmp, files, mapping - ) + strategy = self._select_strategy(dist_name, tag, lib, files, mapping) + with strategy, WheelFile(wheel_path, "w") as wf: + strategy(wf) wf.write_files(unpacked) return wheel_path @@ -237,14 +234,11 @@ def _run_install(self, category: str): _logger.info(f"Installing {category} as non editable") self.run_command(f"install_{category}") - def _populate_wheel( + def _select_strategy( self, - wheel: "WheelFile", name: str, tag: str, - unpacked_dir: Path, build_lib: _Path, - tmp: _Path, outputs: List[str], output_mapping: Dict[str, str], ): @@ -253,57 +247,25 @@ def _populate_wheel( project_dir = Path(self.project_dir) if self.strict or os.getenv("SETUPTOOLS_EDITABLE", None) == "strict": - return self._populate_link_tree( - name, build_name, wheel, build_lib, outputs, output_mapping + auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) + return _LinkTree( + self.distribution, + name, + auxiliary_dir, + build_lib, + outputs, + output_mapping, ) - # Build extensions in-place - self.reinitialize_command("build_ext", inplace=1) - self.run_command("build_ext") - packages = _find_packages(self.distribution) has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) if set(self.package_dir) == {""} and has_simple_layout: # src-layout(ish) is relatively safe for a simple pth file - return self._populate_static_pth(name, project_dir, wheel) + src_dir = self.package_dir[""] + return _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) # Use a MetaPathFinder to avoid adding accidental top-level packages/modules - self._populate_finder(name, wheel) - - def _populate_link_tree( - self, - name: str, - build_name: str, - wheel: "WheelFile", - build_lib: _Path, - outputs: List[str], - output_mapping: Dict[str, str], - ): - """Populate wheel using the "strict" ``link tree`` strategy.""" - auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) - populate = _LinkTree( - self.distribution, - name, - auxiliary_dir, - build_lib, - outputs, - output_mapping, - ) - with populate: - populate(wheel) - - def _populate_static_pth(self, name: str, project_dir: Path, wheel: "WheelFile"): - """Populate wheel using the "lax" ``.pth`` file strategy, for ``src-layout``.""" - src_dir = self.package_dir[""] - populate = _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) - with populate: - populate(wheel) - - def _populate_finder(self, name: str, wheel: "WheelFile"): - """Populate wheel using the "lax" MetaPathFinder strategy.""" - populate = _TopLevelFinder(self.distribution, name) - with populate: - populate(wheel) + return _TopLevelFinder(self.distribution, name) class _StaticPth: From a376bf930cabee548f5f1125a8679cf6fecfde4f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:49:20 +0100 Subject: [PATCH 5/7] editable_wheel: Improve strategy interface --- setuptools/command/editable_wheel.py | 59 ++++++++++++----------- setuptools/tests/test_editable_install.py | 4 +- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index c6c08688e4..bd580acb5e 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -40,6 +40,13 @@ if TYPE_CHECKING: from wheel.wheelfile import WheelFile # noqa +if sys.version_info >= (3, 8): + from typing import Protocol +elif TYPE_CHECKING: + from typing_extensions import Protocol +else: + from abc import ABC as Protocol + _Path = Union[str, Path] _P = TypeVar("_P", bound=_Path) _logger = logging.getLogger(__name__) @@ -221,10 +228,10 @@ def _create_wheel_file(self, bdist_wheel): shutil.copytree(self.dist_info_dir, unpacked_dist_info) self._install_namespaces(unpacked, dist_info.name) files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp) - strategy = self._select_strategy(dist_name, tag, lib, files, mapping) - with strategy, WheelFile(wheel_path, "w") as wf: - strategy(wf) - wf.write_files(unpacked) + strategy = self._select_strategy(dist_name, tag, lib) + with strategy, WheelFile(wheel_path, "w") as wheel_obj: + strategy(wheel_obj, files, mapping) + wheel_obj.write_files(unpacked) return wheel_path @@ -239,23 +246,14 @@ def _select_strategy( name: str, tag: str, build_lib: _Path, - outputs: List[str], - output_mapping: Dict[str, str], - ): + ) -> "EditableStrategy": """Decides which strategy to use to implement an editable installation.""" build_name = f"__editable__.{name}-{tag}" project_dir = Path(self.project_dir) if self.strict or os.getenv("SETUPTOOLS_EDITABLE", None) == "strict": auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) - return _LinkTree( - self.distribution, - name, - auxiliary_dir, - build_lib, - outputs, - output_mapping, - ) + return _LinkTree(self.distribution, name, auxiliary_dir, build_lib) packages = _find_packages(self.distribution) has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) @@ -268,13 +266,24 @@ def _select_strategy( return _TopLevelFinder(self.distribution, name) +class EditableStrategy(Protocol): + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + ... + + def __enter__(self): + ... + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + + class _StaticPth: def __init__(self, dist: Distribution, name: str, path_entries: List[Path]): self.dist = dist self.name = name self.path_entries = path_entries - def __call__(self, wheel: "WheelFile"): + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): entries = "\n".join((str(p.resolve()) for p in self.path_entries)) contents = bytes(f"{entries}\n", "utf-8") wheel.writestr(f"__editable__.{self.name}.pth", contents) @@ -306,19 +315,15 @@ def __init__( name: str, auxiliary_dir: _Path, build_lib: _Path, - outputs: List[str], - output_mapping: Dict[str, str], ): self.auxiliary_dir = Path(auxiliary_dir) self.build_lib = Path(build_lib).resolve() - self.outputs = outputs - self.output_mapping = output_mapping self._file = dist.get_command_obj("build_py").copy_file super().__init__(dist, name, [self.auxiliary_dir]) - def __call__(self, wheel: "WheelFile"): - self._create_links() - super().__call__(wheel) + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + self._create_links(files, mapping) + super().__call__(wheel, files, mapping) def _normalize_output(self, file: str) -> Optional[str]: # Files relative to build_lib will be normalized to None @@ -333,15 +338,15 @@ def _create_file(self, relative_output: str, src_file: str, link=None): dest.parent.mkdir(parents=True) self._file(src_file, dest, link=link) - def _create_links(self): + def _create_links(self, outputs, output_mapping): link_type = "sym" if _can_symlink_files() else "hard" mappings = { self._normalize_output(k): v - for k, v in self.output_mapping.items() + for k, v in output_mapping.items() } mappings.pop(None, None) # remove files that are not relative to build_lib - for output in self.outputs: + for output in outputs: relative = self._normalize_output(output) if relative and relative not in mappings: self._create_file(relative, output) @@ -370,7 +375,7 @@ def __init__(self, dist: Distribution, name: str): self.dist = dist self.name = name - def __call__(self, wheel: "WheelFile"): + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): src_root = self.dist.src_root or os.curdir top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) package_dir = self.dist.package_dir or {} diff --git a/setuptools/tests/test_editable_install.py b/setuptools/tests/test_editable_install.py index faf614fd81..a76ab08233 100644 --- a/setuptools/tests/test_editable_install.py +++ b/setuptools/tests/test_editable_install.py @@ -603,8 +603,8 @@ def test_generated_tree(self, tmp_path): outputs = build_py.get_outputs() output_mapping = build_py.get_output_mapping() - make_tree = _LinkTree(dist, name, aux, build, outputs, output_mapping) - make_tree(wheel) + make_tree = _LinkTree(dist, name, aux, build) + make_tree(wheel, outputs, output_mapping) mod1 = next(aux.glob("**/mod1.py")) expected = tmp_path / "src/mypkg/mod1.py" From fdd9ab363a88665b3678c661721c073f19982737 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 15:40:55 +0100 Subject: [PATCH 6/7] Add docstrings for command classes --- setuptools/__init__.py | 60 +++++++++++++++++++- setuptools/command/build.py | 108 +++++++++++++++++++++++++++++++++++- 2 files changed, 166 insertions(+), 2 deletions(-) diff --git a/setuptools/__init__.py b/setuptools/__init__.py index cff04323e9..ae53570c99 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -94,7 +94,59 @@ def setup(**attrs): class Command(_Command): - __doc__ = _Command.__doc__ + """ + Setuptools internal actions are organized using a *command design pattern*. + This means that each action (or group of closely related actions) executed during + the build should be implemented as a ``Command`` subclass. + + These commands are abstractions and do not necessarily correspond to a command that + can (or should) be executed via a terminal, in a CLI fashion (although historically + they would). + + When creating a new command from scratch, custom defined classes **SHOULD** inherit + from ``setuptools.Command`` and implement a few mandatory methods. + Between these mandatory methods, are listed: + + .. method:: initialize_options(self) + + Set or (reset) all options/attributes/caches used by the command + to their default values. Note that these values may be overwritten during + the build. + + .. method:: finalize_options(self) + + Set final values for all options/attributes used by the command. + Most of the time, each option/attribute/cache should only be set if it does not + have any value yet (e.g. ``if self.attr is None: self.attr = val``). + + .. method: run(self) + + Execute the actions intended by the command. + (Side effects **SHOULD** only take place when ``run`` is executed, + for example, creating new files or writing to the terminal output). + + A useful analogy for command classes is to think of them as subroutines with local + variables called "options". The options are "declared" in ``initialize_options()`` + and "defined" (given their final values, aka "finalized") in ``finalize_options()``, + both of which must be defined by every command class. The "body" of the subroutine, + (where it does all the work) is the ``run()`` method. + Between ``initialize_options()`` and ``finalize_options()``, ``setuptools`` may set + the values for options/attributes based on user's input (or circumstance), + which means that the implementation should be careful to not overwrite values in + ``finalize_options`` unless necessary. + + Please note that other commands (or other parts of setuptools) may also overwrite + the values of the command's options/attributes multiple times during the build + process. + Therefore it is important to consistently implement ``initialize_options()`` and + ``finalize_options()``. For example, all derived attributes (or attributes that + depend on the value of other attributes) **SHOULD** be recomputed in + ``finalize_options``. + + When overwriting existing commands, custom defined classes **MUST** abide by the + same APIs implemented by the original class. They also **SHOULD** inherit from the + original class. + """ command_consumes_arguments = False @@ -122,6 +174,12 @@ def ensure_string_list(self, option): currently a string, we split it either on /,\s*/ or /\s+/, so "foo bar baz", "foo,bar,baz", and "foo, bar baz" all become ["foo", "bar", "baz"]. + + .. + TODO: This method seems to be similar to the one in ``distutils.cmd`` + Probably it is just here for backward compatibility with old Python versions? + + :meta private: """ val = getattr(self, option) if val is None: diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 12a4362209..bf4f71a706 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -1,8 +1,17 @@ -from distutils.command.build import build as _build +import sys import warnings +from typing import TYPE_CHECKING, List, Dict +from distutils.command.build import build as _build from setuptools import SetuptoolsDeprecationWarning +if sys.version_info >= (3, 8): + from typing import Protocol +elif TYPE_CHECKING: + from typing_extensions import Protocol +else: + from abc import ABC as Protocol + _ORIGINAL_SUBCOMMANDS = {"build_py", "build_clib", "build_ext", "build_scripts"} @@ -22,3 +31,100 @@ def run(self): warnings.warn(msg, SetuptoolsDeprecationWarning) self.sub_commands = _build.sub_commands super().run() + + +class SubCommand(Protocol): + """In order to support editable installations (see :pep:`660`) all + build subcommands **SHOULD** implement this protocol. They also **MUST** inherit + from ``setuptools.Command``. + + When creating an :pep:`editable wheel <660>`, ``setuptools`` will try to evaluate + custom ``build`` subcommands using the following procedure: + + 1. ``setuptools`` will set the ``editable_mode`` flag will be set to ``True`` + 2. ``setuptools`` will execute the ``run()`` command. + + .. important:: + Subcommands **SHOULD** take advantage of ``editable_mode=True`` to adequate + its behaviour or perform optimisations. + + For example, if a subcommand don't need to generate any extra file and + everything it does is to copy a source file into the build directory, + ``run()`` **SHOULD** simply "early return". + + Similarly, if the subcommand creates files that would be placed alongside + Python files in the final distribution, during an editable install + the command **SHOULD** generate these files "in place" (i.e. write them to + the original source directory, instead of using the build directory). + Note that ``get_output_mapping()`` should reflect that and include mappings + for "in place" builds accordingly. + + 3. ``setuptools`` use any knowledge it can derive from the return values of + ``get_outputs()`` and ``get_output_mapping()`` to create an editable wheel. + When relevant ``setuptools`` **MAY** attempt to use file links based on the value + of ``get_output_mapping()``. Alternatively, ``setuptools`` **MAY** attempt to use + :doc:`import hooks ` to redirect any attempt to import + to the directory with the original source code and other files built in place. + """ + + editable_mode: bool = False + """Boolean flag that will be set to ``True`` when setuptools is used for an + editable installation (see :pep:`660`). + Implementations **SHOULD** explicitly set the default value of this attribute to + ``False``. + When subcommands run, they can use this flag to perform optimizations or change + their behaviour accordingly. + """ + + build_lib: str + """String representing the directory where the build artifacts should be stored, + e.g. ``build/lib``. + For example, if a distribution wants to provide a Python module named ``pkg.mod``, + then a corresponding file should be written to ``{build_lib}/package/module.py``. + A way of thinking about this is that the files saved under ``build_lib`` + would be eventually copied to one of the directories in :obj:`site.PREFIXES` + upon installation. + + A command that produces platform-independent files (e.g. compiling text templates + into Python functions), **CAN** initialize ``build_lib`` by copying its value from + the ``build_py`` command. On the other hand, a command that produces + platform-specific files **CAN** initialize ``build_lib`` by copying its value from + the ``build_ext`` command. In general this is done inside the ``finalize_options`` + method with the help of the ``set_undefined_options`` command:: + + def finalize_options(self): + self.set_undefined_options("build_py", ("build_lib", "build_lib")) + ... + """ + + def initialize_options(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def finalize_options(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def run(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def get_outputs(self) -> List[str]: + """ + Return a list of files intended for distribution as they would have been + produced by the build. + These files should be strings in the form of + ``"{build_lib}/destination/file/path"``. + + .. note:: + The return value of ``get_output()`` should include all files used as keys + in ``get_output_mapping()`` plus files that are generated during the build + and don't correspond to any source file already present in the project. + """ + + def get_output_mapping(self) -> Dict[str, str]: + """ + Return a mapping between destination files as they would be produced by the + build (dict keys) into the respective existing (source) files (dict values). + Existing (source) files should be represented as strings relative to the project + root directory. + Destination files should be strings in the form of + ``"{build_lib}/destination/file/path"``. + """ From 965458d1d271553d25f431ffebdb68bd12938f9a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 15:57:06 +0100 Subject: [PATCH 7/7] Revert addition of use_links to build_lib --- setuptools/command/build_ext.py | 2 +- setuptools/command/build_py.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 31ef47bf3e..7ad5a87adc 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -104,7 +104,7 @@ def copy_extensions_to_source(self): # Always copy, even if source is older than destination, to ensure # that the right extensions for the current Python/platform are # used. - build_py.copy_file(regular_file, inplace_file) + self.copy_file(regular_file, inplace_file, level=self.verbose) if ext._needs_stub: inplace_stub = self._get_equivalent_stub(ext, inplace_file) diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index a2a6fe2cc8..923a32329f 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -40,17 +40,15 @@ def finalize_options(self): if 'data_files' in self.__dict__: del self.__dict__['data_files'] self.__updated_files = [] - self.use_links = None def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, link=None, level=1): # Overwrite base class to allow using links - link = getattr(self, "use_links", None) if link is None else link if link: infile = str(Path(infile).resolve()) outfile = str(Path(outfile).resolve()) - return super().copy_file(infile, outfile, preserve_mode, - preserve_times, link, level) + return super().copy_file(infile, outfile, preserve_mode, preserve_times, + link, level) def run(self): """Build modules, packages, and copy data files to build directory"""