From 82dcd45af55af239a7be81ee2e86948c6fe75518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Tue, 3 Jan 2023 18:05:12 -0800 Subject: [PATCH] Packaging inherits from pkgenv, deps and document tox 4 packaging changes (#2813) Resolves https://github.com/tox-dev/tox/issues/2543 --- docs/changelog/2543.doc.rst | 1 + docs/changelog/2543.feature.rst | 3 + docs/upgrading.rst | 93 +++++++++++++++++++ src/tox/config/main.py | 6 +- src/tox/config/source/api.py | 2 +- src/tox/config/source/ini.py | 6 +- src/tox/config/source/ini_section.py | 2 + src/tox/tox_env/python/package.py | 11 ++- src/tox/util/ci.py | 7 +- tests/session/cmd/test_show_config.py | 17 ++++ .../package/test_package_pyproject.py | 15 +++ tests/util/test_ci.py | 9 ++ whitelist.txt | 3 + 13 files changed, 166 insertions(+), 9 deletions(-) create mode 100644 docs/changelog/2543.doc.rst create mode 100644 docs/changelog/2543.feature.rst diff --git a/docs/changelog/2543.doc.rst b/docs/changelog/2543.doc.rst new file mode 100644 index 000000000..59af6d2f9 --- /dev/null +++ b/docs/changelog/2543.doc.rst @@ -0,0 +1 @@ +Document breaking changes with tox 4 and packaging environments - by :user:`gaborbernat`. diff --git a/docs/changelog/2543.feature.rst b/docs/changelog/2543.feature.rst new file mode 100644 index 000000000..c605a26db --- /dev/null +++ b/docs/changelog/2543.feature.rst @@ -0,0 +1,3 @@ +Packaging environments now inherit from the ``pkgenv`` section, allowing to set all your packaging options in one place, +and support the ``deps`` key to set additional dependencies that will be installed after ``pyprojec.toml`` static +``requires`` but before backends dynamic requires - by :user:`gaborbernat`. diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 459cfe4a3..98b6c4e5d 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -222,3 +222,96 @@ This is best avoided by updating to non-legacy usage: # or, equivalently... $ tox r -e list + +Packaging environments +---------------------- + +Isolated environment on by default +++++++++++++++++++++++++++++++++++ +``tox`` now always uses an isolated build environment when building your projects package. The previous flag to enable +this called ``isolated_build`` has been removed. + +Packaging configuration and inheritance ++++++++++++++++++++++++++++++++++++++++ +Isolated build environments are tox environments themselves and may be configured on their own. Their name is defined +as follows: + +- For source distributions this environment will match a virtual environment with the same python interpreter as tox is + using. The name of this environment will by default ``.pkg`` (can be changed via :ref:`package_env` config on a per + test environment basis). +- For wheels (including editable wheels as defined by :pep:`660`) their name will be ``.pkg-``, so + for example if you're building a wheel for a Python 3.10 environment the packaging environment will be + ``.pkg-cpython311`` (can be changed via :ref:`wheel_build_env` config on a per test environment basis). + +To change a packaging environments settings you can use: + +.. code-block:: ini + + [testenv:.pkg] + pass_env = + PKG_CONFIG + PKG_CONFIG_PATH + PKG_CONFIG_SYSROOT_DIR + + [testenv:.pkg-cpython311] + pass_env = + PKG_CONFIG + PKG_CONFIG_PATH + PKG_CONFIG_SYSROOT_DIR + +Packaging environments no longer inherit their settings from the ``testenv`` section, as this caused issues when +some test environment settings conflicted with packaging setting. However starting with ``tox>=4.2`` all packaging +environments inherit from the ``pkgenv`` section, allowing you to define packaging common packaging settings in one +central place, while still allowing you to override it when needed on a per package environment basis: + +.. code-block:: ini + + [pkgenv] + pass_env = + PKG_CONFIG + PKG_CONFIG_PATH + PKG_CONFIG_SYSROOT_DIR + + [testenv:.pkg-cpython311] + pass_env = + {[pkgenv]pass_env} + IS_311 = yes + + [testenv:magic] + package = sdist + pass_env = {[pkgenv]pass_env} # sdist install builds wheel -> need packaging settings + +Note that specific packaging environments are defined under ``testenv:.pkg`` and **not** ``pkgenv:.pkg``, this is due +backwards compatibility. + +Universal wheels +++++++++++++++++ +If your project builds universal wheels you can avoid using multiple build environments for each targeted python by +setting :ref:`wheel_build_env` to the same packaging environment via: + +.. code-block:: ini + + [testenv] + package = wheel + wheel_build_env = .pkg + +Editable mode ++++++++++++++ +``tox`` now defaults to using editable wheels when develop mode is enabled and the build backend supports it, +as defined by :pep:`660` by setting :ref:`package` to ``editable``. In case the backend does not support it, will +fallback to :ref:`package` to ``editable-legacy``, and invoke pip with ``-e``. In the later case will also print a +message to make this setting explicit in your configuration (explicit better than implicit): + +.. code-block:: ini + + [testenv:dev] + package = editable-legacy + +If you want to use the new standardized method to achieve the editable install effect you should ensure your backend +version is above the version this feature was added to it, for example for setuptools: + +.. code-block:: ini + + [testenv:dev] + deps = setuptools>=64 + package = editable diff --git a/src/tox/config/main.py b/src/tox/config/main.py index 38efb600b..00aaaed81 100644 --- a/src/tox/config/main.py +++ b/src/tox/config/main.py @@ -152,10 +152,10 @@ def get_env( :param loaders: loaders to use for this configuration (only used for creation) :return: the tox environments config """ - section, base = self._src.get_tox_env_section(item) + section, base_test, base_pkg = self._src.get_tox_env_section(item) conf_set = self.get_section_config( section, - base=None if package else base, + base=base_pkg if package else base_test, of_type=EnvConfigSet, for_env=item, loaders=loaders, @@ -163,7 +163,7 @@ def get_env( return conf_set def clear_env(self, name: str) -> None: - section, _ = self._src.get_tox_env_section(name) + section, _, __ = self._src.get_tox_env_section(name) del self._key_to_conf_set[(section.key, name)] diff --git a/src/tox/config/source/api.py b/src/tox/config/source/api.py index 3b95d5043..e762d4794 100644 --- a/src/tox/config/source/api.py +++ b/src/tox/config/source/api.py @@ -98,7 +98,7 @@ def envs(self, core_conf: CoreConfigSet) -> Iterator[str]: raise NotImplementedError @abstractmethod - def get_tox_env_section(self, item: str) -> tuple[Section, list[str]]: + def get_tox_env_section(self, item: str) -> tuple[Section, list[str], list[str]]: """:returns: the section for a tox environment""" raise NotImplementedError diff --git a/src/tox/config/source/ini.py b/src/tox/config/source/ini.py index f3e0c083e..df6c2876c 100644 --- a/src/tox/config/source/ini.py +++ b/src/tox/config/source/ini.py @@ -14,7 +14,7 @@ from ..loader.section import Section from ..sets import ConfigSet from .api import Source -from .ini_section import CORE, TEST_ENV_PREFIX, IniSection +from .ini_section import CORE, PKG_ENV_PREFIX, TEST_ENV_PREFIX, IniSection class IniSource(Source): @@ -62,8 +62,8 @@ def get_base_sections(self, base: list[str], in_section: Section) -> Iterator[Se if in_section.prefix is not None: # no prefix specified, so this could imply our own prefix yield IniSection(in_section.prefix, a_base) - def get_tox_env_section(self, item: str) -> tuple[Section, list[str]]: - return IniSection.test_env(item), [TEST_ENV_PREFIX] + def get_tox_env_section(self, item: str) -> tuple[Section, list[str], list[str]]: + return IniSection.test_env(item), [TEST_ENV_PREFIX], [PKG_ENV_PREFIX] def envs(self, core_config: ConfigSet) -> Iterator[str]: seen = set() diff --git a/src/tox/config/source/ini_section.py b/src/tox/config/source/ini_section.py index 33a718632..a155dc1fc 100644 --- a/src/tox/config/source/ini_section.py +++ b/src/tox/config/source/ini_section.py @@ -20,10 +20,12 @@ def names(self) -> list[str]: TEST_ENV_PREFIX = "testenv" +PKG_ENV_PREFIX = "pkgenv" CORE = IniSection(None, "tox") __all__ = [ "IniSection", "CORE", "TEST_ENV_PREFIX", + "PKG_ENV_PREFIX", ] diff --git a/src/tox/tox_env/python/package.py b/src/tox/tox_env/python/package.py index 910554a85..289bd7744 100644 --- a/src/tox/tox_env/python/package.py +++ b/src/tox/tox_env/python/package.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Generator, Iterator, Sequence, cast +from typing import TYPE_CHECKING, Any, Generator, Iterator, List, Sequence, cast from packaging.requirements import Requirement @@ -56,6 +56,7 @@ def _setup_env(self) -> None: """setup the tox environment""" super()._setup_env() self._install(self.requires(), PythonPackageToxEnv.__name__, "requires") + self._install(self.conf["deps"], PythonPackageToxEnv.__name__, "deps") @abstractmethod def requires(self) -> tuple[Requirement, ...] | PythonDeps: @@ -63,6 +64,14 @@ def requires(self) -> tuple[Requirement, ...] | PythonDeps: def register_run_env(self, run_env: RunToxEnv) -> Generator[tuple[str, str], PackageToxEnv, None]: yield from super().register_run_env(run_env) + if run_env.conf["package"] != "skip" and "deps" not in self.conf: + self.conf.add_config( + keys="deps", + of_type=List[Requirement], + default=[], + desc="Name of the python dependencies as specified by PEP-440", + ) + if ( not isinstance(run_env, Python) or run_env.conf["package"] not in {"wheel", "editable"} diff --git a/src/tox/util/ci.py b/src/tox/util/ci.py index b65b2be6f..30ba80ed9 100644 --- a/src/tox/util/ci.py +++ b/src/tox/util/ci.py @@ -21,7 +21,12 @@ def is_ci() -> bool: """:return: a flag indicating if running inside a CI env or not""" - return any(e in os.environ if v is None else os.environ.get(e) == v for e, v in _ENV_VARS.items()) + for env_key, value in _ENV_VARS.items(): + if env_key in os.environ if value is None else os.environ.get(env_key) == value: + if env_key == "TEAMCITY_VERSION" and os.environ.get(env_key) == "LOCAL": + continue + return True + return False __all__ = [ diff --git a/tests/session/cmd/test_show_config.py b/tests/session/cmd/test_show_config.py index 646dc2bf7..f0acb3d79 100644 --- a/tests/session/cmd/test_show_config.py +++ b/tests/session/cmd/test_show_config.py @@ -254,3 +254,20 @@ def test_show_config_matching_env_section(tox_project: ToxProjectCreator) -> Non outcome = project.run("c", "-e", "a,b", "-k", "deps") outcome.assert_success() assert outcome.out.count("c>=1") == 2, outcome.out + + +def test_package_env_inherits_from_pkgenv(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: + project = tox_project({"tox.ini": "[pkgenv]\npass_env = A, B\ndeps=C\n D"}) + outcome = project.run("c", "--root", str(demo_pkg_inline), "-k", "deps", "pass_env", "-e", "py,.pkg") + outcome.assert_success() + exp = """ + [testenv:.pkg] + deps = + C + D + pass_env = + A + B + """ + exp = dedent(exp) + assert exp in outcome.out diff --git a/tests/tox_env/python/virtual_env/package/test_package_pyproject.py b/tests/tox_env/python/virtual_env/package/test_package_pyproject.py index 839225b98..a8f6c9ed3 100644 --- a/tests/tox_env/python/virtual_env/package/test_package_pyproject.py +++ b/tests/tox_env/python/virtual_env/package/test_package_pyproject.py @@ -194,3 +194,18 @@ def test_pyproject_no_build_editable_fallback(tox_project: ToxProjectCreator, de ] found_calls = [(i[0][0].conf.name, i[0][3].run_id) for i in execute_calls.call_args_list] assert found_calls == expected_calls + + +@pytest.mark.parametrize("package", ["sdist", "wheel", "editable", "editable-legacy", "skip"]) +def test_project_package_with_deps(tox_project: ToxProjectCreator, demo_pkg_setuptools: Path, package: str) -> None: + ini = f"[testenv]\npackage={package}\n[pkgenv]\ndeps = A" + proj = tox_project({"tox.ini": ini}, base=demo_pkg_setuptools) + execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + result = proj.run("r", "--notest") + result.assert_success() + found_calls = [(i[0][0].conf.name, i[0][3].run_id) for i in execute_calls.call_args_list] + if package == "skip": + assert (".pkg", "install_deps") not in found_calls + else: + assert found_calls[0] == (".pkg", "install_requires") + assert found_calls[1] == (".pkg", "install_deps") diff --git a/tests/util/test_ci.py b/tests/util/test_ci.py index 5cfc9bb20..ef9a089dc 100644 --- a/tests/util/test_ci.py +++ b/tests/util/test_ci.py @@ -54,3 +54,12 @@ def test_is_ci_not(monkeypatch: pytest.MonkeyPatch) -> None: for var in _ENV_VARS: monkeypatch.delenv(var, raising=False) assert not is_ci() + + +def test_is_ci_not_teamcity_local(monkeypatch: pytest.MonkeyPatch) -> None: + # pycharm sets this + for var in _ENV_VARS: + monkeypatch.delenv(var, raising=False) + + monkeypatch.setenv("TEAMCITY_VERSION", "LOCAL") + assert not is_ci() diff --git a/whitelist.txt b/whitelist.txt index ac48cb4b0..c79f0c760 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -111,6 +111,7 @@ openpty ov pathname pep517 +pkgenv platformdirs pluggy pos @@ -124,6 +125,7 @@ purelib py311 py38 py39 +pycharm pygments pypa pyproject @@ -160,6 +162,7 @@ subparsers tcgetattr tcsanow tcsetattr +teamcity termios termux testenv