Skip to content

Commit

Permalink
feedback
Browse files Browse the repository at this point in the history
  • Loading branch information
@mayeut
mayeut committed Jan 5, 2022
1 parent 3c6d509 commit 7837666
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 122 deletions.
3 changes: 3 additions & 0 deletions cibuildwheel/__main__.py
View file
Expand Up @@ -174,6 +174,9 @@ def main() -> None:
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
sys.stdout = Unbuffered(sys.stdout) # type: ignore[assignment]

# create the cache dir before it gets printed & builds performed
CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True)

print_preamble(platform=platform, options=options, identifiers=identifiers)

try:
Expand Down
107 changes: 50 additions & 57 deletions cibuildwheel/macos.py
View file
Expand Up @@ -78,51 +78,49 @@ def get_python_configurations(


def install_cpython(tmp: Path, version: str, url: str) -> Path:
installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines()

# if this version of python isn't installed, get it from python.org and install
python_package_identifier = f"org.python.Python.PythonFramework-{version}"
installation_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}")
with FileLock(CIBW_CACHE_PATH / f"cpython{version}.lock"):
installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines()
# if this version of python isn't installed, get it from python.org and install
python_package_identifier = f"org.python.Python.PythonFramework-{version}"
if python_package_identifier not in installed_system_packages:
if detect_ci_provider() is None:
# if running locally, we don't want to install CPython with sudo
# let the user know & provide a link to the installer
print(
f"Error: CPython {version} is not installed.\n"
"cibuildwheel will not perform system-wide installs when running outside of CI.\n"
f"To build locally, install CPython {version} on this machine, or, disable this version of Python using CIBW_SKIP=cp{version.replace('.', '')}-macosx_*\n"
f"\nDownload link: {url}",
file=sys.stderr,
)
raise SystemExit(1)
pkg_path = tmp / "Python.pkg"
# download the pkg
download(url, pkg_path)
# install
call("sudo", "installer", "-pkg", pkg_path, "-target", "/")
pkg_path.unlink()
env = os.environ.copy()
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
call(installation_path / "bin" / "python3", install_certifi_script, env=env)

if python_package_identifier not in installed_system_packages:
if detect_ci_provider() is None:
# if running locally, we don't want to install CPython with sudo
# let the user know & provide a link to the installer
print(
f"Error: CPython {version} is not installed.\n"
"cibuildwheel will not perform system-wide installs when running outside of CI.\n"
f"To build locally, install CPython {version} on this machine, or, disable this version of Python using CIBW_SKIP=cp{version.replace('.', '')}-macosx_*\n"
f"\nDownload link: {url}",
file=sys.stderr,
)
raise SystemExit(1)
pkg_path = tmp / "Python.pkg"
# download the pkg
download(url, pkg_path)
# install
call("sudo", "installer", "-pkg", pkg_path, "-target", "/")
pkg_path.unlink()
env = os.environ.copy()
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
call(installation_path / "bin" / "python3", install_certifi_script, env=env)

return installation_path
return installation_path / "bin" / "python3"


def install_pypy(tmp: Path, version: str, url: str) -> Path:
pypy_tar_bz2 = url.rsplit("/", 1)[-1]
extension = ".tar.bz2"
assert pypy_tar_bz2.endswith(extension)
pypy_base_filename = pypy_tar_bz2[: -len(extension)]
installation_path = CIBW_CACHE_PATH / pypy_base_filename
if not installation_path.exists():
downloaded_tar_bz2 = tmp / pypy_tar_bz2
download(url, downloaded_tar_bz2)
installation_path.parent.mkdir(parents=True, exist_ok=True)
call("tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2)
downloaded_tar_bz2.unlink()

return installation_path
installation_path = CIBW_CACHE_PATH / pypy_tar_bz2[: -len(extension)]
with FileLock(str(installation_path) + ".lock"):
if not installation_path.exists():
downloaded_tar_bz2 = tmp / pypy_tar_bz2
download(url, downloaded_tar_bz2)
installation_path.parent.mkdir(parents=True, exist_ok=True)
call("tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2)
downloaded_tar_bz2.unlink()
return installation_path / "bin" / "pypy3"


def setup_python(
Expand All @@ -135,22 +133,17 @@ def setup_python(
tmp.mkdir()
implementation_id = python_configuration.identifier.split("-")[0]
log.step(f"Installing Python {implementation_id}...")
CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True)
with FileLock(CIBW_CACHE_PATH / "install.lock"):
if implementation_id.startswith("cp"):
installation_path = install_cpython(
tmp, python_configuration.version, python_configuration.url
)
elif implementation_id.startswith("pp"):
installation_path = install_pypy(
tmp, python_configuration.version, python_configuration.url
)
else:
raise ValueError("Unknown Python implementation")
if implementation_id.startswith("cp"):
base_python = install_cpython(tmp, python_configuration.version, python_configuration.url)
elif implementation_id.startswith("pp"):
base_python = install_pypy(tmp, python_configuration.version, python_configuration.url)
else:
raise ValueError("Unknown Python implementation")
assert base_python.exists()

log.step("Setting up build environment...")
venv_path = tmp / "venv"
env = virtualenv(installation_path, venv_path, dependency_constraint_flags)
env = virtualenv(base_python, venv_path, dependency_constraint_flags)
venv_bin_path = venv_path / "bin"
assert venv_bin_path.exists()
# Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`,
Expand Down Expand Up @@ -298,10 +291,10 @@ def build(options: Options, tmp_path: Path) -> None:
build_options = options.build_options(config.identifier)
log.build_start(config.identifier)

tmp_config_dir = tmp_path / config.identifier
tmp_config_dir.mkdir()
built_wheel_dir = tmp_config_dir / "built_wheel"
repaired_wheel_dir = tmp_config_dir / "repaired_wheel"
identifier_tmp_dir = tmp_path / config.identifier
identifier_tmp_dir.mkdir()
built_wheel_dir = identifier_tmp_dir / "built_wheel"
repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"

config_is_arm64 = config.identifier.endswith("arm64")
config_is_universal2 = config.identifier.endswith("universal2")
Expand All @@ -314,7 +307,7 @@ def build(options: Options, tmp_path: Path) -> None:
]

env = setup_python(
tmp_config_dir / "build",
identifier_tmp_dir / "build",
config,
dependency_constraint_flags,
build_options.environment,
Expand Down Expand Up @@ -459,7 +452,7 @@ def build(options: Options, tmp_path: Path) -> None:
# there are no dependencies that were pulled in at build time.
call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)

venv_dir = tmp_config_dir / "venv-test"
venv_dir = identifier_tmp_dir / "venv-test"

arch_prefix = []
if testing_arch != machine_arch:
Expand Down Expand Up @@ -532,7 +525,7 @@ def shell_with_arch(command: str, **kwargs: Any) -> None:
shutil.move(str(repaired_wheel), build_options.output_dir)

# clean up
shutil.rmtree(tmp_config_dir)
shutil.rmtree(identifier_tmp_dir)

log.build_end()
except subprocess.CalledProcessError as error:
Expand Down
40 changes: 24 additions & 16 deletions cibuildwheel/util.py
View file
@@ -1,6 +1,5 @@
import contextlib
import fnmatch
import functools
import itertools
import os
import re
Expand All @@ -12,6 +11,7 @@
import time
import urllib.request
from enum import Enum
from functools import lru_cache
from pathlib import Path
from time import sleep
from typing import (
Expand Down Expand Up @@ -461,7 +461,7 @@ def get_pip_version(env: Dict[str, str]) -> str:
return pip_version


@functools.lru_cache(maxsize=None)
@lru_cache(maxsize=None)
def _ensure_virtualenv() -> Path:
input_file = resources_dir / "virtualenv.toml"
with input_file.open("rb") as f:
Expand All @@ -478,16 +478,22 @@ def _ensure_virtualenv() -> Path:
def _parse_constraints_for_virtualenv(
dependency_constraint_flags: Sequence[PathOrStr],
) -> Dict[str, str]:
"""
Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where
the key is the package name, and the value is the constraint version.
If a package version cannot be found, its value is "embed" meaning that virtualenv will install
its bundled version, already available locally.
The function does not try to be too smart and just handles basic constraints.
If it can't get an exact version, the real constraint will be handled by the
{macos|windows}.setup_python function.
"""
assert len(dependency_constraint_flags) in {0, 2}
packages = ["pip", "setuptools", "wheel"]
constraints_dict = {package: "embed" for package in packages}
if len(dependency_constraint_flags) == 2:
assert dependency_constraint_flags[0] == "-c"
constraint_path = Path(dependency_constraint_flags[1])
assert constraint_path.exists()
# don't try to be too smart, just handle basic constraints
# if we can't get an exact version, just keep the default, the constraint will be
# handled by the {macos|windows}.setup_python function
with constraint_path.open() as constraint_file:
for line in constraint_file:
line = line.strip()
Expand Down Expand Up @@ -516,25 +522,23 @@ def _parse_constraints_for_virtualenv(


def virtualenv(
install_path: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr]
python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr]
) -> Dict[str, str]:
if IS_WIN:
paths = [str(venv_path), str(venv_path / "Scripts")]
python = install_path / "python.exe"
else:
paths = [str(venv_path / "bin")]
for python_name in ["pypy3", "python3"]:
python = install_path / "bin" / python_name
if python.exists():
break
assert python.exists()
virtualenv_app = _ensure_virtualenv()
constraints = _parse_constraints_for_virtualenv(dependency_constraint_flags)
additional_flags = [f"--{package}={version}" for package, version in constraints.items()]

# Using symlinks to pre-installed seed packages is really the fastest way to get a virtual
# environment. The initial cost is a bit higher but reusing is much faster.
# Windows does not always allow symlinks so just disabling for now.
# Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the
# version of pip that will end-up installed.
# c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder
if (
not IS_WIN
and constraints["pip"] != "embed"
and Version(constraints["pip"]) > Version("19.3")
and Version(constraints["pip"]) >= Version("19.3")
):
additional_flags.append("--symlink-app-data")

Expand All @@ -549,6 +553,10 @@ def virtualenv(
python,
venv_path,
)
if IS_WIN:
paths = [str(venv_path), str(venv_path / "Scripts")]
else:
paths = [str(venv_path / "bin")]
env = os.environ.copy()
env["PATH"] = os.pathsep.join(paths + [env["PATH"]])
return env

0 comments on commit 7837666

Please sign in to comment.