Support GraalPy

README.md

@@ -36,11 +36,12 @@ What does it do?
 | PyPy 3.8 v7.3  | ✅ | ✅⁴ | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A |
 | PyPy 3.9 v7.3  | ✅ | ✅⁴ | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A |
 | PyPy 3.10 v7.3 | ✅ | ✅⁴ | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A |
+| GraalPy 23.0  | ✅ | ✅⁴ | N/A | N/A | N/A | ✅¹ | N/A | ✅¹ | N/A | N/A |
 
-<sup>¹ PyPy is only supported for manylinux wheels.</sup><br>
+<sup>¹ PyPy & GraalPy are only supported for manylinux wheels.</sup><br>
 <sup>² Windows arm64 support is experimental.</sup><br>
 <sup>³ Alpine 3.14 and very briefly 3.15's default python3 [was not able to load](https://github.com/pypa/cibuildwheel/issues/934) musllinux wheels. This has been fixed; please upgrade the python package if using Alpine from before the fix.</sup><br>
-<sup>⁴ Cross-compilation not supported with PyPy - to build these wheels you need to run cibuildwheel on an Apple Silicon machine.</sup><br>
+<sup>⁴ Cross-compilation not supported with PyPy & GraalPy - to build these wheels you need to run cibuildwheel on an Apple Silicon machine.</sup><br>
 <sup>⁵ CPython 3.12 is available using the [CIBW_PRERELEASE_PYTHONS](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons) option.</sup><br>
 
 - Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy

bin/update_pythons.py

@@ -5,6 +5,7 @@
 import copy
 import difflib
 import logging
+import re
 from collections.abc import Mapping, MutableMapping
 from pathlib import Path
 from typing import Any, Union
@@ -51,7 +52,13 @@ class ConfigMacOS(TypedDict):
     url: str
 
 
-AnyConfig = Union[ConfigWinCP, ConfigWinPP, ConfigMacOS]
+class ConfigLinux(TypedDict):
+    identifier: str
+    version: str
+    url: str
+
+
+AnyConfig = Union[ConfigWinCP, ConfigWinPP, ConfigMacOS, ConfigLinux]
 
 
 # The following set of "Versions" classes allow the initial call to the APIs to
@@ -102,6 +109,72 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP | None:
         )
 
 
+class GraalPyVersions:
+    def __init__(self):
+        response = requests.get("https://api.github.com/repos/oracle/graalpython/releases")
+        response.raise_for_status()
+
+        releases = response.json()
+        gp_version_re = re.compile(r"-(\d+\.\d+\.\d+)$")
+        cp_version_re = re.compile(r"a Python (\d+\.\d+(?:\.\d+)?) implementation")
+        for release in releases:
+            m = gp_version_re.search(release["tag_name"])
+            if m:
+                release["graalpy_version"] = Version(m.group(1))
+            m = cp_version_re.search(release["body"])
+            if m:
+                release["python_version"] = Version(m.group(1))
+
+        self.releases = [r for r in releases if "graalpy_version" in r and "python_version" in r]
+
+    def update_version(self, identifier: str, spec: Specifier) -> AnyConfig:
+        if "x86_64" in identifier:
+            arch = "x86_64"
+        elif "arm64" in identifier:
+            arch = "arm64"
+        elif "aarch64" in identifier:
+            arch = "aarch64"
+        else:
+            msg = f"{identifier} not supported yet on GraalPy"
+            raise RuntimeError(msg)
+
+        releases = [r for r in self.releases if spec.contains(r["python_version"])]
+        releases = sorted(releases, key=lambda r: r["graalpy_version"])
+
+        if not releases:
+            msg = f"GraalPy {arch} not found for {spec}!"
+            raise RuntimeError(msg)
+
+        release = releases[-1]
+        version = release["python_version"]
+        gpversion = release["graalpy_version"]
+
+        if "macosx" in identifier:
+            identifier = f"gp{gpversion.major}{gpversion.minor}-macosx_{arch}"
+            config = ConfigMacOS
+            platform = "macos"
+        elif "linux" in identifier:
+            identifier = f"gp{gpversion.major}{gpversion.minor}-manylinux_{arch}"
+            config = ConfigLinux
+            platform = "linux"
+        else:
+            msg = "GraalPy supports on macOS and Linux so far!"
+            raise RuntimeError(msg)
+
+        arch = "amd64" if arch == "x86_64" else "aarch64"
+        (url,) = (
+            rf["browser_download_url"]
+            for rf in release["assets"]
+            if rf["name"].endswith(f"{platform}-{arch}.tar.gz")
+        )
+
+        return config(
+            identifier=identifier,
+            version=f"{version.major}.{version.minor}",
+            url=url,
+        )
+
+
 class PyPyVersions:
     def __init__(self, arch_str: ArchStr):
         response = requests.get("https://downloads.python.org/pypy/versions.json")
@@ -245,6 +318,8 @@ def __init__(self) -> None:
         self.macos_pypy = PyPyVersions("64")
         self.macos_pypy_arm64 = PyPyVersions("ARM64")
 
+        self.graalpy = GraalPyVersions()
+
     def update_config(self, config: MutableMapping[str, str]) -> None:
         identifier = config["identifier"]
         version = Version(config["version"])
@@ -262,6 +337,8 @@ def update_config(self, config: MutableMapping[str, str]) -> None:
                     config_update = self.macos_pypy.update_version_macos(spec)
                 elif "macosx_arm64" in identifier:
                     config_update = self.macos_pypy_arm64.update_version_macos(spec)
+            elif identifier.startswith("gp"):
+                config_update = self.graalpy.update_version(identifier, spec)
         elif "win32" in identifier:
             if identifier.startswith("cp"):
                 config_update = self.windows_32.update_version_windows(spec)
@@ -272,6 +349,11 @@ def update_config(self, config: MutableMapping[str, str]) -> None:
                 config_update = self.windows_pypy_64.update_version_windows(spec)
         elif "win_arm64" in identifier and identifier.startswith("cp"):
             config_update = self.windows_arm64.update_version_windows(spec)
+        elif "linux" in identifier:
+            if identifier.startswith("gp"):
+                config_update = self.graalpy.update_version(identifier, spec)
+            else:
+                return
 
         assert config_update is not None, f"{identifier} not found!"
         config.update(**config_update)
@@ -301,6 +383,9 @@ def update_pythons(force: bool, level: str) -> None:
     with toml_file_path.open("rb") as f:
         configs = tomllib.load(f)
 
+    for config in configs["linux"]["python_configurations"]:
+        all_versions.update_config(config)
+
     for config in configs["windows"]["python_configurations"]:
         all_versions.update_config(config)
 

cibuildwheel/linux.py

@@ -8,17 +8,22 @@
 from pathlib import Path, PurePath, PurePosixPath
 from typing import Tuple
 
+from filelock import FileLock
+
 from ._compat.typing import OrderedDict, assert_never
 from .architecture import Architecture
 from .logger import log
 from .oci_container import OCIContainer
 from .options import Options
 from .typing import PathOrStr
 from .util import (
+    CIBW_CACHE_PATH,
     AlreadyBuiltWheelError,
     BuildSelector,
     NonPlatformWheelError,
     build_frontend_or_default,
+    call,
+    download,
     find_compatible_wheel,
     get_build_verbosity_extra_flags,
     prepare_command,
@@ -34,6 +39,7 @@ class PythonConfiguration:
     version: str
     identifier: str
     path_str: str
+    url: str = ""
 
     @property
     def path(self) -> PurePosixPath:
@@ -113,6 +119,44 @@ def get_build_steps(
     yield from steps.values()
 
 
+def install_python(container: OCIContainer, config: PythonConfiguration) -> bool:
+    url = config.url
+    if not url:
+        return False
+    archive = url.rsplit("/", 1)[-1]
+    parts = archive.rsplit(".", 2)
+    if parts[-1] == "zip":
+        extension = ".zip"
+    elif parts[-1] == "gz":
+        extension = ".tar.gz"
+    else:
+        extension = ".tar.bz2"
+    assert archive.endswith(extension)
+    installation_path = CIBW_CACHE_PATH / archive[: -len(extension)]
+    with FileLock(str(installation_path) + ".lock"):
+        if not installation_path.exists():
+            downloaded_archive = CIBW_CACHE_PATH / archive
+            download(url, downloaded_archive)
+            installation_path.mkdir(parents=True)
+            # strip the top level component, in case the top-level directory
+            # name is inconsistent with the archive name
+            call("tar", "-C", installation_path, "--strip-components=1", "-xzf", downloaded_archive)
+            downloaded_archive.unlink()
+            # make sure pip and wheel are available
+            bin_path = installation_path / "bin"
+            call(bin_path / "python", "-s", "-m", "ensurepip")
+            if not (bin_path / "pip").exists():
+                call("cp", bin_path / "pip3", bin_path / "pip")
+            call(bin_path / "python", "-s", "-m", "pip", "install", "wheel")
+    container.copy_into(installation_path, config.path)
+    try:
+        container.call(["test", "-x", config.path / "bin" / "python"])
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        return True
+
+
 def check_all_python_exist(
     *, platform_configs: Iterable[PythonConfiguration], container: OCIContainer
 ) -> None:
@@ -123,6 +167,8 @@ def check_all_python_exist(
         try:
             container.call(["test", "-x", python_path])
         except subprocess.CalledProcessError:
+            if install_python(container, config):
+                continue
             messages.append(
                 f"  '{python_path}' executable doesn't exist in image '{container.image}' to build '{config.identifier}'."
             )

cibuildwheel/logger.py

@@ -208,6 +208,8 @@ def build_description_from_identifier(identifier: str) -> str:
         build_description += "CPython"
     elif python_interpreter == "pp":
         build_description += "PyPy"
+    elif python_interpreter == "gp":
+        build_description += "GraalPy"
     else:
         msg = f"unknown python {python_interpreter!r}"
         raise Exception(msg)

cibuildwheel/macos.py

@@ -160,6 +160,22 @@ def install_pypy(tmp: Path, url: str) -> Path:
     return installation_path / "bin" / "pypy3"
 
 
+def install_graalpy(tmp: Path, url: str) -> Path:
+    graalpy_archive = url.rsplit("/", 1)[-1]
+    extension = ".tar.gz"
+    assert graalpy_archive.endswith(extension)
+    installation_path = CIBW_CACHE_PATH / graalpy_archive[: -len(extension)]
+    with FileLock(str(installation_path) + ".lock"):
+        if not installation_path.exists():
+            downloaded_archive = tmp / graalpy_archive
+            download(url, downloaded_archive)
+            installation_path.mkdir(parents=True)
+            # GraalPy top-folder name is inconsistent with archive name
+            call("tar", "-C", installation_path, "--strip-components=1", "-xzf", downloaded_archive)
+            downloaded_archive.unlink()
+    return installation_path / "bin" / "graalpy"
+
+
 def setup_python(
     tmp: Path,
     python_configuration: PythonConfiguration,
@@ -174,6 +190,8 @@ def setup_python(
         base_python = install_cpython(tmp, python_configuration.version, python_configuration.url)
     elif implementation_id.startswith("pp"):
         base_python = install_pypy(tmp, python_configuration.url)
+    elif implementation_id.startswith("gp"):
+        base_python = install_graalpy(tmp, python_configuration.url)
     else:
         msg = "Unknown Python implementation"
         raise ValueError(msg)

cibuildwheel/options.py

@@ -427,7 +427,7 @@ def globals(self) -> GlobalOptions:
         package_dir = args.package_dir
         output_dir = args.output_dir
 
-        build_config = self.reader.get("build", env_plat=False, sep=" ") or "*"
+        build_config = self.reader.get("build", env_plat=False, sep=" ") or "[!g]*"
         skip_config = self.reader.get("skip", env_plat=False, sep=" ")
         test_skip = self.reader.get("test-skip", env_plat=False, sep=" ")
 

cibuildwheel/resources/build-platforms.toml

@@ -18,6 +18,7 @@ python_configurations = [
   { identifier = "pp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" },
   { identifier = "pp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" },
   { identifier = "pp310-manylinux_x86_64", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" },
+  { identifier = "gp230-manylinux_x86_64", version = "3.10", path_str = "/opt/python/gp230-graalpy230_310_native_x86_64_linux", url = "https://github.com/oracle/graalpython/releases/download/graal-23.0.0/graalpython-23.0.0-linux-amd64.tar.gz" },
   { identifier = "cp36-manylinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
   { identifier = "cp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
   { identifier = "cp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" },
@@ -43,6 +44,7 @@ python_configurations = [
   { identifier = "pp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" },
   { identifier = "pp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" },
   { identifier = "pp310-manylinux_aarch64", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" },
+  { identifier = "gp230-manylinux_aarch64", version = "3.10", path_str = "/opt/python/gp230-graalpy230_310_native_aarch64_linux", url = "https://github.com/oracle/graalpython/releases/download/graal-23.0.0/graalpython-23.0.0-linux-aarch64.tar.gz" },
   { identifier = "pp37-manylinux_i686", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" },
   { identifier = "pp38-manylinux_i686", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" },
   { identifier = "pp39-manylinux_i686", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" },
@@ -110,6 +112,8 @@ python_configurations = [
   { identifier = "pp39-macosx_arm64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.12-macos_arm64.tar.bz2" },
   { identifier = "pp310-macosx_x86_64", version = "3.10", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.12-macos_x86_64.tar.bz2" },
   { identifier = "pp310-macosx_arm64", version = "3.10", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.12-macos_arm64.tar.bz2" },
+  { identifier = "gp230-macosx_x86_64", version = "3.10", url = "https://github.com/oracle/graalpython/releases/download/graal-23.0.0/graalpython-23.0.0-macos-amd64.tar.gz" },
+  { identifier = "gp230-macosx_arm64", version = "3.10", url = "https://github.com/oracle/graalpython/releases/download/graal-23.0.0/graalpython-23.0.0-macos-aarch64.tar.gz" },
 ]
 
 [windows]

test/test_abi_variants.py

@@ -39,7 +39,7 @@ def test_abi3(tmp_path):
     actual_wheels = utils.cibuildwheel_run(
         project_dir,
         add_env={
-            "CIBW_SKIP": "pp* ",  # PyPy does not have a Py_LIMITED_API equivalent
+            "CIBW_SKIP": "pp* gp*",  # PyPy and GraalPy do not have a Py_LIMITED_API equivalent
         },
     )
 

test/test_build_skip.py

@@ -7,7 +7,7 @@
 project_with_skip_asserts = test_projects.new_c_project(
     setup_py_add=textwrap.dedent(
         r"""
-        # explode if run on PyPyor Python 3.7 (these should be skipped)
+        # explode if run on PyPy or Python 3.7 (these should be skipped)
         if sys.implementation.name != "cpython":
             raise Exception("Only CPython shall be built")
         if sys.version_info[0:2] == (3, 7):

test/test_manylinuxXXXX_only.py

@@ -92,6 +92,9 @@ def test(manylinux_image, tmp_path):
     if manylinux_image == "manylinux_2_28" and platform.machine() == "x86_64":
         # We don't have a manylinux_2_28 image for i686
         add_env["CIBW_ARCHS"] = "x86_64"
+    if manylinux_image != "manylinux2014":
+        # GraalPy only works on manylinux2014
+        add_env["CIBW_SKIP"] = add_env.get("CIBW_SKIP", "") + " gp*"
 
     actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
 
@@ -126,4 +129,8 @@ def test(manylinux_image, tmp_path):
         # We don't have a manylinux_2_28 image for i686
         expected_wheels = [w for w in expected_wheels if "i686" not in w]
 
+    if manylinux_image != "manylinux2014":
+        # No GraalPy wheels on anything except manylinux2014
+        expected_wheels = [w for w in expected_wheels if "graalpy" not in w]
+
     assert set(actual_wheels) == set(expected_wheels)

test/utils.py

@@ -181,6 +181,17 @@ def expected_wheels(
                 "pp310-pypy310_pp73",
             ]
 
+        # GraalPy encodes compilation platform and arch in the tag, because it
+        # can execute native extensions compiled for different platforms
+        if machine_arch in ["x86_64", "AMD64"]:
+            if platform == "linux":
+                python_abi_tags += ["graalpy310-graalpy230_310_native_x86_64_linux-linux_i686"]
+            elif platform == "macos":
+                python_abi_tags += ["graalpy310-graalpy230_310_native_x86_64_darwin-darwin_i686"]
+
+        if machine_arch == "aarch64" and platform == "linux":
+            python_abi_tags += ["graalpy310-graalpy230_310_native_aarch64_linux-linux-aarch64"]
+
         if platform == "macos" and machine_arch == "arm64":
             # arm64 macs are only supported by cp38+
             python_abi_tags = [
@@ -192,6 +203,7 @@ def expected_wheels(
                 "pp38-pypy38_pp73",
                 "pp39-pypy39_pp73",
                 "pp310-pypy310_pp73",
+                "graalpy310-graalpy230_310_native_x86_64_linux-linux_i686",
             ]
 
     wheels = []
@@ -202,7 +214,7 @@ def expected_wheels(
         if platform == "linux":
             architectures = [arch_name_for_linux(machine_arch)]
 
-            if machine_arch == "x86_64":
+            if machine_arch == "x86_64" and not python_abi_tag.startswith("graalpy"):
                 architectures.append("i686")
 
             platform_tags = [
@@ -212,7 +224,7 @@ def expected_wheels(
                 )
                 for architecture in architectures
             ]
-            if len(musllinux_versions) > 0 and not python_abi_tag.startswith("pp"):
+            if len(musllinux_versions) > 0 and not python_abi_tag.startswith(("pp", "graalpy")):
                 platform_tags.extend(
                     [
                         ".".join(