From 3ac5b9dd45c29af8fd0cc62777e2bb564faf3f6e Mon Sep 17 00:00:00 2001 From: bwoodsend Date: Sun, 28 Nov 2021 10:29:43 +0000 Subject: [PATCH] Use cibuildwheel to build wheels. This will enable the building of macOS ARM64 compatible wheels, fixing #456 (and also lets us delete lots of code!!!). --- .github/workflows/deploy-wheels-linux.yml | 89 ------------------- ...ows-macos.yml => deploy-wheels-native.yml} | 45 +++++----- .github/workflows/deploy-wheels-qemu.yml | 81 +++++++++++++++++ scripts/build-manylinux-wheels.sh | 55 ------------ scripts/build-manylinux.py | 52 ----------- 5 files changed, 101 insertions(+), 221 deletions(-) delete mode 100644 .github/workflows/deploy-wheels-linux.yml rename .github/workflows/{deploy-wheels-windows-macos.yml => deploy-wheels-native.yml} (51%) create mode 100644 .github/workflows/deploy-wheels-qemu.yml delete mode 100755 scripts/build-manylinux-wheels.sh delete mode 100755 scripts/build-manylinux.py diff --git a/.github/workflows/deploy-wheels-linux.yml b/.github/workflows/deploy-wheels-linux.yml deleted file mode 100644 index a512f15b..00000000 --- a/.github/workflows/deploy-wheels-linux.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Deploy Linux wheels - -on: - push: - branches: - - main - pull_request: - paths: - - ".github/workflows/deploy-wheels-linux.yml" - - "scripts/build-manylinux-wheels.sh" - release: - types: - - published - -jobs: - build: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - wheel: [ - "manylinux2014_aarch64", - "manylinux2014_i686", - "manylinux2014_x86_64", - "musllinux_1_1_aarch64", - "musllinux_1_1_x86_64", - ] - python-version: ["pypy3", "3.6", "3.7", "3.8", "3.9", "3.10"] - include: - # Add version-tag variable to existing jobs - - { python-version: "pypy3", version-tag: "pp37-pypy37_pp73" } - - { python-version: "3.6", version-tag: "cp36-cp36m" } - - { python-version: "3.7", version-tag: "cp37-cp37m" } - - { python-version: "3.8", version-tag: "cp38-cp38" } - - { python-version: "3.9", version-tag: "cp39-cp39" } - - { python-version: "3.10", version-tag: "cp310-cp310" } - exclude: - # No PyPy3 on musllinux - - { python-version: "pypy3", wheel: "musllinux_1_1_aarch64" } - - { python-version: "pypy3", wheel: "musllinux_1_1_x86_64" } - - steps: - - uses: actions/checkout@v2 - - run: | - git fetch --prune --unshallow - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade -q pip pytest wheel setuptools twine - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Build ${{ matrix.wheel }} wheels - run: | - docker run -v `pwd`:/io quay.io/pypa/${{ matrix.wheel }} /io/scripts/build-manylinux-wheels.sh ${{ matrix.version-tag }} - - - name: Upload as build artifacts - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist/*.whl - - - name: Publish package to PyPI - if: github.event.action == 'published' - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.pypi_password }} - run: twine upload --skip-existing dist/*.whl - - - name: Publish package to TestPyPI - if: | - github.repository == 'ultrajson/ultrajson' && - github.ref == 'refs/heads/main' - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.test_pypi_password }} - run: | - twine upload --repository-url https://test.pypi.org/legacy/ --skip-existing dist/*.whl diff --git a/.github/workflows/deploy-wheels-windows-macos.yml b/.github/workflows/deploy-wheels-native.yml similarity index 51% rename from .github/workflows/deploy-wheels-windows-macos.yml rename to .github/workflows/deploy-wheels-native.yml index 63f3a3d1..2e895dbe 100644 --- a/.github/workflows/deploy-wheels-windows-macos.yml +++ b/.github/workflows/deploy-wheels-native.yml @@ -1,4 +1,4 @@ -name: Deploy Windows/macOS wheels +name: Deploy Windows/macOS/native Linux wheels on: push: @@ -6,7 +6,7 @@ on: - main pull_request: paths: - - ".github/workflows/deploy-wheels-windows-macos.yml" + - ".github/workflows/deploy-wheels-native.yml" release: types: - published @@ -17,34 +17,29 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, macos-latest] - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] - architecture: [x64, x86] - exclude: - - {os: macos-latest, architecture: x86} + os: [windows-latest, macOS-latest, ubuntu-latest] steps: - uses: actions/checkout@v2 - - run: | - git fetch --prune --unshallow + - run: git fetch --prune --unshallow + - uses: actions/setup-python@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.architecture }} - - - name: Install dependencies - run: | - python -m pip install --upgrade -q pip pytest wheel setuptools twine + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.3.0 - - name: Test with pytest - run: | - python -m pip install . - pytest - - - name: Build wheel - run: python setup.py -q bdist_wheel + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + # Options are supplied via environment variables: + env: + # Build separate wheels for macOS's different architectures. + CIBW_ARCHS_MACOS: "x86_64 arm64" + # Build only on Linux architectures that don't need qemu emulation. + CIBW_ARCHS_LINUX: "x86_64 i686" + # Don't build with prerelease Python versions. + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6,<3.11" + # Run the test suite after each build. + CIBW_TEST_REQUIRES: "pytest" + CIBW_TEST_COMMAND: pytest {package}/tests - name: Upload as build artifacts uses: actions/upload-artifact@v2 diff --git a/.github/workflows/deploy-wheels-qemu.yml b/.github/workflows/deploy-wheels-qemu.yml new file mode 100644 index 00000000..6417f051 --- /dev/null +++ b/.github/workflows/deploy-wheels-qemu.yml @@ -0,0 +1,81 @@ +name: Deploy QEMU emulated Linux wheels + +on: + push: + branches: + - main + pull_request: + paths: + - ".github/workflows/deploy-wheels-linux.yml" + - ".github/workflows/deploy-wheels-qemu.yml" + release: + types: + - published + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + architecture: ["aarch64"] + python-version: + - pp37 + - pp38 + - cp36 + - cp37 + - cp38 + - cp39 + - cp310 + + steps: + - uses: actions/checkout@v2 + - run: git fetch --prune --unshallow + - uses: actions/setup-python@v2 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.3.0 + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + # Options are supplied via environment variables: + env: + # Build only the currently selected Linux architecture (so we can + # parallelise for speed). + CIBW_ARCHS_LINUX: "${{ matrix.architecture }}" + # Likewise, select only one Python version per job to speed this up. + CIBW_BUILD: "${{ matrix.python-version }}-*" + # Run the test suite after each build. + CIBW_TEST_REQUIRES: "pytest" + CIBW_TEST_COMMAND: pytest {package}/tests + + - name: Upload as build artifacts + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist/*.whl + + - name: Publish package to PyPI + if: github.event.action == 'published' + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.pypi_password }} + run: twine upload --skip-existing dist/*.whl + + - name: Publish package to TestPyPI + if: | + github.repository == 'ultrajson/ultrajson' && + github.ref == 'refs/heads/main' + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.test_pypi_password }} + run: | + twine upload --repository-url https://test.pypi.org/legacy/ --skip-existing dist/*.whl diff --git a/scripts/build-manylinux-wheels.sh b/scripts/build-manylinux-wheels.sh deleted file mode 100755 index 4145c9ee..00000000 --- a/scripts/build-manylinux-wheels.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -e -x - -# This is to be run by Docker inside a Docker image. -# You can test it locally on a Linux machine by installing Docker and running from this -# repo's root: -# $ docker run -v `pwd`:/io quay.io/pypa/manylinux2014_x86_64 /io/scripts/build-manylinux-wheels.sh [cp310-cp310] - -# The -v gives a directory alias for passing files in and out of the Docker. -# (/io is arbitrary). E.g the setup.py script can be accessed in the Docker via -# /io/setup.py quay.io/pypa/manylinux2014_x86_64 is the full Docker image name. Docker -# downloads it automatically. - -# The last argument is a shell command that the Docker will execute. Filenames must be -# from the Docker's perspective. - -# Wheels are initially generated as you would usually, but put in a temp directory temp-wheels. -# The pip-cache is optional but can speed up local builds having a real permanent pip-cache dir. -mkdir -p /io/pip-cache -mkdir -p /io/temp-wheels - -# Clean out any old existing wheels. -find /io/temp-wheels/ -type f -delete - -# Log the Python versions in the image. -ls /opt/python/ - -build_wheel () { - PYBIN=$1 - "${PYBIN}/pip" install -q -U setuptools wheel pytest --cache-dir /io/pip-cache - (cd /io/ && "${PYBIN}/python" -m pip install .) - (cd /io/ && "${PYBIN}/python" -m pytest) - (cd /io/ && "${PYBIN}/python" setup.py -q bdist_wheel -d /io/temp-wheels) -} - -if [ -n "$1" ]; then - # Build a single wheel - PYBIN=/opt/python/$1/bin - build_wheel ${PYBIN} -else - # Build many wheels - for PYBIN in /opt/python/cp3{6..10}*/bin; do - build_wheel ${PYBIN} - done -fi - -"$PYBIN/pip" install -q auditwheel - -# Wheels aren't considered manylinux unless they have been through -# auditwheel. Audited wheels go in /io/dist/. -mkdir -p /io/dist/ - -for whl in /io/temp-wheels/*.whl; do - auditwheel repair "$whl" -w /io/dist/ -done diff --git a/scripts/build-manylinux.py b/scripts/build-manylinux.py deleted file mode 100755 index d68202a2..00000000 --- a/scripts/build-manylinux.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import os.path -import re -import subprocess - -VERSION = re.compile(r":: Python :: (\d\.\d)$") - -EXES = [] -with open("setup.py") as f: - for line in f: - match = VERSION.search(line) - if match: - major_s, minor_s = match[1].split(".") - major, minor = int(major_s), int(minor_s) - if (major, minor) < (3, 8): - EXES.append(f"cp{major}{minor}-cp{major}{minor}m") - else: - EXES.append(f"cp{major}{minor}-cp{major}{minor}") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("version") - args = parser.parse_args() - - pkg = f"ujson=={args.version}" - - os.makedirs("dist", exist_ok=True) - for exe in EXES: - pip = f"/opt/python/{exe}/bin/pip" - - if subprocess.call( - ( - # fmt: off - "docker", "run", "--rm", - # so files are not root-owned - "--user", f"{os.getuid()}:{os.getgid()}", - "--volume", f'{os.path.abspath("dist")}:/dist:rw', - "quay.io/pypa/manylinux2014_x86_64:latest", - "bash", "-euxc", - f"{pip} wheel -w /tmp/wheels --no-deps {pkg} && " - f"auditwheel repair -w /dist /tmp/wheels/*.whl", - # fmt: on - ) - ): - return 1 - return 0 - - -if __name__ == "__main__": - exit(main())