From adfdc1bb091a20a0433eb93a952914dd0b44014e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Dudfield?= Date: Sat, 23 Oct 2021 13:14:28 +0200 Subject: [PATCH] manylinux: Fix selection of manylinux to build So that more recent manylinux2010 and 2014 wheels are made in addition to the older manylinux1 builds. --- .github/workflows/manylinux.yml | 8 +- buildconfig/manylinux-build/Makefile | 92 +++++++++++-------- buildconfig/manylinux-build/build-wheels.sh | 15 +-- .../docker_base/Dockerfile-aarch64 | 19 ++-- .../docker_base/Dockerfile-i686 | 20 ++-- .../docker_base/Dockerfile-x86_64 | 24 ++--- 6 files changed, 97 insertions(+), 81 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 8297b2ee1a..80031a4cc6 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -33,12 +33,14 @@ jobs: fail-fast: false # if a particular matrix build fails, don't skip the rest matrix: arch: ['x64', 'x86', 'aarch64'] - tag: ['-', '-manylinux-'] # one tag for CPython builds, and another for PyPy + tag: ['-manylinux1-', '-manylinux2010-', -manylinux2014-] exclude: - # no pypy builds on aarch64 for now + # aarch64 only with manylinux2014 for now - arch: 'aarch64' - tag: '-manylinux-' + tag: '-manylinux1-' + - arch: 'aarch64' + tag: '-manylinux2010-' steps: - uses: actions/checkout@v2.3.4 diff --git a/buildconfig/manylinux-build/Makefile b/buildconfig/manylinux-build/Makefile index 8d72392301..68d7f766bb 100644 --- a/buildconfig/manylinux-build/Makefile +++ b/buildconfig/manylinux-build/Makefile @@ -1,87 +1,101 @@ REPO_ROOT = $(abspath ../..) -wheels-manylinux-x64: +wheels-manylinux1-x64: docker run --rm -v ${REPO_ROOT}:/io pygame/manylinux1_base_x86_64 /io/buildconfig/manylinux-build/build-wheels.sh -wheels-manylinux-x86: +wheels-manylinux1-x86: docker run --rm -v ${REPO_ROOT}:/io pygame/manylinux1_base_i686 /io/buildconfig/manylinux-build/build-wheels.sh -wheels-x64: +wheels-manylinux2010-x64: docker run --rm -v ${REPO_ROOT}:/io pygame/manylinux2010_base_x86_64 /io/buildconfig/manylinux-build/build-wheels.sh buildpypy -wheels-x86: +wheels-manylinux2010-x86: docker run --rm -v ${REPO_ROOT}:/io pygame/manylinux2010_base_i686 /io/buildconfig/manylinux-build/build-wheels.sh buildpypy -wheels-aarch64: +wheels-manylinux2014-x64: + docker run --rm -v ${REPO_ROOT}:/io pygame/manylinux2014_base_x86_64 /io/buildconfig/manylinux-build/build-wheels.sh buildpypy + +wheels-manylinux2014-x86: + docker run --rm -v ${REPO_ROOT}:/io pygame/manylinux2014_base_i686 /io/buildconfig/manylinux-build/build-wheels.sh buildpypy + +wheels-manylinux2014-aarch64: docker run --rm -v ${REPO_ROOT}:/io pygame/manylinux2014_base_aarch64 /io/buildconfig/manylinux-build/build-wheels.sh -wheels: wheels-x64 wheels-x86 wheels-aarch64 -wheels-manylinux: wheels-manylinux-x64 wheels-manylinux-x86 +wheels: wheels-manylinux1-x64 wheels-manylinux1-x86 wheels-manylinux2010-x64 wheels-manylinux2010-x86 wheels-manylinux2014-x64 wheels-manylinux2014-x86 wheels-manylinux2014-aarch64 -base-image-manylinux-x64: +base-image-manylinux1-x64: docker build --build-arg BASE_IMAGE=manylinux1_x86_64 --build-arg BASE_IMAGE2=manylinux1_x86_64 -t pygame/manylinux1_base_x86_64 -f docker_base/Dockerfile-x86_64 docker_base -base-image-manylinux-x86: +base-image-manylinux1-x86: docker build --build-arg BASE_IMAGE=manylinux1_i686 --build-arg BASE_IMAGE2=manylinux1_i686 -t pygame/manylinux1_base_i686 -f docker_base/Dockerfile-i686 docker_base -base-images-manylinux: base-image-manylinux-x64 base-image-manylinux-x86 - - +base-image-manylinux2010-x64: + docker build -t pygame/manylinux2010_base_x86_64 -f docker_base/Dockerfile-x86_64 docker_base --build-arg BASE_IMAGE=manylinux2010_x86_64 --build-arg BASE_IMAGE2=manylinux2010_x86_64 --progress=plain -base-image-x64: - docker build -t pygame/manylinux2010_base_x86_64 -f docker_base/Dockerfile-x86_64 docker_base --build-arg BASE_IMAGE=manylinux2010_x86_64 --build-arg BASE_IMAGE2=manylinux2010_x86_64 - -base-image-x86: +base-image-manylinux2010-x86: docker build -t pygame/manylinux2010_base_i686 -f docker_base/Dockerfile-i686 docker_base --build-arg BASE_IMAGE=manylinux2010_i686 --build-arg BASE_IMAGE2=manylinux2010_i686 -base-image-aarch64: - docker build -t pygame/manylinux2014_base_aarch64 -f docker_base/Dockerfile-aarch64 docker_base --build-arg BASE_IMAGE=manylinux2014_aarch64 --build-arg BASE_IMAGE2=manylinux2014_aarch64 - -base-images: base-image-x64 base-image-x86 base-image-aarch64 - +base-image-manylinux2014-x64: + docker build -t pygame/manylinux2014_base_x86_64 -f docker_base/Dockerfile-x86_64 docker_base --build-arg BASE_IMAGE=manylinux2014_x86_64 --build-arg BASE_IMAGE2=manylinux2014_x86_64 --progress=plain +base-image-manylinux2014-x86: + docker build -t pygame/manylinux2014_base_i686 -f docker_base/Dockerfile-i686 docker_base --build-arg BASE_IMAGE=manylinux2014_i686 --build-arg BASE_IMAGE2=manylinux2014_i686 +base-image-manylinux2014-aarch64: + docker build -t pygame/manylinux2014_base_aarch64 -f docker_base/Dockerfile-aarch64 docker_base --build-arg BASE_IMAGE=manylinux2014_aarch64 --build-arg BASE_IMAGE2=manylinux2014_aarch64 -push-manylinux-x64: - docker push pygame/manylinux1_base_x86_64 +# base-images: base-image-manylinux1-x64 base-image-manylinux1-x86 base-image-manylinux2010-x64 base-image-manylinux2010-x86 base-image-manylinux2014-x64 base-image-manylinux2014-x86 base-image-aarch64 +base-images: base-image-manylinux2010-x64 base-image-manylinux2010-x86 base-image-manylinux2014-x64 base-image-manylinux2014-x86 base-image-manylinux2014-aarch64 -push-manylinux-x86: - docker push pygame/manylinux1_base_i686 -push-manylinux: push-manylinux-x64 push-manylinux-x86 +# push-manylinux1-x64: +# docker push pygame/manylinux1_base_x86_64 +# push-manylinux1-x86: +# docker push pygame/manylinux1_base_i686 -push-x64: +push-manylinux2010-x64: docker push pygame/manylinux2010_base_x86_64 -push-x86: +push-manylinux2010-x86: docker push pygame/manylinux2010_base_i686 -push-aarch64: +push-manylinux2014-x64: + docker push pygame/manylinux2014_base_x86_64 + +push-manylinux2014-x86: + docker push pygame/manylinux2014_base_i686 + +push-manylinux2014-aarch64: docker push pygame/manylinux2014_base_aarch64 -push: push-x64 push-x86 push-aarch64 - +# push: push-manylinux1-x64 push-manylinux1-x86 push-manylinux2010-x64 push-manylinux2010-x86 push-manylinux2014-x64 push-manylinux2014-x86 +push: push-manylinux2010-x64 push-manylinux2010-x86 push-manylinux2014-x64 push-manylinux2014-x86 push-manylinux2014-aarch64 -pull-manylinux-x64: +pull-manylinux1-x64: docker pull pygame/manylinux1_base_x86_64 -pull-manylinux-x86: +pull-manylinux1-x86: docker pull pygame/manylinux1_base_i686 -pull-manylinux: pull-manylinux-x64 pull-manylinux-x86 +pull-manylinux2010-x64: + docker pull pygame/manylinux1_base_x86_64 + +pull-manylinux2010-x86: + docker pull pygame/manylinux2010_base_i686 -pull-x64: - docker pull pygame/manylinux2010_base_x86_64 +pull-manylinux2014-x64: + docker pull pygame/manylinux1_base_x86_64 -pull-x86: +pull-manylinux2014-x86: docker pull pygame/manylinux2010_base_i686 -pull-aarch64: +pull-manylinux2014-aarch64: docker pull pygame/manylinux2014_base_aarch64 -pull: pull-x64 pull-x86 pull-aarch64 +pull: pull-manylinux1-x64 pull-manylinux1-x86 pull-manylinux2010-x64 pull-manylinux2010-x86 pull-manylinux2014-x64 pull-manylinux2014-x86 pull-manylinux2014-aarch64 +# pull: pull-manylinux2010-x64 pull-manylinux2010-x86 pull-manylinux2014-x64 pull-manylinux2014-x86 pull-aarch64 diff --git a/buildconfig/manylinux-build/build-wheels.sh b/buildconfig/manylinux-build/build-wheels.sh index 59efb35d71..bcfbc7f05e 100755 --- a/buildconfig/manylinux-build/build-wheels.sh +++ b/buildconfig/manylinux-build/build-wheels.sh @@ -1,14 +1,15 @@ #!/bin/bash set -e -x -if [ `uname -m` == "aarch64" ]; then - export SUPPORTED_PYTHONS="cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310" -else - export SUPPORTED_PYTHONS="cp27-cp27mu cp35-cp35m cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39" -fi if [[ "$1" == "buildpypy" ]]; then - export SUPPORTED_PYTHONS="pp27-pypy_75 pp37-pypy37_pp75 cp310-cp310" + export SUPPORTED_PYTHONS="cp27-cp27mu cp35-cp35m cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 pp27-pypy_75 pp37-pypy37_pp75" +else + if [ `uname -m` == "aarch64" ]; then + export SUPPORTED_PYTHONS="cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310" + else + export SUPPORTED_PYTHONS="cp27-cp27mu cp35-cp35m cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39" + fi fi @@ -39,7 +40,7 @@ for PYVER in $SUPPORTED_PYTHONS; do PYTHON="/opt/python/${PYVER}/bin/pypy" fi - ${PYTHON} -m pip wheel --global-option="build_ext" --global-option="-j4" -vvv /io/ -w wheelhouse/ + ${PYTHON} -m pip wheel -global-option="build_ext" --global-option="-j4" -vvv /io/ -w wheelhouse/ done # Bundle external shared libraries into the wheels diff --git a/buildconfig/manylinux-build/docker_base/Dockerfile-aarch64 b/buildconfig/manylinux-build/docker_base/Dockerfile-aarch64 index 5ef8fb0daa..752b75048a 100644 --- a/buildconfig/manylinux-build/docker_base/Dockerfile-aarch64 +++ b/buildconfig/manylinux-build/docker_base/Dockerfile-aarch64 @@ -1,6 +1,6 @@ ARG BASE_IMAGE=manylinux2014_aarch64 FROM quay.io/pypa/$BASE_IMAGE -ENV MAKEFLAGS="-j 8" +ENV MAKEFLAGS="-j 16" # Set up repoforge COPY RPM-GPG-KEY.dag.txt /tmp/ @@ -12,7 +12,7 @@ RUN rpm --import /tmp/RPM-GPG-KEY.dag.txt #RUN rpm -i /tmp/${RPMFORGE_FILE} # Install SDL and portmidi dependencies -RUN yum install -y libX11-devel\ +RUN yum install -y zlib-devel libX11-devel\ mesa-libGLU-devel audiofile-devel \ java-1.7.0-openjdk-devel jpackage-utils \ giflib-devel dbus-devel \ @@ -24,14 +24,14 @@ RUN yum install -y libcap-devel libxkbcommon-devel ADD brotli /brotli_build/ RUN ["bash", "/brotli_build/build-brotli.sh"] -ADD bzip2 /bzip2_build/ -RUN ["bash", "/bzip2_build/build-bzip2.sh"] +#ADD bzip2 /bzip2_build/ +#RUN ["bash", "/bzip2_build/build-bzip2.sh"] -ADD zlib-ng /zlib-ng_build/ -RUN ["bash", "/zlib-ng_build/build-zlib-ng.sh"] +#ADD zlib-ng /zlib-ng_build/ +#RUN ["bash", "/zlib-ng_build/build-zlib-ng.sh"] ADD libjpegturbo /libjpegturbo_build/ -RUN ["bash", "/libjpegturbo_build/build-libjpegturbo.sh"] +RUN ["bash", "/libjpegturbo_build/build-jpeg-turbo.sh"] ADD libpng /libpng_build/ RUN ["bash", "/libpng_build/build-png.sh"] @@ -51,11 +51,10 @@ RUN ["bash", "/opus_build/build-opus.sh"] ADD freetype /freetype_build/ RUN ["bash", "/freetype_build/build-freetype.sh"] +RUN ["rm", "-f", "/lib64/libasound*"] +RUN ["rm", "-f", "/lib/libasound*"] ADD alsa /alsa_build/ RUN ["bash", "/alsa_build/build-alsa.sh"] -# Replace yum-installed libasound with the one we just compiled. -RUN ["rm", "/lib64/libasound.so.2.0.0"] -RUN ["ln", "-s", "/usr/lib/libasound.so.2.0.0", "/lib64/"] ADD ogg /ogg_build/ RUN ["bash", "/ogg_build/build-ogg.sh"] diff --git a/buildconfig/manylinux-build/docker_base/Dockerfile-i686 b/buildconfig/manylinux-build/docker_base/Dockerfile-i686 index e2bcb6bec0..219576983d 100644 --- a/buildconfig/manylinux-build/docker_base/Dockerfile-i686 +++ b/buildconfig/manylinux-build/docker_base/Dockerfile-i686 @@ -1,6 +1,6 @@ ARG BASE_IMAGE=manylinux1_i686 FROM quay.io/pypa/$BASE_IMAGE -ENV MAKEFLAGS="-j 4" +ENV MAKEFLAGS="-j 16" # Set up repoforge COPY RPM-GPG-KEY.dag.txt /tmp/ @@ -12,7 +12,7 @@ RUN rpm --import /tmp/RPM-GPG-KEY.dag.txt #RUN rpm -i /tmp/${RPMFORGE_FILE} # Install SDL and portmidi dependencies -RUN linux32 yum install -y libX11-devel \ +RUN linux32 yum install -y zlib-devel libX11-devel \ mesa-libGLU-devel audiofile-devel \ java-1.7.0-openjdk-devel jpackage-utils \ giflib-devel @@ -25,14 +25,14 @@ RUN linux32 yum install -y libcap-devel libxkbcommon-devel ADD brotli /brotli_build/ RUN ["linux32", "bash", "/brotli_build/build-brotli.sh"] -ADD bzip2 /bzip2_build/ -RUN ["linux32", "bash", "/bzip2_build/build-bzip2.sh"] +#ADD bzip2 /bzip2_build/ +#RUN ["linux32", "bash", "/bzip2_build/build-bzip2.sh"] -ADD zlib-ng /zlib-ng_build/ -RUN ["linux32", "bash", "/zlib-ng_build/build-zlib-ng.sh"] +#ADD zlib-ng /zlib-ng_build/ +#RUN ["linux32", "bash", "/zlib-ng_build/build-zlib-ng.sh"] ADD libjpegturbo /libjpegturbo_build/ -RUN ["linux32", "bash", "/libjpegturbo_build/build-libjpegturbo.sh"] +RUN ["linux32", "bash", "/libjpegturbo_build/build-jpeg-turbo.sh"] ADD libpng /libpng_build/ RUN ["linux32", "bash", "/libpng_build/build-png.sh"] @@ -52,11 +52,10 @@ RUN ["linux32", "bash", "/opus_build/build-opus.sh"] ADD freetype /freetype_build/ RUN ["linux32", "bash", "/freetype_build/build-freetype.sh"] +# Replace yum-installed libasound with the one we just compiled. +RUN ["linux32", "rm", "-f", "/usr/lib/libasound*"] ADD alsa /alsa_build/ RUN ["linux32", "bash", "/alsa_build/build-alsa.sh"] -# Replace yum-installed libasound with the one we just compiled. -RUN ["rm", "/lib/libasound.so.2.0.0"] -RUN ["ln", "-s", "/usr/lib/libasound.so.2.0.0", "/lib/"] ADD ogg /ogg_build/ RUN ["linux32", "bash", "/ogg_build/build-ogg.sh"] @@ -96,3 +95,4 @@ RUN ["linux32", "bash", "/portmidi_build/build-portmidi.sh"] ADD pypy /pypy_build/ ARG BASE_IMAGE2=manylinux1_i686 RUN if [ "$BASE_IMAGE2" = "manylinux2010_i686" ] ; then linux32 bash /pypy_build/getpypy32.sh ; else echo "no pypy on manylinux1" ; fi +RUN if [ "$BASE_IMAGE2" = "manylinux2014_i686" ] ; then linux32 bash /pypy_build/getpypy32.sh ; else echo "no pypy on manylinux1" ; fi diff --git a/buildconfig/manylinux-build/docker_base/Dockerfile-x86_64 b/buildconfig/manylinux-build/docker_base/Dockerfile-x86_64 index a17664dc57..28ad91c770 100644 --- a/buildconfig/manylinux-build/docker_base/Dockerfile-x86_64 +++ b/buildconfig/manylinux-build/docker_base/Dockerfile-x86_64 @@ -1,6 +1,6 @@ ARG BASE_IMAGE=manylinux1_x86_64 FROM quay.io/pypa/$BASE_IMAGE -ENV MAKEFLAGS="-j 4" +ENV MAKEFLAGS="-j 16" # Set up repoforge COPY RPM-GPG-KEY.dag.txt /tmp/ @@ -12,10 +12,10 @@ RUN rpm --import /tmp/RPM-GPG-KEY.dag.txt #RUN rpm -i /tmp/${RPMFORGE_FILE} # Install SDL and portmidi dependencies -RUN yum install -y libX11-devel\ +RUN yum install -y zlib-devel libX11-devel\ mesa-libGLU-devel audiofile-devel \ - java-1.7.0-openjdk-devel jpackage-utils \ - mikmod-devel smpeg-devel giflib-devel dbus-devel \ + jpackage-utils \ + mikmod-devel giflib-devel dbus-devel \ dejavu-sans-fonts fontconfig \ libXcursor-devel libXi-devel libXxf86vm-devel \ libXrandr-devel libXinerama-devel libXcomposite-devel mesa-libGLU-devel xz @@ -24,14 +24,14 @@ RUN yum install -y libcap-devel libxkbcommon-devel ADD brotli /brotli_build/ RUN ["bash", "/brotli_build/build-brotli.sh"] -ADD bzip2 /bzip2_build/ -RUN ["bash", "/bzip2_build/build-bzip2.sh"] +#ADD bzip2 /bzip2_build/ +#RUN ["bash", "/bzip2_build/build-bzip2.sh"] -ADD zlib-ng /zlib-ng_build/ -RUN ["bash", "/zlib-ng_build/build-zlib-ng.sh"] +#ADD zlib-ng /zlib-ng_build/ +#RUN ["bash", "/zlib-ng_build/build-zlib-ng.sh"] ADD libjpegturbo /libjpegturbo_build/ -RUN ["bash", "/libjpegturbo_build/build-libjpegturbo.sh"] +RUN ["bash", "/libjpegturbo_build/build-jpeg-turbo.sh"] ADD libpng /libpng_build/ RUN ["bash", "/libpng_build/build-png.sh"] @@ -51,11 +51,10 @@ RUN ["bash", "/opus_build/build-opus.sh"] ADD freetype /freetype_build/ RUN ["bash", "/freetype_build/build-freetype.sh"] +RUN ["rm", "-f", "/lib64/libasound*"] +RUN ["rm", "-f", "/lib/libasound*"] ADD alsa /alsa_build/ RUN ["bash", "/alsa_build/build-alsa.sh"] -# Replace yum-installed libasound with the one we just compiled. -RUN ["rm", "/lib64/libasound.so.2.0.0"] -RUN ["ln", "-s", "/usr/lib/libasound.so.2.0.0", "/lib64/"] ADD ogg /ogg_build/ RUN ["bash", "/ogg_build/build-ogg.sh"] @@ -98,4 +97,5 @@ RUN echo "$BASE_IMAGE" ADD pypy /pypy_build/ ARG BASE_IMAGE2=manylinux1_x86_64 RUN if [ "$BASE_IMAGE2" = "manylinux2010_x86_64" ] ; then bash /pypy_build/getpypy64.sh ; else echo "no pypy on manylinux1" ; fi +RUN if [ "$BASE_IMAGE2" = "manylinux2014_x86_64" ] ; then bash /pypy_build/getpypy64.sh ; else echo "no pypy on manylinux1" ; fi