Update CI matrix, aarch64 builds (#595)

This does the following changes: * Updates the Python versions for the OSX builds to the latest ones * Removes the version pin on wheel * setuptools 48.0.0 broke asyncpg, distutils is deprecated now, I've rewrote it to setuptools entirely * CI pipelines now have names * aarch64 tests against the latest python and postgresql are now done * manylinux2014 wheels for aarch64 are also built now * bumped Ubuntu to focal (20.04) * bumped Cython to fix compatibility with python 3.10 * CI builds now actually install the built wheel and not the latest pypi release * version test is skipped for dev builds as the git commit hash mixes up the version number * resolved all travis config warnings * postgres 9.2-9.4 are EOL and removed from the tests The manylinux containers from pypa only work on the architecture they are made for, so a aarch64 manylinux wheel can only be built on aarch64. Therefore, two containers now perform the builds. It might be worth thinking about letting the aarch64 container handle the upload alone for all wheels as it is ages slower than the x86 one anyways. I've tested pretty much everything except the S3 bucket and pypi push, the wheels themselves get built and the other pipelines pass.
MagicStack · Jul 18, 2020 · ac6a2fc · ac6a2fc
1 parent aa67d61
commit ac6a2fc
Show file tree

Hide file tree

Showing 6 changed files with 175 additions and 106 deletions.
diff --git a/.ci/build-manylinux-wheels.sh b/.ci/build-manylinux-wheels.sh
@@ -2,22 +2,25 @@
 
 set -e -x
 
+# iproute isn't included in CentOS 7
+yum install -y iproute
+
 # Compile wheels
 PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
 PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
-${PIP} install --upgrade setuptools pip wheel~=0.31.1
+${PIP} install --upgrade setuptools pip wheel
 cd /io
 make clean
 ${PYTHON} setup.py bdist_wheel
 
 # Bundle external shared libraries into the wheels.
 for whl in /io/dist/*.whl; do
-    auditwheel repair $whl -w /io/dist/
+    auditwheel repair $whl -w /tmp/
+    ${PIP} install /tmp/*.whl
+    mv /tmp/*.whl /io/dist/
     rm /io/dist/*-linux_*.whl
 done
 
-${PIP} install ${PYMODULE} -f "file:///io/dist"
-
 # Grab docker host, where Postgres should be running.
 export PGHOST=$(ip route | awk '/default/ { print $3 }' | uniq)
 export PGUSER="postgres"

diff --git a/.ci/travis-build-wheels.sh b/.ci/travis-build-wheels.sh
@@ -45,9 +45,16 @@ if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
                    s='m' if tuple('${pyver}'.split('.')) < ('3', '8') \
                      else ''))")
 
-
-        for arch in x86_64 i686; do
-            ML_IMAGE="quay.io/pypa/manylinux1_${arch}"
+        if [[ "$(uname -m)" = "x86_64" ]]; then
+            ARCHES="x86_64 i686"
+            MANYLINUX_VERSION="1"
+        elif [[ "$(uname -m)" = "aarch64" ]]; then
+            ARCHES="aarch64"
+            MANYLINUX_VERSION="2014"
+        fi
+
+        for arch in $ARCHES; do
+            ML_IMAGE="quay.io/pypa/manylinux${MANYLINUX_VERSION}_${arch}"
             docker pull "${ML_IMAGE}"
             docker run --rm \
                 -v "${_root}":/io \
@@ -64,9 +71,11 @@ elif [ "${TRAVIS_OS_NAME}" == "osx" ]; then
     export PGINSTALLATION="/usr/local/opt/postgresql@${PGVERSION}/bin"
 
     make clean
-    python setup.py bdist_wheel
+    python setup.py bdist_wheel --dist-dir /tmp/
 
-    pip install ${PYMODULE} -f "file:///${_root}/dist"
+    pip install /tmp/*.whl
+    mkdir -p "${_root}/dist"
+    mv /tmp/*.whl "${_root}/dist/"
     make -C "${_root}" ASYNCPG_VERSION="${PACKAGE_VERSION}" testinstalled
 
     _upload_wheels

diff --git a/.ci/travis-release.sh b/.ci/travis-release.sh
@@ -18,13 +18,17 @@ if [ "${PACKAGE_VERSION}" == "${PYPI_VERSION}" ]; then
 fi
 
 # Check if all expected wheels have been built and uploaded.
-release_platforms=(
-    "macosx_10_??_x86_64"
-    "manylinux1_i686"
-    "manylinux1_x86_64"
-    "win32"
-    "win_amd64"
-)
+if [[ "$(uname -m)" = "x86_64" ]]; then
+    release_platforms=(
+        "macosx_10_??_x86_64"
+        "manylinux1_i686"
+        "manylinux1_x86_64"
+        "win32"
+        "win_amd64"
+    )
+elif [[ "$(uname -m)" = "aarch64" ]]; then
+    release_platforms="manylinux2014_aarch64"
+fi
 
 P="${PYMODULE}-${PACKAGE_VERSION}"
 expected_wheels=()

diff --git a/.travis.yml b/.travis.yml
@@ -18,12 +18,6 @@ env:
 
         - DOCS_PUSH_KEY_LABEL=0760b951e99c
 
-addons:
-    apt:
-        sources:
-            - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main'
-              key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
-
 branches:
     # Avoid building PR branches.
     only:
@@ -32,132 +26,190 @@ branches:
         - releases
         - /^v\d+(\.\d+)*$/
 
-matrix:
+jobs:
     fast_finish:
         true
 
     include:
         # Do quick test runs for each supported version of PostgreSQL
         # minus the latest.
-        - os: linux
-          dist: xenial
-          language: python
-          python: "3.6"
-          env: BUILD=quicktests PGVERSION=9.2
-          addons:
-              apt: {packages: [postgresql-9.2, postgresql-contrib-9.2]}
-
-        - os: linux
-          dist: xenial
-          language: python
-          python: "3.6"
-          env: BUILD=quicktests PGVERSION=9.3
-          addons:
-              apt: {packages: [postgresql-9.3, postgresql-contrib-9.3]}
-
-        - os: linux
-          dist: xenial
+        - name: "Quicktest psql 9.5"
+          os: linux
+          dist: focal
           language: python
-          python: "3.6"
-          env: BUILD=quicktests PGVERSION=9.4
-          addons:
-              apt: {packages: [postgresql-9.4, postgresql-contrib-9.4]}
-
-        - os: linux
-          dist: xenial
-          language: python
-          python: "3.6"
+          python: "3.8"
           env: BUILD=quicktests PGVERSION=9.5
           addons:
-              apt: {packages: [postgresql-9.5, postgresql-contrib-9.5]}
-
-        - os: linux
-          dist: xenial
+              apt:
+                  sources:
+                      - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main'
+                        key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+                  packages:
+                      - postgresql-9.5
+                      - postgresql-contrib-9.5
+
+        - name: "Quicktest psql 9.6"
+          os: linux
+          dist: focal
           language: python
-          python: "3.6"
+          python: "3.8"
           env: BUILD=quicktests PGVERSION=9.6
           addons:
-              apt: {packages: [postgresql-9.6, postgresql-contrib-9.6]}
-
-        - os: linux
-          dist: xenial
+              apt:
+                  sources:
+                      - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main'
+                        key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+                  packages:
+                      - postgresql-9.6
+                      - postgresql-contrib-9.6
+
+        - name: "Quicktest psql 10"
+          os: linux
+          dist: focal
           language: python
-          python: "3.6"
+          python: "3.8"
           env: BUILD=quicktests PGVERSION=10
           addons:
-              apt: {packages: [postgresql-10]}
-
-        - os: linux
-          dist: xenial
+              apt:
+                  sources:
+                      - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main'
+                        key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+                  packages:
+                      - postgresql-10
+
+        - name: "Quicktest psql 11"
+          os: linux
+          dist: focal
           language: python
-          python: "3.6"
+          python: "3.8"
           env: BUILD=quicktests PGVERSION=11
           addons:
-              apt: {packages: [postgresql-11]}
+              apt:
+                  sources:
+                      - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main'
+                        key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+                  packages:
+                      - postgresql-11
 
         # Do a full test run on the latest supported version of PostgreSQL
         # on each supported version of Python.
-        - os: linux
-          dist: xenial
-          sudo: required
+        - name: "Test py 3.5"
+          os: linux
+          dist: focal
           language: python
           python: "3.5"
           env: BUILD=tests PGVERSION=12
           addons:
-              apt: {packages: [postgresql-12]}
-
-        - os: linux
-          dist: xenial
-          sudo: required
+              apt:
+                  sources:
+                      - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main'
+                        key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+                  packages:
+                      - postgresql-12
+
+        - name: "Test py 3.6"
+          os: linux
+          dist: focal
           language: python
           python: "3.6"
           env: BUILD=tests PGVERSION=12
           addons:
-              apt: {packages: [postgresql-12]}
-
-        - os: linux
-          dist: xenial
-          sudo: true
+              apt:
+                  sources:
+                      - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main'
+                        key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+                  packages:
+                      - postgresql-12
+
+        - name: "Test py 3.7"
+          os: linux
+          dist: focal
           language: python
           python: "3.7"
           env: BUILD=tests PGVERSION=12
           addons:
-              apt: {packages: [postgresql-12]}
-
-        - os: linux
-          dist: xenial
-          sudo: true
+              apt:
+                  sources:
+                      - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main'
+                        key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+                  packages:
+                      - postgresql-12
+
+        - name: "Test py 3.8"
+          os: linux
+          dist: focal
           language: python
           python: "3.8"
           env: BUILD=tests PGVERSION=12
           addons:
-              apt: {packages: [postgresql-12]}
+              apt:
+                  sources:
+                      - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main'
+                        key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+                  packages:
+                      - postgresql-12
+
+        # Only test on recent aarch64 distribution
+        # 3.7 is the latest supported by Travis
+        # https://docs.travis-ci.com/user/languages/python/#python-versions
+        # The shipped Postgres 9.X collides with the 12 on aarch64
+        # until fixed, use official ubuntu repos
+        - name: "Test aarch64 py 3.8-dev"
+          os: linux
+          arch: arm64
+          dist: focal
+          language: python
+          python: "3.8-dev"
+          env: BUILD=tests PGVERSION=12
+          addons:
+              postgresql: "12"
 
         # Build manylinux wheels.  Each wheel will be tested,
         # so there is no need for BUILD=tests here.
         # Also use this job to publish the releases and build
         # the documentation.
-        - os: linux
-          dist: bionic
-          sudo: required
+        - name: "x86 wheels and docs"
+          os: linux
+          dist: focal
           language: python
-          python: "3.6"
+          python: "3.8"
           env: BUILD=wheels,docs,release PGVERSION=12
           services: [docker]
           addons:
-              apt: {packages: [postgresql-12]}
+              apt:
+                  sources:
+                      - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main'
+                        key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+                  packages:
+                      - postgresql-12
+
+        # Same for the aarch64 manylinux wheel
+        - name: "aarch64 wheels"
+          os: linux
+          arch: arm64
+          dist: focal
+          language: python
+          python: "3.8-dev"
+          env: BUILD=wheels,release PGVERSION=12
+          services: [docker]
+          addons:
+              postgresql: "12"
 
-        - os: osx
-          env: BUILD=tests,wheels PYTHON_VERSION=3.5.7 PGVERSION=10
+        - name: "OSX py 3.5"
+          os: osx
+          env: BUILD=tests,wheels PYTHON_VERSION=3.5.9 PGVERSION=12
 
-        - os: osx
-          env: BUILD=tests,wheels PYTHON_VERSION=3.6.9 PGVERSION=10
+        - name: "OSX py 3.6"
+          os: osx
+          env: BUILD=tests,wheels PYTHON_VERSION=3.6.10 PGVERSION=12
 
-        - os: osx
-          env: BUILD=tests,wheels PYTHON_VERSION=3.7.4 PGVERSION=10
+        - name: "OSX py 3.7"
+          os: osx
+          env: BUILD=tests,wheels PYTHON_VERSION=3.7.7 PGVERSION=12
 
-        - os: osx
-          env: BUILD=tests,wheels PYTHON_VERSION=3.8.0 PGVERSION=10
+        - name: "OSX py 3.8"
+          os: osx
+          env: BUILD=tests,wheels PYTHON_VERSION=3.8.3 PGVERSION=12
 
 cache:
     pip