From ac6a2fcf508f81bca3300a5a5cf59eae6537e3dd Mon Sep 17 00:00:00 2001 From: Jens Reidel Date: Sat, 18 Jul 2020 18:19:39 +0200 Subject: [PATCH] Update CI matrix, aarch64 builds (#595) This does the following changes: * Updates the Python versions for the OSX builds to the latest ones * Removes the version pin on wheel * setuptools 48.0.0 broke asyncpg, distutils is deprecated now, I've rewrote it to setuptools entirely * CI pipelines now have names * aarch64 tests against the latest python and postgresql are now done * manylinux2014 wheels for aarch64 are also built now * bumped Ubuntu to focal (20.04) * bumped Cython to fix compatibility with python 3.10 * CI builds now actually install the built wheel and not the latest pypi release * version test is skipped for dev builds as the git commit hash mixes up the version number * resolved all travis config warnings * postgres 9.2-9.4 are EOL and removed from the tests The manylinux containers from pypa only work on the architecture they are made for, so a aarch64 manylinux wheel can only be built on aarch64. Therefore, two containers now perform the builds. It might be worth thinking about letting the aarch64 container handle the upload alone for all wheels as it is ages slower than the x86 one anyways. I've tested pretty much everything except the S3 bucket and pypi push, the wheels themselves get built and the other pipelines pass. --- .ci/build-manylinux-wheels.sh | 11 +- .ci/travis-build-wheels.sh | 19 ++- .ci/travis-release.sh | 18 +-- .travis.yml | 216 +++++++++++++++++++++------------- setup.py | 15 ++- tests/test__environment.py | 2 + 6 files changed, 175 insertions(+), 106 deletions(-) diff --git a/.ci/build-manylinux-wheels.sh b/.ci/build-manylinux-wheels.sh index b5dc579b..40030785 100755 --- a/.ci/build-manylinux-wheels.sh +++ b/.ci/build-manylinux-wheels.sh @@ -2,22 +2,25 @@ set -e -x +# iproute isn't included in CentOS 7 +yum install -y iproute + # Compile wheels PYTHON="/opt/python/${PYTHON_VERSION}/bin/python" PIP="/opt/python/${PYTHON_VERSION}/bin/pip" -${PIP} install --upgrade setuptools pip wheel~=0.31.1 +${PIP} install --upgrade setuptools pip wheel cd /io make clean ${PYTHON} setup.py bdist_wheel # Bundle external shared libraries into the wheels. for whl in /io/dist/*.whl; do - auditwheel repair $whl -w /io/dist/ + auditwheel repair $whl -w /tmp/ + ${PIP} install /tmp/*.whl + mv /tmp/*.whl /io/dist/ rm /io/dist/*-linux_*.whl done -${PIP} install ${PYMODULE} -f "file:///io/dist" - # Grab docker host, where Postgres should be running. export PGHOST=$(ip route | awk '/default/ { print $3 }' | uniq) export PGUSER="postgres" diff --git a/.ci/travis-build-wheels.sh b/.ci/travis-build-wheels.sh index 6249190c..ae8c52af 100755 --- a/.ci/travis-build-wheels.sh +++ b/.ci/travis-build-wheels.sh @@ -45,9 +45,16 @@ if [ "${TRAVIS_OS_NAME}" == "linux" ]; then s='m' if tuple('${pyver}'.split('.')) < ('3', '8') \ else ''))") - - for arch in x86_64 i686; do - ML_IMAGE="quay.io/pypa/manylinux1_${arch}" + if [[ "$(uname -m)" = "x86_64" ]]; then + ARCHES="x86_64 i686" + MANYLINUX_VERSION="1" + elif [[ "$(uname -m)" = "aarch64" ]]; then + ARCHES="aarch64" + MANYLINUX_VERSION="2014" + fi + + for arch in $ARCHES; do + ML_IMAGE="quay.io/pypa/manylinux${MANYLINUX_VERSION}_${arch}" docker pull "${ML_IMAGE}" docker run --rm \ -v "${_root}":/io \ @@ -64,9 +71,11 @@ elif [ "${TRAVIS_OS_NAME}" == "osx" ]; then export PGINSTALLATION="/usr/local/opt/postgresql@${PGVERSION}/bin" make clean - python setup.py bdist_wheel + python setup.py bdist_wheel --dist-dir /tmp/ - pip install ${PYMODULE} -f "file:///${_root}/dist" + pip install /tmp/*.whl + mkdir -p "${_root}/dist" + mv /tmp/*.whl "${_root}/dist/" make -C "${_root}" ASYNCPG_VERSION="${PACKAGE_VERSION}" testinstalled _upload_wheels diff --git a/.ci/travis-release.sh b/.ci/travis-release.sh index 4b6a999a..3840c620 100755 --- a/.ci/travis-release.sh +++ b/.ci/travis-release.sh @@ -18,13 +18,17 @@ if [ "${PACKAGE_VERSION}" == "${PYPI_VERSION}" ]; then fi # Check if all expected wheels have been built and uploaded. -release_platforms=( - "macosx_10_??_x86_64" - "manylinux1_i686" - "manylinux1_x86_64" - "win32" - "win_amd64" -) +if [[ "$(uname -m)" = "x86_64" ]]; then + release_platforms=( + "macosx_10_??_x86_64" + "manylinux1_i686" + "manylinux1_x86_64" + "win32" + "win_amd64" + ) +elif [[ "$(uname -m)" = "aarch64" ]]; then + release_platforms="manylinux2014_aarch64" +fi P="${PYMODULE}-${PACKAGE_VERSION}" expected_wheels=() diff --git a/.travis.yml b/.travis.yml index 1ace44f9..680f91f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,12 +18,6 @@ env: - DOCS_PUSH_KEY_LABEL=0760b951e99c -addons: - apt: - sources: - - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main' - key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' - branches: # Avoid building PR branches. only: @@ -32,132 +26,190 @@ branches: - releases - /^v\d+(\.\d+)*$/ -matrix: +jobs: fast_finish: true include: # Do quick test runs for each supported version of PostgreSQL # minus the latest. - - os: linux - dist: xenial - language: python - python: "3.6" - env: BUILD=quicktests PGVERSION=9.2 - addons: - apt: {packages: [postgresql-9.2, postgresql-contrib-9.2]} - - - os: linux - dist: xenial - language: python - python: "3.6" - env: BUILD=quicktests PGVERSION=9.3 - addons: - apt: {packages: [postgresql-9.3, postgresql-contrib-9.3]} - - - os: linux - dist: xenial + - name: "Quicktest psql 9.5" + os: linux + dist: focal language: python - python: "3.6" - env: BUILD=quicktests PGVERSION=9.4 - addons: - apt: {packages: [postgresql-9.4, postgresql-contrib-9.4]} - - - os: linux - dist: xenial - language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=9.5 addons: - apt: {packages: [postgresql-9.5, postgresql-contrib-9.5]} - - - os: linux - dist: xenial + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-9.5 + - postgresql-contrib-9.5 + + - name: "Quicktest psql 9.6" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=9.6 addons: - apt: {packages: [postgresql-9.6, postgresql-contrib-9.6]} - - - os: linux - dist: xenial + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-9.6 + - postgresql-contrib-9.6 + + - name: "Quicktest psql 10" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=10 addons: - apt: {packages: [postgresql-10]} - - - os: linux - dist: xenial + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-10 + + - name: "Quicktest psql 11" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=11 addons: - apt: {packages: [postgresql-11]} + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-11 # Do a full test run on the latest supported version of PostgreSQL # on each supported version of Python. - - os: linux - dist: xenial - sudo: required + - name: "Test py 3.5" + os: linux + dist: focal language: python python: "3.5" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} - - - os: linux - dist: xenial - sudo: required + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + - name: "Test py 3.6" + os: linux + dist: focal language: python python: "3.6" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} - - - os: linux - dist: xenial - sudo: true + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + - name: "Test py 3.7" + os: linux + dist: focal language: python python: "3.7" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} - - - os: linux - dist: xenial - sudo: true + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + - name: "Test py 3.8" + os: linux + dist: focal language: python python: "3.8" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + # Only test on recent aarch64 distribution + # 3.7 is the latest supported by Travis + # https://docs.travis-ci.com/user/languages/python/#python-versions + # The shipped Postgres 9.X collides with the 12 on aarch64 + # until fixed, use official ubuntu repos + - name: "Test aarch64 py 3.8-dev" + os: linux + arch: arm64 + dist: focal + language: python + python: "3.8-dev" + env: BUILD=tests PGVERSION=12 + addons: + postgresql: "12" # Build manylinux wheels. Each wheel will be tested, # so there is no need for BUILD=tests here. # Also use this job to publish the releases and build # the documentation. - - os: linux - dist: bionic - sudo: required + - name: "x86 wheels and docs" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=wheels,docs,release PGVERSION=12 services: [docker] addons: - apt: {packages: [postgresql-12]} + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + # Same for the aarch64 manylinux wheel + - name: "aarch64 wheels" + os: linux + arch: arm64 + dist: focal + language: python + python: "3.8-dev" + env: BUILD=wheels,release PGVERSION=12 + services: [docker] + addons: + postgresql: "12" - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.5.7 PGVERSION=10 + - name: "OSX py 3.5" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.5.9 PGVERSION=12 - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.6.9 PGVERSION=10 + - name: "OSX py 3.6" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.6.10 PGVERSION=12 - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.7.4 PGVERSION=10 + - name: "OSX py 3.7" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.7.7 PGVERSION=12 - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.8.0 PGVERSION=10 + - name: "OSX py 3.8" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.8.3 PGVERSION=12 cache: pip diff --git a/setup.py b/setup.py index 5e2e1494..5bb9e693 100644 --- a/setup.py +++ b/setup.py @@ -19,15 +19,13 @@ # We use vanilla build_ext, to avoid importing Cython via # the setuptools version. -from distutils import extension as distutils_extension -from distutils.command import build_ext as distutils_build_ext - import setuptools from setuptools.command import build_py as setuptools_build_py from setuptools.command import sdist as setuptools_sdist +from setuptools.command import build_ext as setuptools_build_ext -CYTHON_DEPENDENCY = 'Cython==0.29.14' +CYTHON_DEPENDENCY = 'Cython==0.29.20' # Minimal dependencies required to test asyncpg. TEST_DEPENDENCIES = [ @@ -138,9 +136,9 @@ def build_module(self, module, module_file, package): return outfile, copied -class build_ext(distutils_build_ext.build_ext): +class build_ext(setuptools_build_ext.build_ext): - user_options = distutils_build_ext.build_ext.user_options + [ + user_options = setuptools_build_ext.build_ext.user_options + [ ('cython-always', None, 'run cythonize() even if .c files are present'), ('cython-annotate', None, @@ -264,6 +262,7 @@ def finalize_options(self): 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Database :: Front-Ends', ], @@ -278,13 +277,13 @@ def finalize_options(self): provides=['asyncpg'], include_package_data=True, ext_modules=[ - distutils_extension.Extension( + setuptools.extension.Extension( "asyncpg.pgproto.pgproto", ["asyncpg/pgproto/pgproto.pyx"], extra_compile_args=CFLAGS, extra_link_args=LDFLAGS), - distutils_extension.Extension( + setuptools.extension.Extension( "asyncpg.protocol.protocol", ["asyncpg/protocol/record/recordobj.c", "asyncpg/protocol/protocol.pyx"], diff --git a/tests/test__environment.py b/tests/test__environment.py index 878fb2fc..3141fe8b 100644 --- a/tests/test__environment.py +++ b/tests/test__environment.py @@ -30,6 +30,8 @@ async def test_environment_server_version(self): @unittest.skipIf(not os.environ.get('ASYNCPG_VERSION'), "environ[ASYNCPG_VERSION] is not set") + @unittest.skipIf("dev" in asyncpg.__version__, + "development version with git commit data") async def test_environment_asyncpg_version(self): apgver = os.environ.get('ASYNCPG_VERSION') self.assertEqual(