From 6dec022975e8f6721cbc80a5a8c34e30ad9152ed Mon Sep 17 00:00:00 2001 From: Jens Reidel Date: Sat, 4 Jul 2020 18:43:40 +0200 Subject: [PATCH 1/2] Update CI builds --- .ci/build-manylinux-wheels.sh | 11 +- .ci/travis-before-install.sh | 2 +- .ci/travis-build-wheels.sh | 19 ++- .ci/travis-release.sh | 18 +-- .travis.yml | 216 +++++++++++++++++++++------------- asyncpg/__init__.py | 2 +- asyncpg/connect_utils.py | 13 +- asyncpg/connection.py | 9 +- asyncpg/pgproto | 2 +- setup.py | 15 ++- tests/test__environment.py | 2 + tests/test_connect.py | 38 ++++++ tests/test_copy.py | 43 +++++++ 13 files changed, 279 insertions(+), 111 deletions(-) diff --git a/.ci/build-manylinux-wheels.sh b/.ci/build-manylinux-wheels.sh index b5dc579b..40030785 100755 --- a/.ci/build-manylinux-wheels.sh +++ b/.ci/build-manylinux-wheels.sh @@ -2,22 +2,25 @@ set -e -x +# iproute isn't included in CentOS 7 +yum install -y iproute + # Compile wheels PYTHON="/opt/python/${PYTHON_VERSION}/bin/python" PIP="/opt/python/${PYTHON_VERSION}/bin/pip" -${PIP} install --upgrade setuptools pip wheel~=0.31.1 +${PIP} install --upgrade setuptools pip wheel cd /io make clean ${PYTHON} setup.py bdist_wheel # Bundle external shared libraries into the wheels. for whl in /io/dist/*.whl; do - auditwheel repair $whl -w /io/dist/ + auditwheel repair $whl -w /tmp/ + ${PIP} install /tmp/*.whl + mv /tmp/*.whl /io/dist/ rm /io/dist/*-linux_*.whl done -${PIP} install ${PYMODULE} -f "file:///io/dist" - # Grab docker host, where Postgres should be running. export PGHOST=$(ip route | awk '/default/ { print $3 }' | uniq) export PGUSER="postgres" diff --git a/.ci/travis-before-install.sh b/.ci/travis-before-install.sh index 3dd6b8a8..dd624ffa 100755 --- a/.ci/travis-before-install.sh +++ b/.ci/travis-before-install.sh @@ -27,7 +27,7 @@ if [[ "${TRAVIS_OS_NAME}" == "linux" && "${BUILD}" == *wheels* ]]; then fi fi - sudo service postgresql start ${PGVERSION} + sudo pg_ctlcluster ${PGVERSION} main restart fi if [ "${TRAVIS_OS_NAME}" == "osx" ]; then diff --git a/.ci/travis-build-wheels.sh b/.ci/travis-build-wheels.sh index 6249190c..11445af2 100755 --- a/.ci/travis-build-wheels.sh +++ b/.ci/travis-build-wheels.sh @@ -3,7 +3,7 @@ set -e -x -if [[ "${TRAVIS_BRANCH}" != "releases" || "${BUILD}" != *wheels* ]]; then +if [[ "${TRAVIS_BRANCH}" != "release" || "${BUILD}" != *wheels* ]]; then # Not a release exit 0 fi @@ -45,9 +45,16 @@ if [ "${TRAVIS_OS_NAME}" == "linux" ]; then s='m' if tuple('${pyver}'.split('.')) < ('3', '8') \ else ''))") + if [[ "$(uname -m)" = "x86_64" ]]; then + ARCHES="x86_64 i686" + MANYLINUX_VERSION="1" + elif [[ "$(uname -m)" = "aarch64" ]]; then + ARCHES="aarch64" + MANYLINUX_VERSION="2014" + fi - for arch in x86_64 i686; do - ML_IMAGE="quay.io/pypa/manylinux1_${arch}" + for arch in $ARCHES; do + ML_IMAGE="quay.io/pypa/manylinux${MANYLINUX_VERSION}_${arch}" docker pull "${ML_IMAGE}" docker run --rm \ -v "${_root}":/io \ @@ -64,9 +71,11 @@ elif [ "${TRAVIS_OS_NAME}" == "osx" ]; then export PGINSTALLATION="/usr/local/opt/postgresql@${PGVERSION}/bin" make clean - python setup.py bdist_wheel + python setup.py bdist_wheel --dist-dir /tmp/ - pip install ${PYMODULE} -f "file:///${_root}/dist" + pip install /tmp/*.whl + mkdir -p "${_root}/dist" + mv /tmp/*.whl "${_root}/dist/" make -C "${_root}" ASYNCPG_VERSION="${PACKAGE_VERSION}" testinstalled _upload_wheels diff --git a/.ci/travis-release.sh b/.ci/travis-release.sh index 4b6a999a..3840c620 100755 --- a/.ci/travis-release.sh +++ b/.ci/travis-release.sh @@ -18,13 +18,17 @@ if [ "${PACKAGE_VERSION}" == "${PYPI_VERSION}" ]; then fi # Check if all expected wheels have been built and uploaded. -release_platforms=( - "macosx_10_??_x86_64" - "manylinux1_i686" - "manylinux1_x86_64" - "win32" - "win_amd64" -) +if [[ "$(uname -m)" = "x86_64" ]]; then + release_platforms=( + "macosx_10_??_x86_64" + "manylinux1_i686" + "manylinux1_x86_64" + "win32" + "win_amd64" + ) +elif [[ "$(uname -m)" = "aarch64" ]]; then + release_platforms="manylinux2014_aarch64" +fi P="${PYMODULE}-${PACKAGE_VERSION}" expected_wheels=() diff --git a/.travis.yml b/.travis.yml index be029141..680f91f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,12 +18,6 @@ env: - DOCS_PUSH_KEY_LABEL=0760b951e99c -addons: - apt: - sources: - - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main' - key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' - branches: # Avoid building PR branches. only: @@ -32,132 +26,190 @@ branches: - releases - /^v\d+(\.\d+)*$/ -matrix: +jobs: fast_finish: true include: # Do quick test runs for each supported version of PostgreSQL # minus the latest. - - os: linux - dist: xenial - language: python - python: "3.6" - env: BUILD=quicktests PGVERSION=9.2 - addons: - apt: {packages: [postgresql-9.2, postgresql-contrib-9.2]} - - - os: linux - dist: xenial - language: python - python: "3.6" - env: BUILD=quicktests PGVERSION=9.3 - addons: - apt: {packages: [postgresql-9.3, postgresql-contrib-9.3]} - - - os: linux - dist: xenial + - name: "Quicktest psql 9.5" + os: linux + dist: focal language: python - python: "3.6" - env: BUILD=quicktests PGVERSION=9.4 - addons: - apt: {packages: [postgresql-9.4, postgresql-contrib-9.4]} - - - os: linux - dist: xenial - language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=9.5 addons: - apt: {packages: [postgresql-9.5, postgresql-contrib-9.5]} - - - os: linux - dist: xenial + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-9.5 + - postgresql-contrib-9.5 + + - name: "Quicktest psql 9.6" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=9.6 addons: - apt: {packages: [postgresql-9.6, postgresql-contrib-9.6]} - - - os: linux - dist: xenial + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-9.6 + - postgresql-contrib-9.6 + + - name: "Quicktest psql 10" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=10 addons: - apt: {packages: [postgresql-10]} - - - os: linux - dist: xenial + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-10 + + - name: "Quicktest psql 11" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=11 addons: - apt: {packages: [postgresql-11]} + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-11 # Do a full test run on the latest supported version of PostgreSQL # on each supported version of Python. - - os: linux - dist: xenial - sudo: required + - name: "Test py 3.5" + os: linux + dist: focal language: python python: "3.5" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} - - - os: linux - dist: xenial - sudo: required + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + - name: "Test py 3.6" + os: linux + dist: focal language: python python: "3.6" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} - - - os: linux - dist: xenial - sudo: true + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + - name: "Test py 3.7" + os: linux + dist: focal language: python python: "3.7" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} - - - os: linux - dist: xenial - sudo: true + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + - name: "Test py 3.8" + os: linux + dist: focal language: python python: "3.8" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + # Only test on recent aarch64 distribution + # 3.7 is the latest supported by Travis + # https://docs.travis-ci.com/user/languages/python/#python-versions + # The shipped Postgres 9.X collides with the 12 on aarch64 + # until fixed, use official ubuntu repos + - name: "Test aarch64 py 3.8-dev" + os: linux + arch: arm64 + dist: focal + language: python + python: "3.8-dev" + env: BUILD=tests PGVERSION=12 + addons: + postgresql: "12" # Build manylinux wheels. Each wheel will be tested, # so there is no need for BUILD=tests here. # Also use this job to publish the releases and build # the documentation. - - os: linux - dist: xenial - sudo: required + - name: "x86 wheels and docs" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=wheels,docs,release PGVERSION=12 services: [docker] addons: - apt: {packages: [postgresql-12]} + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + # Same for the aarch64 manylinux wheel + - name: "aarch64 wheels" + os: linux + arch: arm64 + dist: focal + language: python + python: "3.8-dev" + env: BUILD=wheels,release PGVERSION=12 + services: [docker] + addons: + postgresql: "12" - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.5.7 PGVERSION=10 + - name: "OSX py 3.5" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.5.9 PGVERSION=12 - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.6.9 PGVERSION=10 + - name: "OSX py 3.6" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.6.10 PGVERSION=12 - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.7.4 PGVERSION=10 + - name: "OSX py 3.7" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.7.7 PGVERSION=12 - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.8.0 PGVERSION=10 + - name: "OSX py 3.8" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.8.3 PGVERSION=12 cache: pip diff --git a/asyncpg/__init__.py b/asyncpg/__init__.py index ee66f932..b0cef053 100644 --- a/asyncpg/__init__.py +++ b/asyncpg/__init__.py @@ -31,4 +31,4 @@ # snapshots will automatically include the git revision # in __version__, for example: '0.16.0.dev0+ge06ad03' -__version__ = '0.20.0' +__version__ = '0.21.0.dev0' diff --git a/asyncpg/connect_utils.py b/asyncpg/connect_utils.py index ec3d1090..2678b358 100644 --- a/asyncpg/connect_utils.py +++ b/asyncpg/connect_utils.py @@ -21,6 +21,7 @@ import typing import urllib.parse import warnings +import inspect from . import compat from . import exceptions @@ -601,6 +602,16 @@ async def _connect_addr(*, addr, loop, timeout, params, config, raise asyncio.TimeoutError connected = _create_future(loop) + + params_input = params + if callable(params.password): + if inspect.iscoroutinefunction(params.password): + password = await params.password() + else: + password = params.password() + + params = params._replace(password=password) + proto_factory = lambda: protocol.Protocol( addr, connected, params, loop) @@ -633,7 +644,7 @@ async def _connect_addr(*, addr, loop, timeout, params, config, tr.close() raise - con = connection_class(pr, tr, loop, addr, config, params) + con = connection_class(pr, tr, loop, addr, config, params_input) pr.set_connection(con) return con diff --git a/asyncpg/connection.py b/asyncpg/connection.py index ef1b595d..5fa2ddae 100644 --- a/asyncpg/connection.py +++ b/asyncpg/connection.py @@ -813,7 +813,7 @@ async def _copy_in(self, copy_stmt, source, timeout): if path is not None: # a path - f = await run_in_executor(None, open, path, 'wb') + f = await run_in_executor(None, open, path, 'rb') opened_by_us = True elif hasattr(source, 'read'): # file-like @@ -1566,6 +1566,10 @@ async def connect(dsn=None, *, other users and applications may be able to read it without needing specific privileges. It is recommended to use *passfile* instead. + Password may be either a string, or a callable that returns a string. + If a callable is provided, it will be called each time a new connection + is established. + :param passfile: The name of the file used to store passwords (defaults to ``~/.pgpass``, or ``%APPDATA%\postgresql\pgpass.conf`` @@ -1646,6 +1650,9 @@ async def connect(dsn=None, *, Added ability to specify multiple hosts in the *dsn* and *host* arguments. + .. versionchanged:: 0.21.0 + The *password* argument now accepts a callable or an async function. + .. _SSLContext: https://docs.python.org/3/library/ssl.html#ssl.SSLContext .. _create_default_context: https://docs.python.org/3/library/ssl.html#ssl.create_default_context diff --git a/asyncpg/pgproto b/asyncpg/pgproto index 1b4af3c6..76091445 160000 --- a/asyncpg/pgproto +++ b/asyncpg/pgproto @@ -1 +1 @@ -Subproject commit 1b4af3c685bfda6c74c082504fe8947c0a55e358 +Subproject commit 76091445db8b49a7d78504b47eb34fcbfbb89567 diff --git a/setup.py b/setup.py index 5e2e1494..5bb9e693 100644 --- a/setup.py +++ b/setup.py @@ -19,15 +19,13 @@ # We use vanilla build_ext, to avoid importing Cython via # the setuptools version. -from distutils import extension as distutils_extension -from distutils.command import build_ext as distutils_build_ext - import setuptools from setuptools.command import build_py as setuptools_build_py from setuptools.command import sdist as setuptools_sdist +from setuptools.command import build_ext as setuptools_build_ext -CYTHON_DEPENDENCY = 'Cython==0.29.14' +CYTHON_DEPENDENCY = 'Cython==0.29.20' # Minimal dependencies required to test asyncpg. TEST_DEPENDENCIES = [ @@ -138,9 +136,9 @@ def build_module(self, module, module_file, package): return outfile, copied -class build_ext(distutils_build_ext.build_ext): +class build_ext(setuptools_build_ext.build_ext): - user_options = distutils_build_ext.build_ext.user_options + [ + user_options = setuptools_build_ext.build_ext.user_options + [ ('cython-always', None, 'run cythonize() even if .c files are present'), ('cython-annotate', None, @@ -264,6 +262,7 @@ def finalize_options(self): 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Database :: Front-Ends', ], @@ -278,13 +277,13 @@ def finalize_options(self): provides=['asyncpg'], include_package_data=True, ext_modules=[ - distutils_extension.Extension( + setuptools.extension.Extension( "asyncpg.pgproto.pgproto", ["asyncpg/pgproto/pgproto.pyx"], extra_compile_args=CFLAGS, extra_link_args=LDFLAGS), - distutils_extension.Extension( + setuptools.extension.Extension( "asyncpg.protocol.protocol", ["asyncpg/protocol/record/recordobj.c", "asyncpg/protocol/protocol.pyx"], diff --git a/tests/test__environment.py b/tests/test__environment.py index 878fb2fc..3141fe8b 100644 --- a/tests/test__environment.py +++ b/tests/test__environment.py @@ -30,6 +30,8 @@ async def test_environment_server_version(self): @unittest.skipIf(not os.environ.get('ASYNCPG_VERSION'), "environ[ASYNCPG_VERSION] is not set") + @unittest.skipIf("dev" in asyncpg.__version__, + "development version with git commit data") async def test_environment_asyncpg_version(self): apgver = os.environ.get('ASYNCPG_VERSION') self.assertEqual( diff --git a/tests/test_connect.py b/tests/test_connect.py index f0767827..116b8ad9 100644 --- a/tests/test_connect.py +++ b/tests/test_connect.py @@ -204,6 +204,44 @@ async def test_auth_password_cleartext(self): user='password_user', password='wrongpassword') + async def test_auth_password_cleartext_callable(self): + def get_correctpassword(): + return 'correctpassword' + + def get_wrongpassword(): + return 'wrongpassword' + + conn = await self.connect( + user='password_user', + password=get_correctpassword) + await conn.close() + + with self.assertRaisesRegex( + asyncpg.InvalidPasswordError, + 'password authentication failed for user "password_user"'): + await self._try_connect( + user='password_user', + password=get_wrongpassword) + + async def test_auth_password_cleartext_callable_coroutine(self): + async def get_correctpassword(): + return 'correctpassword' + + async def get_wrongpassword(): + return 'wrongpassword' + + conn = await self.connect( + user='password_user', + password=get_correctpassword) + await conn.close() + + with self.assertRaisesRegex( + asyncpg.InvalidPasswordError, + 'password authentication failed for user "password_user"'): + await self._try_connect( + user='password_user', + password=get_wrongpassword) + async def test_auth_password_md5(self): conn = await self.connect( user='md5_user', password='correctpassword') diff --git a/tests/test_copy.py b/tests/test_copy.py index 257cc79c..dd01153f 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -8,6 +8,7 @@ import asyncio import datetime import io +import os import tempfile import asyncpg @@ -582,6 +583,48 @@ async def __anext__(self): finally: await self.con.execute('DROP TABLE copytab') + async def test_copy_to_table_from_file_path(self): + await self.con.execute(''' + CREATE TABLE copytab(a text, "b~" text, i int); + ''') + + f = tempfile.NamedTemporaryFile(delete=False) + try: + f.write( + '\n'.join([ + 'a1\tb1\t1', + 'a2\tb2\t2', + 'a3\tb3\t3', + 'a4\tb4\t4', + 'a5\tb5\t5', + '*\t\\N\t\\N', + '' + ]).encode('utf-8') + ) + f.close() + + res = await self.con.copy_to_table('copytab', source=f.name) + self.assertEqual(res, 'COPY 6') + + output = await self.con.fetch(""" + SELECT * FROM copytab ORDER BY a + """) + self.assertEqual( + output, + [ + ('*', None, None), + ('a1', 'b1', 1), + ('a2', 'b2', 2), + ('a3', 'b3', 3), + ('a4', 'b4', 4), + ('a5', 'b5', 5), + ] + ) + + finally: + await self.con.execute('DROP TABLE public.copytab') + os.unlink(f.name) + async def test_copy_records_to_table_1(self): await self.con.execute(''' CREATE TABLE copytab(a text, b int, c timestamptz); From e2b800d7def8e9375172465e78a67ffa8dd28fcc Mon Sep 17 00:00:00 2001 From: Jens Reidel Date: Sat, 4 Jul 2020 18:44:53 +0200 Subject: [PATCH 2/2] Fix branch name --- .ci/travis-build-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/travis-build-wheels.sh b/.ci/travis-build-wheels.sh index 11445af2..ae8c52af 100755 --- a/.ci/travis-build-wheels.sh +++ b/.ci/travis-build-wheels.sh @@ -3,7 +3,7 @@ set -e -x -if [[ "${TRAVIS_BRANCH}" != "release" || "${BUILD}" != *wheels* ]]; then +if [[ "${TRAVIS_BRANCH}" != "releases" || "${BUILD}" != *wheels* ]]; then # Not a release exit 0 fi