From f053640dc7766af8a1d472b0765fd33710faf4d6 Mon Sep 17 00:00:00 2001 From: Albert Wang Date: Tue, 9 Nov 2021 23:23:09 -0500 Subject: [PATCH 01/92] Remove broken loop argument --- aiomysql/connection.py | 8 +++----- aiomysql/pool.py | 2 +- tests/_testutils.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/aiomysql/connection.py b/aiomysql/connection.py index 8bf5cbf4..afce4e8e 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -477,8 +477,7 @@ async def _connect(self): self._reader, self._writer = await \ asyncio.wait_for( asyncio.open_unix_connection( - self._unix_socket, - loop=self._loop), + self._unix_socket), timeout=self.connect_timeout) self.host_info = "Localhost via UNIX socket: " + \ self._unix_socket @@ -487,8 +486,7 @@ async def _connect(self): asyncio.wait_for( asyncio.open_connection( self._host, - self._port, - loop=self._loop), + self._port), timeout=self.connect_timeout) self._set_keep_alive() self.host_info = "socket %s:%d" % (self._host, self._port) @@ -706,7 +704,7 @@ async def _request_authentication(self): # open_connection will cause it to negotiate TLS on an existing # connection not initiate a new one. self._reader, self._writer = await asyncio.open_connection( - sock=raw_sock, ssl=self._ssl_context, loop=self._loop, + sock=raw_sock, ssl=self._ssl_context, server_hostname=self._host ) diff --git a/aiomysql/pool.py b/aiomysql/pool.py index 1f194dc0..a17e3fca 100644 --- a/aiomysql/pool.py +++ b/aiomysql/pool.py @@ -43,7 +43,7 @@ def __init__(self, minsize, maxsize, echo, pool_recycle, loop, **kwargs): self._conn_kwargs = kwargs self._acquiring = 0 self._free = collections.deque(maxlen=maxsize) - self._cond = asyncio.Condition(loop=loop) + self._cond = asyncio.Condition() self._used = set() self._terminated = set() self._closing = False diff --git a/tests/_testutils.py b/tests/_testutils.py index 3ccb4c8a..1f198cc9 100644 --- a/tests/_testutils.py +++ b/tests/_testutils.py @@ -12,7 +12,7 @@ def run_until_complete(fun): def wrapper(test, *args, **kw): loop = test.loop ret = loop.run_until_complete( - asyncio.wait_for(fun(test, *args, **kw), 15, loop=loop)) + asyncio.wait_for(fun(test, *args, **kw), 15)) return ret return wrapper From 0c12aa37c6b9fe152d1fc86d0998fbbf22b1edd5 Mon Sep 17 00:00:00 2001 From: Albert Wang Date: Thu, 11 Nov 2021 20:57:53 -0500 Subject: [PATCH 02/92] Update changelog --- CHANGES.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index d0f6b604..31066c91 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,12 @@ Changes ------- +0.0.22 (2021-11-11) +^^^^^^^^^^^^^^^^^^^ + +* Support python 3.10 #505 + + 0.0.21 (2020-11-26) ^^^^^^^^^^^^^^^^^^^ From 81aadb22ba0222570e96037d47caaa2c41dfe4de Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 14 Nov 2021 11:59:37 +0200 Subject: [PATCH 03/92] Bump to 0.22.0 --- CHANGES.txt | 2 +- aiomysql/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 31066c91..1edf9c66 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,7 @@ Changes ------- -0.0.22 (2021-11-11) +0.0.22 (2021-11-14) ^^^^^^^^^^^^^^^^^^^ * Support python 3.10 #505 diff --git a/aiomysql/__init__.py b/aiomysql/__init__.py index f5e74aee..1b08ab56 100644 --- a/aiomysql/__init__.py +++ b/aiomysql/__init__.py @@ -33,7 +33,7 @@ from .cursors import Cursor, SSCursor, DictCursor, SSDictCursor from .pool import create_pool, Pool -__version__ = '0.0.21' +__version__ = '0.0.22' __all__ = [ From 45d241e48072d230cef8403e8178302ab31d1aef Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 4 Dec 2021 19:07:15 +0200 Subject: [PATCH 04/92] Update README.rst --- README.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.rst b/README.rst index 388252e1..a0e80228 100644 --- a/README.rst +++ b/README.rst @@ -29,12 +29,6 @@ Documentation ------------- https://aiomysql.readthedocs.io/ - -Mailing List ------------- -https://groups.google.com/forum/#!forum/aio-libs - - Basic Example ------------- From e5af7f5fe4a1f48f8bbcf4b71d66cf8c81ebe437 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Mon, 10 Jan 2022 14:49:52 +0100 Subject: [PATCH 05/92] Implement CI in github actions (#645) --- .github/workflows/ci.yml | 149 ++++++++++++++++++++ .github/workflows/lint.yml | 41 ++++++ .travis.yml | 50 ------- README.rst | 6 +- docs/index.rst | 8 +- requirements-dev.txt | 18 ++- setup.py | 9 +- tests/conftest.py | 264 +++++++++++++---------------------- tests/test_issues.py | 2 +- tests/test_load_local.py | 2 +- tests/test_pool.py | 14 ++ tests/test_sha_connection.py | 8 +- 12 files changed, 327 insertions(+), 244 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..f73c2937 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,149 @@ +name: CI + +on: + push: + branches: + - 'master' + tags: + - 'v*' + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + + +jobs: + test: + name: Test + strategy: + matrix: + # service containers are only supported on ubuntu currently + os: + - ubuntu-latest + py: + - '3.7' + - '3.8' + - '3.9' + # - '3.10' + # - '3.11.0-alpha.3' + db: + - 'mysql:5.7' + - 'mysql:8.0' + - 'mariadb:10.2' + - 'mariadb:10.3' + - 'mariadb:10.4' + - 'mariadb:10.5' + - 'mariadb:10.6' + - 'mariadb:10.7' + + fail-fast: false + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + + env: + MYSQL_ROOT_PASSWORD: rootpw + + services: + mysql: + image: '${{ matrix.db }}' + ports: + - 3306:3306 + options: '--name=mysqld' + env: + MYSQL_ROOT_PASSWORD: rootpw + + steps: + - name: Checkout + uses: actions/checkout@v2.4.0 + + - name: Setup Python ${{ matrix.py }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.py }} + + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" # - name: Cache + + - name: Cache PyPI + uses: actions/cache@v2.1.7 + with: + key: pip-ci-${{ runner.os }}-${{ matrix.py }} + path: ${{ steps.pip-cache.outputs.dir }} + + - name: Update pip, wheel, setuptools, build, twine, codecov + run: | + python -m pip install -U pip wheel setuptools build twine codecov + + - name: Install dependencies + run: | + python -m pip install --upgrade --requirement requirements-dev.txt + + - name: Install aiomysql + run: | + python -m pip install . + + - name: Check rst + run: | + python setup.py check --restructuredtext + + - name: Run pyroma + run: | + python -m pyroma -d . + + # this ensures our database is ready. typically by the time the preparations have completed its first start logic. + # unfortunately we need this hacky workaround as GitHub Actions service containers can't reference data from our repo. + - name: Prepare mysql + run: | + # ensure server is started up + while : + do + sleep 1 + mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break + done + + # inject tls configuration + docker container stop mysqld + docker container cp "${{ github.workspace }}/tests/ssl_resources/ssl" mysqld:/etc/mysql/ssl + docker container cp "${{ github.workspace }}/tests/ssl_resources/tls.cnf" mysqld:/etc/mysql/conf.d/aiomysql-tls.cnf + docker container start mysqld + + # ensure server is started up + while : + do + sleep 1 + mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break + done + + mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e "SET GLOBAL local_infile=on" + + - name: Run tests + run: | + export DB="${MATRIX_DB%%:*}" + export DBTAG="${MATRIX_DB##*:}" + + # timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs + timeout --preserve-status --signal=INT --verbose 5m \ + pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql ./tests + env: + PYTHONUNBUFFERED: 1 + MATRIX_DB: '${{ matrix.db }}' + timeout-minutes: 6 + + - name: Build coverage flag + run: | + COVERAGE_FLAG="${MATRIX_OS}_${MATRIX_PY}_${MATRIX_DB//:/-}" + echo "COVERAGE_FLAG=$COVERAGE_FLAG" | tee -a "$GITHUB_ENV" + env: + MATRIX_OS: '${{ matrix.os }}' + MATRIX_PY: '${{ matrix.py }}' + MATRIX_DB: '${{ matrix.db }}' + + - name: Upload coverage + uses: codecov/codecov-action@v2.1.0 + with: + file: ./coverage.xml + flags: "${{ env.COVERAGE_FLAG }}" + fail_ci_if_error: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..38af6d93 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,41 @@ +name: lint + +on: + push: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2.4.0 + + - name: Setup Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" # - name: Cache + + - name: Cache PyPI + uses: actions/cache@v2.1.7 + with: + key: pip-lint + path: ${{ steps.pip-cache.outputs.dir }} + + - name: flake8 Lint + uses: py-actions/flake8@v2.0.0 + with: + flake8-version: 4.0.1 + path: aiomysql + args: tests examples diff --git a/.travis.yml b/.travis.yml index cc2076e9..81725577 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,54 +1,10 @@ language: python -python: - - 3.6 - - 3.7 - - 3.8 - env: matrix: - PYTHONASYNCIODEBUG=1 - PYTHONASYNCIODEBUG= -services: - - docker - -matrix: - include: - - python: 3.6 - env: - - PYTHONASYNCIODEBUG= - - DB=mariadb - - DBTAG=5.5 - - python: 3.6 - env: - - PYTHONASYNCIODEBUG=1 - - DB=mariadb - - DBTAG=10.0 - addons: - mariadb: '10.0' - - python: 3.6 - env: - - PYTHONASYNCIODEBUG= - - DB=mariadb - - DBTAG=10.5 - - python: 3.6 - env: - - PYTHONASYNCIODEBUG= - - DB=mysql - - DBTAG=5.7 - - -#before_script: -# - "mysql -e 'SELECT VERSION()'" -# - "mysql -e 'DROP DATABASE IF EXISTS test_pymysql; create database test_pymysql DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'" -# - "mysql -e 'DROP DATABASE IF EXISTS test_pymysql2; create database test_pymysql2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'" - -install: - - pip install -Ur requirements-dev.txt - - pip install . - - pip install codecov - deploy: provider: pypi user: aio-libs-bot @@ -60,9 +16,3 @@ deploy: repo: aio-libs/aiomysql all_branches: true python: 3.6 - -script: - - make cov - -after_success: - - codecov diff --git a/README.rst b/README.rst index a0e80228..97bb9c34 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ aiomysql ======== -.. image:: https://travis-ci.com/aio-libs/aiomysql.svg?branch=master - :target: https://travis-ci.com/aio-libs/aiomysql +.. image:: https://github.com/aio-libs/aiomysql/actions/workflows/ci.yml/badge.svg?branch=master + :target: https://github.com/aio-libs/aiomysql/actions/workflows/ci.yml .. image:: https://codecov.io/gh/aio-libs/aiomysql/branch/master/graph/badge.svg :target: https://codecov.io/gh/aio-libs/aiomysql :alt: Code coverage @@ -104,7 +104,7 @@ for aiopg_ user.: Requirements ------------ -* Python_ 3.5.3+ +* Python_ 3.7+ * PyMySQL_ diff --git a/docs/index.rst b/docs/index.rst index 684d08c1..8dc92185 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -96,16 +96,16 @@ Please feel free to file an issue on `bug tracker `_ if you have found a bug or have some suggestion for library improvement. -The library uses `Travis `_ for -Continious Integration and `Coveralls -`_ for +The library uses `GitHub Actions +`_ for Continuous Integration +and `Codecov `_ for coverage reports. Dependencies ------------ -- Python 3.5.3+ +- Python 3.7+ - :term:`PyMySQL` - aiomysql.sa requires :term:`sqlalchemy`. diff --git a/requirements-dev.txt b/requirements-dev.txt index 70e6fd13..c3662f8d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,14 +1,12 @@ -coverage>=4.5.1,<=5.1 -flake8>=3.5.0,<=3.7.9 -ipdb>=0.11,<=0.13.2 -ipython>=7.0.1,<=7.13.0 -pytest>=3.9.1,<=5.4.1 -pytest-cov>=2.6.0,<=2.8.1 -pytest-sugar>=0.9.1,<=0.9.3 +coverage==6.2 +flake8==4.0.1 +ipdb==0.13.9 +pytest==6.2.5 +pytest-cov==3.0.0 +pytest-sugar==0.9.4 PyMySQL>=0.9,<=0.9.3 -docker>=3.5.1,<=4.2.0 sphinx>=1.8.1, <=3.0.3 sphinxcontrib-asyncio==0.2.0 sqlalchemy>1.2.12,<=1.3.16 -uvloop>=0.11.2,<=0.14.0; python_version >= '3.5' -pyroma==2.6 +uvloop==0.16.0 +pyroma==3.2 diff --git a/setup.py b/setup.py index 290c2075..6e9c0bf2 100755 --- a/setup.py +++ b/setup.py @@ -9,8 +9,8 @@ PY_VER = sys.version_info -if not PY_VER >= (3, 5, 3): - raise RuntimeError("aiomysql doesn't support Python earlier than 3.5.3") +if not PY_VER >= (3, 7, 0): + raise RuntimeError("aiomysql doesn't support Python earlier than 3.7.0") def read(f): @@ -37,8 +37,9 @@ def read_version(): 'License :: OSI Approved :: MIT License', 'Intended Audience :: Developers', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Operating System :: POSIX', 'Environment :: Web Environment', 'Development Status :: 3 - Alpha', diff --git a/tests/conftest.py b/tests/conftest.py index 5381498d..988643b5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,24 +2,12 @@ import gc import os import ssl -import socket -import sys -import time import uuid -from distutils.version import StrictVersion -from docker import APIClient - import aiomysql import pymysql import pytest - - -PY_35 = sys.version_info >= (3, 5) -if PY_35: - import uvloop -else: - uvloop = None +import uvloop @pytest.fixture @@ -34,15 +22,6 @@ def disable_gc(): gc.enable() -@pytest.fixture(scope='session') -def unused_port(): - def f(): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(('127.0.0.1', 0)) - return s.getsockname()[1] - return f - - def pytest_generate_tests(metafunc): if 'loop_type' in metafunc.fixturenames: loop_type = ['asyncio', 'uvloop'] if uvloop else ['asyncio'] @@ -50,17 +29,17 @@ def pytest_generate_tests(metafunc): # This is here unless someone fixes the generate_tests bit -@pytest.yield_fixture(scope='session') +@pytest.fixture(scope='session') def mysql_tag(): return os.environ.get('DBTAG', '10.5') -@pytest.yield_fixture(scope='session') +@pytest.fixture(scope='session') def mysql_image(): return os.environ.get('DB', 'mariadb') -@pytest.yield_fixture +@pytest.fixture def loop(request, loop_type): loop = asyncio.new_event_loop() asyncio.set_event_loop(None) @@ -85,7 +64,7 @@ def pytest_pycollect_makeitem(collector, name, obj): if collector.funcnamefilter(name): if not callable(obj): return - item = pytest.Function(name, parent=collector) + item = pytest.Function.from_parent(collector, name=name) if 'run_loop' in item.keywords: return list(collector._genfunctions(name, obj)) @@ -111,26 +90,21 @@ def pytest_runtest_setup(item): item.fixturenames.append('loop') -def pytest_ignore_collect(path, config): - if 'pep492' in str(path): - if sys.version_info < (3, 5, 0): - return True - - -def pytest_addoption(parser): - parser.addoption("--mysql_tag", action="append", default=[], - help=("MySQL server versions. " - "May be used several times. " - "Available values: 5.6, 5.7, 8.0, all")) - parser.addoption("--no-pull", action="store_true", default=False, - help="Don't perform docker images pulling") +def pytest_configure(config): + config.addinivalue_line( + "markers", + "run_loop" + ) + config.addinivalue_line( + "markers", + "mysql_version(db, version): run only on specific database versions" + ) @pytest.fixture def mysql_params(mysql_server): params = {**mysql_server['conn_params'], "db": os.environ.get('MYSQL_DB', 'test_pymysql'), - # "password": os.environ.get('MYSQL_PASSWORD', ''), "local_infile": True, "use_unicode": True, } @@ -144,14 +118,14 @@ def _cursor_wrapper(conn): return cur -@pytest.yield_fixture +@pytest.fixture def cursor(connection, loop): cur = loop.run_until_complete(_cursor_wrapper(connection)) yield cur loop.run_until_complete(cur.close()) -@pytest.yield_fixture +@pytest.fixture def connection(mysql_params, loop): coro = aiomysql.connect(loop=loop, **mysql_params) conn = loop.run_until_complete(coro) @@ -159,7 +133,7 @@ def connection(mysql_params, loop): loop.run_until_complete(conn.ensure_closed()) -@pytest.yield_fixture +@pytest.fixture def connection_creator(mysql_params, loop): connections = [] @@ -181,7 +155,7 @@ def f(**kw): pass -@pytest.yield_fixture +@pytest.fixture def pool_creator(mysql_params, loop): pools = [] @@ -201,7 +175,7 @@ def f(**kw): loop.run_until_complete(pool.wait_closed()) -@pytest.yield_fixture +@pytest.fixture def table_cleanup(loop, connection): table_list = [] cursor = loop.run_until_complete(_cursor_wrapper(connection)) @@ -222,141 +196,97 @@ def session_id(): return str(uuid.uuid4()) -@pytest.fixture(scope='session') -def docker(): - return APIClient(version='auto') - - @pytest.fixture(autouse=True) -def ensure_mysql_verison(request, mysql_tag): - if StrictVersion(pytest.__version__) >= StrictVersion('4.0.0'): - mysql_version = request.node.get_closest_marker('mysql_verison') - else: - mysql_version = request.node.get_marker('mysql_verison') - - if mysql_version and mysql_version.args[0] != mysql_tag: - pytest.skip('Not applicable for MySQL version: {0}'.format(mysql_tag)) +def ensure_mysql_version(request, mysql_image, mysql_tag): + mysql_version = request.node.get_closest_marker('mysql_version') + if mysql_version and ( + mysql_version.args[0] != mysql_image + or mysql_version.args[1] != mysql_tag): -@pytest.fixture(scope='session') -def mysql_server(unused_port, docker, session_id, - mysql_image, mysql_tag, request): - print('\nSTARTUP CONTAINER - {0}\n'.format(mysql_tag)) - - if not request.config.option.no_pull: - docker.pull('{}:{}'.format(mysql_image, mysql_tag)) + pytest.skip('Not applicable for {0} version: {1}' + .format(mysql_image, mysql_tag)) - # bound IPs do not work on OSX - host = "127.0.0.1" - host_port = unused_port() - # As TLS is optional, might as well always configure it +@pytest.fixture(scope='session') +def mysql_server(mysql_image, mysql_tag): ssl_directory = os.path.join(os.path.dirname(__file__), 'ssl_resources', 'ssl') ca_file = os.path.join(ssl_directory, 'ca.pem') - tls_cnf = os.path.join(os.path.dirname(__file__), - 'ssl_resources', 'tls.cnf') ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) ctx.check_hostname = False ctx.load_verify_locations(cafile=ca_file) # ctx.verify_mode = ssl.CERT_NONE - container_args = dict( - image='{}:{}'.format(mysql_image, mysql_tag), - name='aiomysql-test-server-{}-{}'.format(mysql_tag, session_id), - ports=[3306], - detach=True, - host_config=docker.create_host_config( - port_bindings={3306: (host, host_port)}, - binds={ - ssl_directory: {'bind': '/etc/mysql/ssl', 'mode': 'ro'}, - tls_cnf: {'bind': '/etc/mysql/conf.d/tls.cnf', 'mode': 'ro'}, - } - ), - environment={'MYSQL_ROOT_PASSWORD': 'rootpw'} - ) - - container = docker.create_container(**container_args) + server_params = { + 'host': '127.0.0.1', + 'port': 3306, + 'user': 'root', + 'password': os.environ.get("MYSQL_ROOT_PASSWORD"), + 'ssl': ctx, + } try: - docker.start(container=container['Id']) - - # MySQL restarts at least 4 times in the container before its ready - time.sleep(10) - - server_params = { - 'host': host, - 'port': host_port, - 'user': 'root', - 'password': 'rootpw', - 'ssl': ctx - } - delay = 0.001 - for _ in range(100): - try: - connection = pymysql.connect( - db='mysql', - charset='utf8mb4', - cursorclass=pymysql.cursors.DictCursor, - **server_params) - - with connection.cursor() as cursor: - cursor.execute("SHOW VARIABLES LIKE '%ssl%';") - - result = cursor.fetchall() - result = {item['Variable_name']: - item['Value'] for item in result} - - assert result['have_ssl'] == "YES", \ - "SSL Not Enabled on docker'd MySQL" - - cursor.execute("SHOW STATUS LIKE 'Ssl_version%'") - - result = cursor.fetchone() - # As we connected with TLS, it should start with that :D - assert result['Value'].startswith('TLS'), \ - "Not connected to the database with TLS" - - # Create Databases - cursor.execute('CREATE DATABASE test_pymysql ' - 'DEFAULT CHARACTER SET utf8 ' - 'DEFAULT COLLATE utf8_general_ci;') - cursor.execute('CREATE DATABASE test_pymysql2 ' - 'DEFAULT CHARACTER SET utf8 ' - 'DEFAULT COLLATE utf8_general_ci;') - - # Do MySQL8+ Specific Setup - if mysql_tag in ('8.0',): - # Create Users to test SHA256 - cursor.execute('CREATE USER user_sha256 ' - 'IDENTIFIED WITH "sha256_password" ' - 'BY "pass_sha256"') - cursor.execute('CREATE USER nopass_sha256 ' - 'IDENTIFIED WITH "sha256_password"') - cursor.execute('CREATE USER user_caching_sha2 ' - 'IDENTIFIED ' - 'WITH "caching_sha2_password" ' - 'BY "pass_caching_sha2"') - cursor.execute('CREATE USER nopass_caching_sha2 ' - 'IDENTIFIED ' - 'WITH "caching_sha2_password" ' - 'PASSWORD EXPIRE NEVER') - cursor.execute('FLUSH PRIVILEGES') - - break - except Exception: - time.sleep(delay) - delay *= 2 - else: - pytest.fail("Cannot start MySQL server") - - container['host'] = host - container['port'] = host_port - container['conn_params'] = server_params - - yield container - finally: - print('\nTEARDOWN CONTAINER - {0}\n'.format(mysql_tag)) - docker.kill(container=container['Id']) - docker.remove_container(container['Id']) + connection = pymysql.connect( + db='mysql', + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor, + **server_params) + + with connection.cursor() as cursor: + cursor.execute("SHOW VARIABLES LIKE '%ssl%';") + + result = cursor.fetchall() + result = {item['Variable_name']: + item['Value'] for item in result} + + assert result['have_ssl'] == "YES", \ + "SSL Not Enabled on MySQL" + + cursor.execute("SHOW STATUS LIKE 'Ssl_version%'") + + result = cursor.fetchone() + # As we connected with TLS, it should start with that :D + assert result['Value'].startswith('TLS'), \ + "Not connected to the database with TLS" + + # Drop possibly existing old databases + cursor.execute('DROP DATABASE IF EXISTS test_pymysql;') + cursor.execute('DROP DATABASE IF EXISTS test_pymysql2;') + + # Create Databases + cursor.execute('CREATE DATABASE test_pymysql ' + 'DEFAULT CHARACTER SET utf8 ' + 'DEFAULT COLLATE utf8_general_ci;') + cursor.execute('CREATE DATABASE test_pymysql2 ' + 'DEFAULT CHARACTER SET utf8 ' + 'DEFAULT COLLATE utf8_general_ci;') + + # Do MySQL8+ Specific Setup + if mysql_image == "mysql" and mysql_tag in ('8.0',): + # Drop existing users + cursor.execute('DROP USER IF EXISTS user_sha256;') + cursor.execute('DROP USER IF EXISTS nopass_sha256;') + cursor.execute('DROP USER IF EXISTS user_caching_sha2;') + cursor.execute('DROP USER IF EXISTS nopass_caching_sha2;') + + # Create Users to test SHA256 + cursor.execute('CREATE USER user_sha256 ' + 'IDENTIFIED WITH "sha256_password" ' + 'BY "pass_sha256"') + cursor.execute('CREATE USER nopass_sha256 ' + 'IDENTIFIED WITH "sha256_password"') + cursor.execute('CREATE USER user_caching_sha2 ' + 'IDENTIFIED ' + 'WITH "caching_sha2_password" ' + 'BY "pass_caching_sha2"') + cursor.execute('CREATE USER nopass_caching_sha2 ' + 'IDENTIFIED ' + 'WITH "caching_sha2_password" ' + 'PASSWORD EXPIRE NEVER') + cursor.execute('FLUSH PRIVILEGES') + except Exception: + pytest.fail("Cannot initialize MySQL environment") + + return {'conn_params': server_params} diff --git a/tests/test_issues.py b/tests/test_issues.py index a35e9d86..942bc8ed 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -29,7 +29,7 @@ async def test_issue_3(connection): assert r[0] is None await c.execute("select ts from issue3") r = await c.fetchone() - assert isinstance(r[0], datetime.datetime) + assert type(r[0]) in (type(None), datetime.datetime) finally: await c.execute("drop table issue3") diff --git a/tests/test_load_local.py b/tests/test_load_local.py index aa3b45cc..637515b0 100644 --- a/tests/test_load_local.py +++ b/tests/test_load_local.py @@ -6,7 +6,7 @@ from pymysql.err import OperationalError -@pytest.yield_fixture +@pytest.fixture def table_local_file(connection, loop): async def prepare_table(conn): diff --git a/tests/test_pool.py b/tests/test_pool.py index 0f1ec140..ff1acf5e 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -435,6 +435,8 @@ async def test_drop_connection_if_timedout(pool_creator, conn = await connection_creator() await _set_global_conn_timeout(conn, 2) await conn.ensure_closed() + + pool = conn = None try: pool = await pool_creator(minsize=3, maxsize=3) # sleep, more then connection timeout @@ -444,9 +446,21 @@ async def test_drop_connection_if_timedout(pool_creator, # query should not throw exception OperationalError await cur.execute('SELECT 1;') pool.release(conn) + conn = None pool.close() await pool.wait_closed() finally: + # TODO: this could probably be done better + # if this isn't closed it blocks forever + try: + if conn is not None: + pool.release(conn) + if pool is not None: + pool.close() + await pool.wait_closed() + except Exception: + pass + # setup default timeouts conn = await connection_creator() await _set_global_conn_timeout(conn, 28800) diff --git a/tests/test_sha_connection.py b/tests/test_sha_connection.py index f2a108d8..eb57ec3d 100644 --- a/tests/test_sha_connection.py +++ b/tests/test_sha_connection.py @@ -21,7 +21,7 @@ # ]) -@pytest.mark.mysql_verison('8.0') +@pytest.mark.mysql_version('mysql', '8.0') @pytest.mark.run_loop async def test_sha256_nopw(mysql_server, loop): connection_data = copy.copy(mysql_server['conn_params']) @@ -36,7 +36,7 @@ async def test_sha256_nopw(mysql_server, loop): assert conn._auth_plugin_used == 'sha256_password' -@pytest.mark.mysql_verison('8.0') +@pytest.mark.mysql_version('mysql', '8.0') @pytest.mark.run_loop async def test_sha256_pw(mysql_server, loop): connection_data = copy.copy(mysql_server['conn_params']) @@ -51,7 +51,7 @@ async def test_sha256_pw(mysql_server, loop): assert conn._auth_plugin_used == 'sha256_password' -@pytest.mark.mysql_verison('8.0') +@pytest.mark.mysql_version('mysql', '8.0') @pytest.mark.run_loop async def test_cached_sha256_nopw(mysql_server, loop): connection_data = copy.copy(mysql_server['conn_params']) @@ -66,7 +66,7 @@ async def test_cached_sha256_nopw(mysql_server, loop): assert conn._auth_plugin_used == 'caching_sha2_password' -@pytest.mark.mysql_verison('8.0') +@pytest.mark.mysql_version('mysql', '8.0') @pytest.mark.run_loop async def test_cached_sha256_pw(mysql_server, loop): connection_data = copy.copy(mysql_server['conn_params']) From 93f4c6086c38504e870812ef5a130638de6967f8 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 10 Jan 2022 15:54:47 +0200 Subject: [PATCH 06/92] Drop Travis config, the deployment should use GitHub Actions --- .travis.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 81725577..00000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: python - -env: - matrix: - - PYTHONASYNCIODEBUG=1 - - PYTHONASYNCIODEBUG= - -deploy: - provider: pypi - user: aio-libs-bot - distributions: "sdist bdist_wheel" - password: - secure: G9vr3UOuK7tJifGEzO1kForcz+DCq1IdtIVKr/e4gPenvENinCFrRRWw206BQf1ba1+EjvRqc/yJFfZrgFUJlxgrtahvFmTCzKiLHCwr2vJsEYyr6JLWKRE81//RKTOykDWwvCAjk6sgV8lYtKL6R5sTtjzLJq8CKsLoKZ97yniPKmNu/+7IJvp+vSOA9gIL+GWbDTP8lmNDLWwphFxq6mm4WQ0VWqwDb0SF3FG/QPGDYU19mPLsLqgf1cxaBuRtb/epRDLLG70M9l9/aBPbtAHdbxY+O+/Fv5RPxfo2xFB7ry7yQEsIGrOwot/TxsZDRwRnfPm8N4OV9AfnKjiF5sBpidRR5kQr2pgFP2xq7LEROOPydMbY+YbSHCRBGCHWsHusjwCCL1veVTZ10EB9j3j0O9C6rAaF6Ssdlfq3kzhbWUItQfIZ/h7C0Z0ucVqFB8uBug7jNxT8hD3pR4ftM6Y94HY4BFOlkmSUH9u7owCeUoV9WQT/QAZHOLswRpp1wjsu2c0zKh3wiiuzCYJ64cD/BQW8rQMp0QiGqsNebqR+3L6yNLLSMpDWp3q/pnVbjsI/yepvpVbpp3PltSfZJL0uUfE0OR+xU67JP4npVS8B/A7aTARcM/ljx7IYNYYf/peXQ6UmBOrR2OgPnRPv5LfG9NGdEy0WpQKCVuSydxk= - on: - tags: true - repo: aio-libs/aiomysql - all_branches: true - python: 3.6 From b66ac1abc4018824da83bef5136d07f284a72aeb Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 10 Jan 2022 15:55:09 +0200 Subject: [PATCH 07/92] Drop the pyup config --- .pyup.yml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .pyup.yml diff --git a/.pyup.yml b/.pyup.yml deleted file mode 100644 index 75f97116..00000000 --- a/.pyup.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Label PRs with `deps-update` label -label_prs: deps-update - -schedule: every week From 38a383a6b708d3eb7d9fc6dd864151908c3ac115 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 10 Jan 2022 15:56:25 +0200 Subject: [PATCH 08/92] Setup dependabot config --- .github/dependabot.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..19941394 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + open-pull-requests-limit: 10 + target-branch: master +- package-ecosystem: github-actions + directory: / + open-pull-requests-limit: 10 From 4f9438af1e34f24ffe27282c2fb87a27a0342460 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 10 Jan 2022 15:58:36 +0200 Subject: [PATCH 09/92] Fix dependabot schedule --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 19941394..10c63edc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,8 +2,12 @@ version: 2 updates: - package-ecosystem: pip directory: "/" + schedule: + interval: daily open-pull-requests-limit: 10 target-branch: master - package-ecosystem: github-actions directory: / + schedule: + interval: daily open-pull-requests-limit: 10 From 624848e88eebed4336972763547df4c37a685856 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 17:37:02 +0200 Subject: [PATCH 10/92] Bump sphinxcontrib-asyncio from 0.2.0 to 0.3.0 (#655) Bumps [sphinxcontrib-asyncio](https://github.com/aio-libs/sphinxcontrib-asyncio) from 0.2.0 to 0.3.0. - [Release notes](https://github.com/aio-libs/sphinxcontrib-asyncio/releases) - [Changelog](https://github.com/aio-libs/sphinxcontrib-asyncio/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/sphinxcontrib-asyncio/compare/v0.2.0...v0.3.0) --- updated-dependencies: - dependency-name: sphinxcontrib-asyncio dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c3662f8d..e09297c2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ pytest-cov==3.0.0 pytest-sugar==0.9.4 PyMySQL>=0.9,<=0.9.3 sphinx>=1.8.1, <=3.0.3 -sphinxcontrib-asyncio==0.2.0 +sphinxcontrib-asyncio==0.3.0 sqlalchemy>1.2.12,<=1.3.16 uvloop==0.16.0 pyroma==3.2 From c6b3f05df186510fb6c86f581440067137a7e82a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 10 Jan 2022 19:11:48 +0200 Subject: [PATCH 11/92] Don't run duplicated lint jobs (#658) --- .github/workflows/lint.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 38af6d93..a89afdf6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,6 +2,10 @@ name: lint on: push: + branches: + - 'master' + tags: + - 'v*' pull_request: concurrency: From 4274fbce6c9b2559bfe9e7db16b49850ab01bded Mon Sep 17 00:00:00 2001 From: Lorenzo Delmonte Date: Mon, 10 Jan 2022 18:15:18 +0100 Subject: [PATCH 12/92] Fix typo (#567) --- examples/example_executemany.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example_executemany.py b/examples/example_executemany.py index 30ff1a30..087b456e 100644 --- a/examples/example_executemany.py +++ b/examples/example_executemany.py @@ -22,7 +22,7 @@ async def test_example_executemany(loop): await cur.execute("INSERT INTO music_style VALUES(3,'power metal');") await conn.commit() - # insert 3 row by one long query using *executemane* method + # insert 3 row by one long query using *executemany* method data = [(4, 'gothic metal'), (5, 'doom metal'), (6, 'post metal')] await cur.executemany( "INSERT INTO music_style (id, name)" From f8dc6004e353ad672d08e9690686aecdfc65b14d Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Thu, 13 Jan 2022 02:48:11 +0100 Subject: [PATCH 13/92] create bug report template, based on aiohttp bug report template (#661) --- .github/ISSUE_TEMPLATE/bug_report.yml | 114 ++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..466e73ab --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,114 @@ +--- +name: Bug Report +description: Create a report to help us improve. +labels: [bug] +body: +- type: markdown + attributes: + value: | + **Thanks for taking a minute to file a bug report!** + + โš  + Verify first that your issue is not [already reported on + GitHub][issue search]. + + _Please fill out the form below with as many precise + details as possible._ + + [issue search]: ../search?q=is%3Aissue&type=issues + +- type: textarea + attributes: + label: Describe the bug + description: >- + A clear and concise description of what the bug is. + validations: + required: true + +- type: textarea + attributes: + label: To Reproduce + description: >- + Describe the steps to reproduce this bug. + placeholder: | + 1. Have certain environment + 2. Then run '...' + 3. An error occurs. + validations: + required: true + +- type: textarea + attributes: + label: Expected behavior + description: >- + A clear and concise description of what you expected to happen. + validations: + required: true + +- type: textarea + attributes: + label: Logs/tracebacks + description: | + If applicable, add logs/tracebacks to help explain your problem. + Paste the output of the steps above, including the commands + themselves and their output/traceback etc. + render: python-traceback + validations: + required: true + +- type: textarea + attributes: + label: Python Version + description: Attach your version of Python. + render: console + value: | + $ python --version + validations: + required: true +- type: textarea + attributes: + label: aiomysql Version + description: Attach your version of aiomysql. + render: console + value: | + $ python -m pip show aiomysql + validations: + required: true + +- type: textarea + attributes: + label: OS + placeholder: >- + For example, Arch Linux, Windows, macOS, etc. + validations: + required: true + +- type: textarea + attributes: + label: Database type and version + description: Attach your version of MariaDB/MySQL. + render: console + value: | + SELECT VERSION(); + validations: + required: true + +- type: textarea + attributes: + label: Additional context + description: | + Add any other context about the problem here. + + Describe the environment you have that lead to your issue. + +- type: checkboxes + attributes: + label: Code of Conduct + description: | + Read the [aio-libs Code of Conduct][CoC] first. + + [CoC]: https://github.com/aio-libs/.github/blob/master/CODE_OF_CONDUCT.md + options: + - label: I agree to follow the aio-libs Code of Conduct + required: true +... From 0309e5f5b5115c58d23220fea32168478f8811a1 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Thu, 13 Jan 2022 18:17:42 +0100 Subject: [PATCH 14/92] add PyMySQL and SQLAlchemy version to bug report template (#662) --- .github/ISSUE_TEMPLATE/bug_report.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 466e73ab..99aa53df 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -74,6 +74,22 @@ body: $ python -m pip show aiomysql validations: required: true +- type: textarea + attributes: + label: PyMySQL Version + description: Attach your version of PyMySQL. + render: console + value: | + $ python -m pip show PyMySQL + validations: + required: true +- type: textarea + attributes: + label: SQLAlchemy Version + description: Attach your version of SQLAlchemy if you're using it. + render: console + value: | + $ python -m pip show sqlalchemy - type: textarea attributes: From ecbd675ea5326a7879a3003fffedaa69494930e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:43:41 +0100 Subject: [PATCH 15/92] Update sphinx requirement from <=3.0.3,>=1.8.1 to >=1.8.1,<4.3.3 (#654) Updates the requirements on [sphinx](https://github.com/sphinx-doc/sphinx) to permit the latest version. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/4.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v1.8.1...v4.3.2) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e09297c2..1fc0696e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,7 @@ pytest==6.2.5 pytest-cov==3.0.0 pytest-sugar==0.9.4 PyMySQL>=0.9,<=0.9.3 -sphinx>=1.8.1, <=3.0.3 +sphinx>=1.8.1, <4.3.3 sphinxcontrib-asyncio==0.3.0 sqlalchemy>1.2.12,<=1.3.16 uvloop==0.16.0 From 282c1e24e0b800da57afd291199495208e8ac078 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 09:14:29 +0100 Subject: [PATCH 16/92] Update sphinx requirement from <4.3.3,>=1.8.1 to >=1.8.1,<4.4.1 (#672) Updates the requirements on [sphinx](https://github.com/sphinx-doc/sphinx) to permit the latest version. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/4.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v1.8.1...v4.4.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1fc0696e..f30f2dcf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,7 @@ pytest==6.2.5 pytest-cov==3.0.0 pytest-sugar==0.9.4 PyMySQL>=0.9,<=0.9.3 -sphinx>=1.8.1, <4.3.3 +sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 sqlalchemy>1.2.12,<=1.3.16 uvloop==0.16.0 From 3a08952dc159bd3d4fea0b2ab411a329b049f626 Mon Sep 17 00:00:00 2001 From: Anand <40204976+anand2312@users.noreply.github.com> Date: Fri, 21 Jan 2022 22:19:49 +0530 Subject: [PATCH 17/92] Update examples to use async/await (#582) * Update examples to use async/await * docs: add missing async in def, fix indent Co-authored-by: Richard Schwab --- docs/connection.rst | 19 +++++----- docs/cursors.rst | 89 ++++++++++++++++++++++----------------------- docs/pool.rst | 25 ++++++------- docs/sa.rst | 39 ++++++++++---------- docs/tutorial.rst | 23 ++++++------ 5 files changed, 94 insertions(+), 101 deletions(-) diff --git a/docs/connection.rst b/docs/connection.rst index 97c41676..73a61a28 100644 --- a/docs/connection.rst +++ b/docs/connection.rst @@ -23,18 +23,17 @@ Example:: loop = asyncio.get_event_loop() - @asyncio.coroutine - def test_example(): - conn = yield from aiomysql.connect(host='127.0.0.1', port=3306, - user='root', password='', db='mysql', - loop=loop) - - cur = yield from conn.cursor() - yield from cur.execute("SELECT Host,User FROM user") + async def test_example(): + conn = await aiomysql.connect(host='127.0.0.1', port=3306, + user='root', password='', db='mysql', + loop=loop) + + cur = await conn.cursor() + await cur.execute("SELECT Host,User FROM user") print(cur.description) - r = yield from cur.fetchall() + r = await cur.fetchall() print(r) - yield from cur.close() + await cur.close() conn.close() loop.run_until_complete(test_example()) diff --git a/docs/cursors.rst b/docs/cursors.rst index 70af47ba..40cb4604 100644 --- a/docs/cursors.rst +++ b/docs/cursors.rst @@ -26,23 +26,22 @@ Cursor loop = asyncio.get_event_loop() - @asyncio.coroutine - def test_example(): - conn = yield from aiomysql.connect(host='127.0.0.1', port=3306, - user='root', password='', - db='mysql', loop=loop) + async def test_example(): + conn = await aiomysql.connect(host='127.0.0.1', port=3306, + user='root', password='', + db='mysql', loop=loop) # create default cursor - cursor = yield from conn.cursor() + cursor = await conn.cursor() # execute sql query - yield from cursor.execute("SELECT Host, User FROM user") + await cursor.execute("SELECT Host, User FROM user") # fetch all results - r = yield from cursor.fetchall() + r = await cursor.fetchall() # detach cursor from connection - yield from cursor.close() + await cursor.close() # close connection conn.close() @@ -137,7 +136,7 @@ Cursor For example, getting all rows where id is 5:: - yield from cursor.execute("SELECT * FROM t1 WHERE id=%s", (5,)) + await cursor.execute("SELECT * FROM t1 WHERE id=%s", (5,)) :param str query: sql statement :param list args: tuple or list of arguments for sql query @@ -157,7 +156,7 @@ Cursor ] stmt = "INSERT INTO employees (name, phone) VALUES ('%s','%s')" - yield from cursor.executemany(stmt, data) + await cursor.executemany(stmt, data) `INSERT` statements are optimized by batching the data, that is using the MySQL multiple rows syntax. @@ -183,22 +182,22 @@ Cursor query using :meth:`Cursor.execute()` to get any OUT or INOUT values. Basic usage example:: - conn = yield from aiomysql.connect(host='127.0.0.1', port=3306, - user='root', password='', - db='mysql', loop=self.loop) + conn = await aiomysql.connect(host='127.0.0.1', port=3306, + user='root', password='', + db='mysql', loop=self.loop) - cur = yield from conn.cursor() - yield from cur.execute("""CREATE PROCEDURE myinc(p1 INT) - BEGIN - SELECT p1 + 1; - END - """) + cur = await conn.cursor() + await cur.execute("""CREATE PROCEDURE myinc(p1 INT) + BEGIN + SELECT p1 + 1; + END + """) - yield from cur.callproc('myinc', [1]) - (ret, ) = yield from cur.fetchone() + await cur.callproc('myinc', [1]) + (ret, ) = await cur.fetchone() assert 2, ret - yield from cur.close() + await cur.close() conn.close() Compatibility warning: The act of calling a stored procedure @@ -229,15 +228,15 @@ Cursor due to the specified number of rows not being available, fewer rows may be returned :: - cursor = yield from connection.cursor() - yield from cursor.execute("SELECT * FROM test;") + cursor = await connection.cursor() + await cursor.execute("SELECT * FROM test;") r = cursor.fetchmany(2) print(r) # [(1, 100, "abc'def"), (2, None, 'dada')] - r = yield from cursor.fetchmany(2) + r = await cursor.fetchmany(2) print(r) # [(3, 42, 'bar')] - r = yield from cursor.fetchmany(2) + r = await cursor.fetchmany(2) print(r) # [] @@ -248,8 +247,8 @@ Cursor :ref:`Coroutine ` returns all rows of a query result set:: - yield from cursor.execute("SELECT * FROM test;") - r = yield from cursor.fetchall() + await cursor.execute("SELECT * FROM test;") + r = await cursor.fetchall() print(r) # [(1, 100, "abc'def"), (2, None, 'dada'), (3, 42, 'bar')] @@ -274,7 +273,7 @@ Cursor probably to catch both exceptions in your code:: try: - yield from cur.scroll(1000 * 1000) + await cur.scroll(1000 * 1000) except (ProgrammingError, IndexError), exc: deal_with_it(exc) @@ -292,21 +291,20 @@ Cursor loop = asyncio.get_event_loop() - @asyncio.coroutine - def test_example(): - conn = yield from aiomysql.connect(host='127.0.0.1', port=3306, - user='root', password='', - db='mysql', loop=loop) + async def test_example(): + conn = await aiomysql.connect(host='127.0.0.1', port=3306, + user='root', password='', + db='mysql', loop=loop) # create dict cursor - cursor = yield from conn.cursor(aiomysql.DictCursor) + cursor = await conn.cursor(aiomysql.DictCursor) # execute sql query - yield from cursor.execute( + await cursor.execute( "SELECT * from people where name='bob'") # fetch all results - r = yield from cursor.fetchone() + r = await cursor.fetchone() print(r) # {'age': 20, 'DOB': datetime.datetime(1990, 2, 6, 23, 4, 56), # 'name': 'bob'} @@ -332,21 +330,20 @@ Cursor loop = asyncio.get_event_loop() - @asyncio.coroutine - def test_example(): - conn = yield from aiomysql.connect(host='127.0.0.1', port=3306, - user='root', password='', - db='mysql', loop=loop) + async def test_example(): + conn = await aiomysql.connect(host='127.0.0.1', port=3306, + user='root', password='', + db='mysql', loop=loop) # create your dict cursor - cursor = yield from conn.cursor(AttrDictCursor) + cursor = await conn.cursor(AttrDictCursor) # execute sql query - yield from cursor.execute( + await cursor.execute( "SELECT * from people where name='bob'") # fetch all results - r = yield from cursor.fetchone() + r = await cursor.fetchone() print(r) # {'age': 20, 'DOB': datetime.datetime(1990, 2, 6, 23, 4, 56), # 'name': 'bob'} diff --git a/docs/pool.rst b/docs/pool.rst index c8323db0..ae67e724 100644 --- a/docs/pool.rst +++ b/docs/pool.rst @@ -14,20 +14,19 @@ The basic usage is:: loop = asyncio.get_event_loop() - @asyncio.coroutine - def go(): - pool = yield from aiomysql.create_pool(host='127.0.0.1', port=3306, - user='root', password='', - db='mysql', loop=loop, autocommit=False) - - with (yield from pool) as conn: - cur = yield from conn.cursor() - yield from cur.execute("SELECT 10") + async def go(): + pool = await aiomysql.create_pool(host='127.0.0.1', port=3306, + user='root', password='', + db='mysql', loop=loop, autocommit=False) + + async with pool.acquire() as conn: + cur = await conn.cursor() + await cur.execute("SELECT 10") # print(cur.description) - (r,) = yield from cur.fetchone() + (r,) = await cur.fetchone() assert r == 10 pool.close() - yield from pool.wait_closed() + await pool.wait_closed() loop.run_until_complete(go()) @@ -62,8 +61,8 @@ The basic usage is:: The most important way to use it is getting connection in *with statement*:: - with (yield from pool) as conn: - cur = yield from conn.cursor() + async with pool.acquire() as conn: + cur = await conn.cursor() See also :meth:`Pool.acquire` and :meth:`Pool.release` for acquring diff --git a/docs/sa.rst b/docs/sa.rst index 339e2f81..42b059c5 100644 --- a/docs/sa.rst +++ b/docs/sa.rst @@ -35,17 +35,16 @@ Example:: sa.Column('val', sa.String(255))) - @asyncio.coroutine - def go(): - engine = yield from create_engine(user='root', + async def go(): + engine = await create_engine(user='root', db='test_pymysql', host='127.0.0.1', password='') - with (yield from engine) as conn: - yield from conn.execute(tbl.insert().values(val='abc')) + async with engine.acquire() as conn: + await conn.execute(tbl.insert().values(val='abc')) - res = yield from conn.execute(tbl.select()) + res = await conn.execute(tbl.select()) for row in res: print(row.id, row.val) @@ -202,26 +201,26 @@ Connection to be used in the execution. Typically, the format is either a dictionary passed to \*multiparams:: - yield from conn.execute( + await conn.execute( table.insert(), {"id":1, "value":"v1"} ) ...or individual key/values interpreted by \**params:: - yield from conn.execute( + await conn.execute( table.insert(), id=1, value="v1" ) In the case that a plain SQL string is passed, a tuple or individual values in \*multiparams may be passed:: - yield from conn.execute( + await conn.execute( "INSERT INTO table (id, value) VALUES (%d, %s)", (1, "v1") ) - yield from conn.execute( + await conn.execute( "INSERT INTO table (id, value) VALUES (%s, %s)", 1, "v1" ) @@ -257,10 +256,10 @@ Connection an emulated transaction within the scope of the enclosing transaction, that is:: - trans = yield from conn.begin() # outermost transaction - trans2 = yield from conn.begin() # "inner" - yield from trans2.commit() # does nothing - yield from trans.commit() # actually commits + trans = await conn.begin() # outermost transaction + trans2 = await conn.begin() # "inner" + await trans2.commit() # does nothing + await trans.commit() # actually commits Calls to :meth:`.Transaction.commit` only have an effect when invoked via the outermost :class:`.Transaction` object, though the @@ -364,7 +363,7 @@ ResultProxy case-sensitive column name, or by :class:`sqlalchemy.schema.Column`` object. e.g.:: - for row in (yield from conn.execute(...)): + async for row in conn.execute(...): col1 = row[0] # access via integer position col2 = row['col2'] # access via name col3 = row[mytable.c.mycol] # access via Column object. @@ -531,14 +530,14 @@ Transaction objects calling the :meth:`SAConnection.begin` method of :class:`SAConnection`:: - with (yield from engine) as conn: - trans = yield from conn.begin() + async with engine.acquire() as conn: + trans = await conn.begin() try: - yield from conn.execute("insert into x (a, b) values (1, 2)") + await conn.execute("insert into x (a, b) values (1, 2)") except Exception: - yield from trans.rollback() + await trans.rollback() else: - yield from trans.commit() + await trans.commit() The object provides :meth:`.rollback` and :meth:`.commit` methods in order to control transaction boundaries. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 17a4d2d2..c8c04d11 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -6,7 +6,7 @@ Tutorial Python database access modules all have similar interfaces, described by the :term:`DBAPI`. Most relational databases use the same synchronous interface, *aiomysql* tries to provide same api you just need -to use ``yield from conn.f()`` instead of just call ``conn.f()`` for +to use ``await conn.f()`` instead of just call ``conn.f()`` for every method. Installation @@ -29,18 +29,17 @@ Lets start from basic example:: loop = asyncio.get_event_loop() - @asyncio.coroutine - def test_example(): - conn = yield from aiomysql.connect(host='127.0.0.1', port=3306, + async def test_example(): + conn = await aiomysql.connect(host='127.0.0.1', port=3306, user='root', password='', db='mysql', loop=loop) - cur = yield from conn.cursor() - yield from cur.execute("SELECT Host,User FROM user") + cur = await conn.cursor() + await cur.execute("SELECT Host,User FROM user") print(cur.description) - r = yield from cur.fetchall() + r = await cur.fetchall() print(r) - yield from cur.close() + await cur.close() conn.close() loop.run_until_complete(test_example()) @@ -60,10 +59,10 @@ processing statements. Example uses cursor to issue a ``SELECT Host,User FROM user;`` statement, which returns a list of `host` and `user` from :term:`MySQL` system table ``user``:: - cur = yield from conn.cursor() - yield from cur.execute("SELECT Host,User FROM user") + cur = await conn.cursor() + await cur.execute("SELECT Host,User FROM user") print(cur.description) - r = yield from cur.fetchall() + r = await cur.fetchall() The cursor object's :meth:`Cursor.execute()` method sends the query the server and :meth:`Cursor.fetchall()` retrieves rows. @@ -72,7 +71,7 @@ Finally, the script invokes :meth:`Cursor.close()` coroutine and connection object's :meth:`Connection.close()` method to disconnect from the server:: - yield from cur.close() + await cur.close() conn.close() After that, ``conn`` becomes invalid and should not be used to access the From c1c2fb53c28d4f1a88a96a134063698d46519d80 Mon Sep 17 00:00:00 2001 From: wooptroop <62987377+wooptroop@users.noreply.github.com> Date: Sat, 22 Jan 2022 20:14:30 +0530 Subject: [PATCH 18/92] Add parameter "pool_recycle" to Pool object. (#527) --- docs/pool.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/pool.rst b/docs/pool.rst index ae67e724..b2683715 100644 --- a/docs/pool.rst +++ b/docs/pool.rst @@ -44,6 +44,9 @@ The basic usage is:: :param kwargs: The function accepts all parameters that :func:`aiomysql.connect` does plus optional keyword-only parameters *loop*, *minsize*, *maxsize*. + :param float pool_recycle: number of seconds after which connection is + recycled, helps to deal with stale connections in pool, default + value is -1, means recycling logic is disabled. :returns: :class:`Pool` instance. From 0a64588d04c2f52721a8aa0fc147b36d67fce819 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sat, 22 Jan 2022 16:32:12 +0100 Subject: [PATCH 19/92] Don't send sys.argv[0] as program_name to MySQL server by default (#681) fixes #620, closes #621 --- CHANGES.txt | 6 ++++++ aiomysql/connection.py | 4 +--- docs/connection.rst | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1edf9c66..6081dbdf 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,12 @@ Changes ------- +To be included in 1.0.0 (unreleased) +^^^^^^^^^^^^^^^^^^^ + +* Don't send sys.argv[0] as program_name to MySQL server by default #620 + + 0.0.22 (2021-11-14) ^^^^^^^^^^^^^^^^^^^ diff --git a/aiomysql/connection.py b/aiomysql/connection.py index afce4e8e..673c9787 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -131,7 +131,7 @@ def __init__(self, host="localhost", user=None, password="", when using IAM authentication with Amazon RDS. (default: Server Default) :param program_name: Program name string to provide when - handshaking with MySQL. (default: sys.argv[0]) + handshaking with MySQL. (omitted by default) :param server_public_key: SHA256 authentication plugin public key value. :param loop: asyncio loop @@ -185,8 +185,6 @@ def __init__(self, host="localhost", user=None, password="", } if program_name: self._connect_attrs["program_name"] = program_name - elif sys.argv: - self._connect_attrs["program_name"] = sys.argv[0] self._unix_socket = unix_socket if charset: diff --git a/docs/connection.rst b/docs/connection.rst index 73a61a28..b8426678 100644 --- a/docs/connection.rst +++ b/docs/connection.rst @@ -88,7 +88,9 @@ Example:: when using IAM authentication with Amazon RDS. (default: Server Default) :param program_name: Program name string to provide when - handshaking with MySQL. (default: sys.argv[0]) + handshaking with MySQL. (omitted by default) + .. versionchanged:: 1.0 + ``sys.argv[0]`` is no longer passed by default :param server_public_key: SHA256 authenticaiton plugin public key value. :param loop: asyncio event loop instance or ``None`` for default one. :returns: :class:`Connection` instance. From 71ea9f3597dc80d27e6e9e02c8d8cd7c316c54d2 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sat, 22 Jan 2022 17:01:12 +0100 Subject: [PATCH 20/92] fix formatting in documentation after #681 (#683) --- docs/connection.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/connection.rst b/docs/connection.rst index b8426678..ef9482ac 100644 --- a/docs/connection.rst +++ b/docs/connection.rst @@ -89,8 +89,9 @@ Example:: (default: Server Default) :param program_name: Program name string to provide when handshaking with MySQL. (omitted by default) - .. versionchanged:: 1.0 - ``sys.argv[0]`` is no longer passed by default + + .. versionchanged:: 1.0 + ``sys.argv[0]`` is no longer passed by default :param server_public_key: SHA256 authenticaiton plugin public key value. :param loop: asyncio event loop instance or ``None`` for default one. :returns: :class:`Connection` instance. From 9100eab2210024199ef44b32b97085685091f03a Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sat, 22 Jan 2022 21:12:59 +0100 Subject: [PATCH 21/92] move pyroma to lint ci workflow, fail if rating is below 10/10 (#685) --- .github/workflows/ci.yml | 4 ---- .github/workflows/lint.yml | 8 ++++++++ CHANGES.txt | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f73c2937..f7698f39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,10 +89,6 @@ jobs: run: | python setup.py check --restructuredtext - - name: Run pyroma - run: | - python -m pyroma -d . - # this ensures our database is ready. typically by the time the preparations have completed its first start logic. # unfortunately we need this hacky workaround as GitHub Actions service containers can't reference data from our repo. - name: Prepare mysql diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a89afdf6..0b000eb7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,6 +37,14 @@ jobs: key: pip-lint path: ${{ steps.pip-cache.outputs.dir }} + - name: Install dependencies + run: | + python -m pip install --upgrade --requirement requirements-dev.txt + + - name: Run pyroma + run: | + python -m pyroma --min 10 --directory . + - name: flake8 Lint uses: py-actions/flake8@v2.0.0 with: diff --git a/CHANGES.txt b/CHANGES.txt index 6081dbdf..2dd42fdd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,7 +2,7 @@ Changes ------- To be included in 1.0.0 (unreleased) -^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * Don't send sys.argv[0] as program_name to MySQL server by default #620 From 72a8432359bd2135725211911adea994dd98825d Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sun, 23 Jan 2022 15:03:35 +0100 Subject: [PATCH 22/92] Allow getpass.getuser() to fail (#687) * Allow getpass.getuser() to fail Fixes https://github.com/aio-libs/aiomysql/issues/587 * add changelog entry Co-authored-by: Richard Schwab --- CHANGES.txt | 1 + aiomysql/connection.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 2dd42fdd..4f3c82ef 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,7 @@ To be included in 1.0.0 (unreleased) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * Don't send sys.argv[0] as program_name to MySQL server by default #620 +* Allow running process as anonymous uid #587 0.0.22 (2021-11-14) diff --git a/aiomysql/connection.py b/aiomysql/connection.py index 673c9787..01496d1c 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -42,7 +42,10 @@ from .utils import _ConnectionContextManager, _ContextManager from .log import logger -DEFAULT_USER = getpass.getuser() +try: + DEFAULT_USER = getpass.getuser() +except KeyError: + DEFAULT_USER = "unknown" def connect(host="localhost", user=None, password="", From fce0355353bf779b7db28c1f68fce78b2c559128 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 10:33:17 +0100 Subject: [PATCH 23/92] Bump coverage from 6.2 to 6.3 (#691) Bumps [coverage](https://github.com/nedbat/coveragepy) from 6.2 to 6.3. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/6.2...6.3) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f30f2dcf..14a65bc2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -coverage==6.2 +coverage==6.3 flake8==4.0.1 ipdb==0.13.9 pytest==6.2.5 From 6887375d0c991b125801423ba1de6a9bca1ee24b Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Wed, 26 Jan 2022 18:41:39 +0100 Subject: [PATCH 24/92] Fix MySQL 8.0 tests, properly close timed out connections (#660) * fix closed MySQL 8.0 connections not always being detected properly ensure connections are closed properly when the server connection was lost * implement a custom StreamReader to avoid accessing the internal attribute `_eof` on asyncio.StreamReader * ensure connections are closed when raising an InternalError --- CHANGES.txt | 4 +++ aiomysql/connection.py | 73 ++++++++++++++++++++++++++++++++++++++---- aiomysql/pool.py | 10 +++++- 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4f3c82ef..b5b276b1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,10 @@ To be included in 1.0.0 (unreleased) * Don't send sys.argv[0] as program_name to MySQL server by default #620 * Allow running process as anonymous uid #587 +* Fix timed out MySQL 8.0 connections raising InternalError rather than OperationalError #660 +* Fix timed out MySQL 8.0 connections being returned from Pool #660 +* Ensure connections are properly closed before raising an OperationalError when the server connection is lost #660 +* Ensure connections are properly closed before raising an InternalError when packet sequence numbers are out of sync #660 0.0.22 (2021-11-14) diff --git a/aiomysql/connection.py b/aiomysql/connection.py index 01496d1c..f2b139b1 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -15,6 +15,7 @@ from pymysql.constants import SERVER_STATUS from pymysql.constants import CLIENT from pymysql.constants import COMMAND +from pymysql.constants import CR from pymysql.constants import FIELD_TYPE from pymysql.util import byte2int, int2byte from pymysql.converters import (escape_item, encoders, decoders, @@ -79,6 +80,57 @@ async def _connect(*args, **kwargs): return conn +async def _open_connection(host=None, port=None, **kwds): + """This is based on asyncio.open_connection, allowing us to use a custom + StreamReader. + + `limit` arg has been removed as we don't currently use it. + """ + loop = asyncio.events.get_running_loop() + reader = _StreamReader(loop=loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=loop) + transport, _ = await loop.create_connection( + lambda: protocol, host, port, **kwds) + writer = asyncio.StreamWriter(transport, protocol, reader, loop) + return reader, writer + + +async def _open_unix_connection(path=None, **kwds): + """This is based on asyncio.open_unix_connection, allowing us to use a custom + StreamReader. + + `limit` arg has been removed as we don't currently use it. + """ + loop = asyncio.events.get_running_loop() + + reader = _StreamReader(loop=loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=loop) + transport, _ = await loop.create_unix_connection( + lambda: protocol, path, **kwds) + writer = asyncio.StreamWriter(transport, protocol, reader, loop) + return reader, writer + + +class _StreamReader(asyncio.StreamReader): + """This StreamReader exposes whether EOF was received, allowing us to + discard the associated connection instead of returning it from the pool + when checking free connections in Pool._fill_free_pool(). + + `limit` arg has been removed as we don't currently use it. + """ + def __init__(self, loop=None): + self._eof_received = False + super().__init__(loop=loop) + + def feed_eof(self) -> None: + self._eof_received = True + super().feed_eof() + + @property + def eof_received(self): + return self._eof_received + + class Connection: """Representation of a socket with a mysql server. @@ -471,13 +523,13 @@ async def set_charset(self, charset): async def _connect(self): # TODO: Set close callback - # raise OperationalError(2006, + # raise OperationalError(CR.CR_SERVER_GONE_ERROR, # "MySQL server has gone away (%r)" % (e,)) try: if self._unix_socket and self._host in ('localhost', '127.0.0.1'): self._reader, self._writer = await \ asyncio.wait_for( - asyncio.open_unix_connection( + _open_unix_connection( self._unix_socket), timeout=self.connect_timeout) self.host_info = "Localhost via UNIX socket: " + \ @@ -485,7 +537,7 @@ async def _connect(self): else: self._reader, self._writer = await \ asyncio.wait_for( - asyncio.open_connection( + _open_connection( self._host, self._port), timeout=self.connect_timeout) @@ -570,6 +622,13 @@ async def _read_packet(self, packet_type=MysqlPacket): # we increment in both write_packet and read_packet. The count # is reset at new COMMAND PHASE. if packet_number != self._next_seq_id: + self.close() + if packet_number == 0: + # MySQL 8.0 sends error packet with seqno==0 when shutdown + raise OperationalError( + CR.CR_SERVER_LOST, + "Lost connection to MySQL server during query") + raise InternalError( "Packet sequence number wrong - got %d expected %d" % (packet_number, self._next_seq_id)) @@ -597,10 +656,12 @@ async def _read_bytes(self, num_bytes): data = await self._reader.readexactly(num_bytes) except asyncio.IncompleteReadError as e: msg = "Lost connection to MySQL server during query" - raise OperationalError(2013, msg) from e + self.close() + raise OperationalError(CR.CR_SERVER_LOST, msg) from e except (IOError, OSError) as e: msg = "Lost connection to MySQL server during query (%s)" % (e,) - raise OperationalError(2013, msg) from e + self.close() + raise OperationalError(CR.CR_SERVER_LOST, msg) from e return data def _write_bytes(self, data): @@ -704,7 +765,7 @@ async def _request_authentication(self): # TCP connection not at start. Passing in a socket to # open_connection will cause it to negotiate TLS on an existing # connection not initiate a new one. - self._reader, self._writer = await asyncio.open_connection( + self._reader, self._writer = await _open_connection( sock=raw_sock, ssl=self._ssl_context, server_hostname=self._host ) diff --git a/aiomysql/pool.py b/aiomysql/pool.py index a17e3fca..3eacb47d 100644 --- a/aiomysql/pool.py +++ b/aiomysql/pool.py @@ -143,7 +143,7 @@ async def _acquire(self): await self._cond.wait() async def _fill_free_pool(self, override_min): - # iterate over free connections and remove timeouted ones + # iterate over free connections and remove timed out ones free_size = len(self._free) n = 0 while n < free_size: @@ -152,6 +152,14 @@ async def _fill_free_pool(self, override_min): self._free.pop() conn.close() + # On MySQL 8.0 a timed out connection sends an error packet before + # closing the connection, preventing us from relying on at_eof(). + # This relies on our custom StreamReader, as eof_received is not + # present in asyncio.StreamReader. + elif conn._reader.eof_received: + self._free.pop() + conn.close() + elif (self._recycle > -1 and self._loop.time() - conn.last_usage > self._recycle): self._free.pop() From 6191960ee5b67516b976dfbadfd4cf29f4b83355 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Wed, 26 Jan 2022 18:54:29 +0100 Subject: [PATCH 25/92] remove deprecated asyncio.coroutines syntax from tests (#663) tests._testutils.run_until_complete is not referenced anymore since moving to pytest.mark.run_loop in 819d4ec5fb967a679cad134eba885a1ce4bb5cee --- tests/_testutils.py | 15 ---------- tests/base.py | 69 ++++++++++++++++++++++----------------------- tests/conftest.py | 16 ++++------- 3 files changed, 39 insertions(+), 61 deletions(-) diff --git a/tests/_testutils.py b/tests/_testutils.py index 1f198cc9..c3761a54 100644 --- a/tests/_testutils.py +++ b/tests/_testutils.py @@ -1,21 +1,6 @@ import asyncio import unittest -from functools import wraps - - -def run_until_complete(fun): - if not asyncio.iscoroutinefunction(fun): - fun = asyncio.coroutine(fun) - - @wraps(fun) - def wrapper(test, *args, **kw): - loop = test.loop - ret = loop.run_until_complete( - asyncio.wait_for(fun(test, *args, **kw), 15)) - return ret - return wrapper - class BaseTest(unittest.TestCase): """Base test case for unittests. diff --git a/tests/base.py b/tests/base.py index ba306a87..193667c6 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,4 +1,3 @@ -import asyncio import os import aiomysql from tests._testutils import BaseTest @@ -6,24 +5,23 @@ class AIOPyMySQLTestCase(BaseTest): - @asyncio.coroutine - def _connect_all(self): - conn1 = yield from aiomysql.connect(loop=self.loop, host=self.host, - port=self.port, user=self.user, - db=self.db, - password=self.password, - use_unicode=True, echo=True) - conn2 = yield from aiomysql.connect(loop=self.loop, host=self.host, - port=self.port, user=self.user, - db=self.other_db, - password=self.password, - use_unicode=False, echo=True) - conn3 = yield from aiomysql.connect(loop=self.loop, host=self.host, - port=self.port, user=self.user, - db=self.db, - password=self.password, - use_unicode=True, echo=True, - local_infile=True) + async def _connect_all(self): + conn1 = await aiomysql.connect(loop=self.loop, host=self.host, + port=self.port, user=self.user, + db=self.db, + password=self.password, + use_unicode=True, echo=True) + conn2 = await aiomysql.connect(loop=self.loop, host=self.host, + port=self.port, user=self.user, + db=self.other_db, + password=self.password, + use_unicode=False, echo=True) + conn3 = await aiomysql.connect(loop=self.loop, host=self.host, + port=self.port, user=self.user, + db=self.db, + password=self.password, + use_unicode=True, echo=True, + local_infile=True) self.connections = [conn1, conn2, conn3] @@ -45,9 +43,9 @@ def tearDown(self): self.doCleanups() super(AIOPyMySQLTestCase, self).tearDown() - @asyncio.coroutine - def connect(self, host=None, user=None, password=None, - db=None, use_unicode=True, no_delay=None, port=None, **kwargs): + async def connect(self, host=None, user=None, password=None, + db=None, use_unicode=True, no_delay=None, port=None, + **kwargs): if host is None: host = self.host if user is None: @@ -58,18 +56,17 @@ def connect(self, host=None, user=None, password=None, db = self.db if port is None: port = self.port - conn = yield from aiomysql.connect(loop=self.loop, host=host, - user=user, password=password, - db=db, use_unicode=use_unicode, - no_delay=no_delay, port=port, - **kwargs) + conn = await aiomysql.connect(loop=self.loop, host=host, + user=user, password=password, + db=db, use_unicode=use_unicode, + no_delay=no_delay, port=port, + **kwargs) self.addCleanup(conn.close) return conn - @asyncio.coroutine - def create_pool(self, host=None, user=None, password=None, - db=None, use_unicode=True, no_delay=None, - port=None, **kwargs): + async def create_pool(self, host=None, user=None, password=None, + db=None, use_unicode=True, no_delay=None, + port=None, **kwargs): if host is None: host = self.host if user is None: @@ -80,10 +77,10 @@ def create_pool(self, host=None, user=None, password=None, db = self.db if port is None: port = self.port - pool = yield from aiomysql.create_pool(loop=self.loop, host=host, - user=user, password=password, - db=db, use_unicode=use_unicode, - no_delay=no_delay, port=port, - **kwargs) + pool = await aiomysql.create_pool(loop=self.loop, host=host, + user=user, password=password, + db=db, use_unicode=use_unicode, + no_delay=no_delay, port=port, + **kwargs) self.addCleanup(pool.close) return pool diff --git a/tests/conftest.py b/tests/conftest.py index 988643b5..22472cb8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -112,10 +112,8 @@ def mysql_params(mysql_server): # TODO: fix this workaround -@asyncio.coroutine -def _cursor_wrapper(conn): - cur = yield from conn.cursor() - return cur +async def _cursor_wrapper(conn): + return await conn.cursor() @pytest.fixture @@ -137,12 +135,11 @@ def connection(mysql_params, loop): def connection_creator(mysql_params, loop): connections = [] - @asyncio.coroutine - def f(**kw): + async def f(**kw): conn_kw = mysql_params.copy() conn_kw.update(kw) _loop = conn_kw.pop('loop', loop) - conn = yield from aiomysql.connect(loop=_loop, **conn_kw) + conn = await aiomysql.connect(loop=_loop, **conn_kw) connections.append(conn) return conn @@ -159,12 +156,11 @@ def f(**kw): def pool_creator(mysql_params, loop): pools = [] - @asyncio.coroutine - def f(**kw): + async def f(**kw): conn_kw = mysql_params.copy() conn_kw.update(kw) _loop = conn_kw.pop('loop', loop) - pool = yield from aiomysql.create_pool(loop=_loop, **conn_kw) + pool = await aiomysql.create_pool(loop=_loop, **conn_kw) pools.append(pool) return pool From ee9fde22183fb49d1956679122260d0b3cf231f2 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 28 Jan 2022 13:29:22 +0100 Subject: [PATCH 26/92] restructure CI workflow (#692) --- .github/workflows/ci.yml | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7698f39..61a19ebe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,14 +28,14 @@ jobs: # - '3.10' # - '3.11.0-alpha.3' db: - - 'mysql:5.7' - - 'mysql:8.0' - - 'mariadb:10.2' - - 'mariadb:10.3' - - 'mariadb:10.4' - - 'mariadb:10.5' - - 'mariadb:10.6' - - 'mariadb:10.7' + - [mysql, '5.7'] + - [mysql, '8.0'] + - [mariadb, '10.2'] + - [mariadb, '10.3'] + - [mariadb, '10.4'] + - [mariadb, '10.5'] + - [mariadb, '10.6'] + - [mariadb, '10.7'] fail-fast: false runs-on: ${{ matrix.os }} @@ -46,7 +46,7 @@ jobs: services: mysql: - image: '${{ matrix.db }}' + image: "${{ join(matrix.db, ':') }}" ports: - 3306:3306 options: '--name=mysqld' @@ -117,29 +117,18 @@ jobs: - name: Run tests run: | - export DB="${MATRIX_DB%%:*}" - export DBTAG="${MATRIX_DB##*:}" - # timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs timeout --preserve-status --signal=INT --verbose 5m \ - pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql ./tests + pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql ./tests env: PYTHONUNBUFFERED: 1 - MATRIX_DB: '${{ matrix.db }}' + DB: '${{ matrix.db[0] }}' + DBTAG: '${{ matrix.db[1] }}' timeout-minutes: 6 - - name: Build coverage flag - run: | - COVERAGE_FLAG="${MATRIX_OS}_${MATRIX_PY}_${MATRIX_DB//:/-}" - echo "COVERAGE_FLAG=$COVERAGE_FLAG" | tee -a "$GITHUB_ENV" - env: - MATRIX_OS: '${{ matrix.os }}' - MATRIX_PY: '${{ matrix.py }}' - MATRIX_DB: '${{ matrix.db }}' - - name: Upload coverage uses: codecov/codecov-action@v2.1.0 with: file: ./coverage.xml - flags: "${{ env.COVERAGE_FLAG }}" + flags: "${{ matrix.os }}_${{ matrix.py }}_${{ join(matrix.db, '-') }}" fail_ci_if_error: true From b5158590dd76f255645af98389966ead58911095 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 28 Jan 2022 17:23:15 +0100 Subject: [PATCH 27/92] ensure test_scroll_errors and test_sscursor_scroll_errors initialize their tables (#694) this resulted in test failures in some cases. --- tests/test_cursor.py | 1 + tests/test_sscursor.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 2cbb9d6b..23b87a98 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -94,6 +94,7 @@ async def test_scroll_absolute(connection_creator): @pytest.mark.run_loop async def test_scroll_errors(connection_creator): conn = await connection_creator() + await _prepare(conn) cur = await conn.cursor() with pytest.raises(ProgrammingError): diff --git a/tests/test_sscursor.py b/tests/test_sscursor.py index 0d6ac264..c020fe81 100644 --- a/tests/test_sscursor.py +++ b/tests/test_sscursor.py @@ -132,6 +132,7 @@ async def test_sscursor_scroll_absolute(connection): @pytest.mark.run_loop async def test_sscursor_scroll_errors(connection): conn = connection + await _prepare(conn) cursor = await conn.cursor(SSCursor) await cursor.execute('SELECT * FROM tz_data;') @@ -151,7 +152,7 @@ async def test_sscursor_scroll_errors(connection): async def test_sscursor_cancel(connection): conn = connection cur = await conn.cursor(SSCursor) - # Prepare ALOT of data + # Prepare A LOT of data await cur.execute('DROP TABLE IF EXISTS long_seq;') await cur.execute( From 29550527f3de2d608ed45b7aad8634aecc667f77 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 28 Jan 2022 17:32:18 +0100 Subject: [PATCH 28/92] parameterize mysql_address in tests, allow passing one or more addresses via pytest args (#693) --- .github/workflows/ci.yml | 2 +- tests/conftest.py | 50 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61a19ebe..83465c67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,7 +119,7 @@ jobs: run: | # timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs timeout --preserve-status --signal=INT --verbose 5m \ - pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql ./tests + pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql ./tests --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306" env: PYTHONUNBUFFERED: 1 DB: '${{ matrix.db[0] }}' diff --git a/tests/conftest.py b/tests/conftest.py index 22472cb8..a42172c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,41 @@ def pytest_generate_tests(metafunc): loop_type = ['asyncio', 'uvloop'] if uvloop else ['asyncio'] metafunc.parametrize("loop_type", loop_type) + if "mysql_address" in metafunc.fixturenames: + mysql_addresses = [] + ids = [] + + opt_mysql_address = list(metafunc.config.getoption("mysql_address")) + for i in range(len(opt_mysql_address)): + if "=" in opt_mysql_address[i]: + label, addr = opt_mysql_address[i].split("=", 1) + ids.append(label) + else: + addr = opt_mysql_address[i] + ids.append("tcp{}".format(i)) + + if ":" in addr: + addr = addr.split(":", 1) + mysql_addresses.append((addr[0], int(addr[1]))) + else: + mysql_addresses.append((addr, 3306)) + + # default to connecting to localhost + if len(mysql_addresses) == 0: + mysql_addresses = [("127.0.0.1", 3306)] + ids = ["tcp-local"] + + assert len(mysql_addresses) == len(set(mysql_addresses)), \ + "mysql targets are not unique" + assert len(ids) == len(set(ids)), \ + "mysql target names are not unique" + + metafunc.parametrize("mysql_address", + mysql_addresses, + ids=ids, + scope="session", + ) + # This is here unless someone fixes the generate_tests bit @pytest.fixture(scope='session') @@ -101,6 +136,15 @@ def pytest_configure(config): ) +def pytest_addoption(parser): + parser.addoption( + "--mysql-address", + action="append", + default=[], + help="list of addresses to connect to: [name=]host[:port]", + ) + + @pytest.fixture def mysql_params(mysql_server): params = {**mysql_server['conn_params'], @@ -205,7 +249,7 @@ def ensure_mysql_version(request, mysql_image, mysql_tag): @pytest.fixture(scope='session') -def mysql_server(mysql_image, mysql_tag): +def mysql_server(mysql_image, mysql_tag, mysql_address): ssl_directory = os.path.join(os.path.dirname(__file__), 'ssl_resources', 'ssl') ca_file = os.path.join(ssl_directory, 'ca.pem') @@ -216,8 +260,8 @@ def mysql_server(mysql_image, mysql_tag): # ctx.verify_mode = ssl.CERT_NONE server_params = { - 'host': '127.0.0.1', - 'port': 3306, + 'host': mysql_address[0], + 'port': mysql_address[1], 'user': 'root', 'password': os.environ.get("MYSQL_ROOT_PASSWORD"), 'ssl': ctx, From 8fe7e535726414eafc49db3e18d7cb5d95c77b0c Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 28 Jan 2022 19:13:00 +0100 Subject: [PATCH 29/92] implement a shortcut for determining secure connections, now supporting unix sockets (#695) ports https://github.com/PyMySQL/PyMySQL/pull/696 ports check for server tls support for secure connections from https://github.com/PyMySQL/PyMySQL/pull/353 --- CHANGES.txt | 1 + aiomysql/connection.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b5b276b1..b537f279 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -10,6 +10,7 @@ To be included in 1.0.0 (unreleased) * Fix timed out MySQL 8.0 connections being returned from Pool #660 * Ensure connections are properly closed before raising an OperationalError when the server connection is lost #660 * Ensure connections are properly closed before raising an InternalError when packet sequence numbers are out of sync #660 +* Unix sockets are now internally considered secure, allowing sha256_password and caching_sha2_password auth methods to be used #695 0.0.22 (2021-11-14) diff --git a/aiomysql/connection.py b/aiomysql/connection.py index f2b139b1..1bd279d3 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -229,6 +229,7 @@ def __init__(self, host="localhost", user=None, password="", self._client_auth_plugin = auth_plugin self._server_auth_plugin = "" self._auth_plugin_used = "" + self._secure = False self.server_public_key = server_public_key self.salt = None @@ -526,7 +527,7 @@ async def _connect(self): # raise OperationalError(CR.CR_SERVER_GONE_ERROR, # "MySQL server has gone away (%r)" % (e,)) try: - if self._unix_socket and self._host in ('localhost', '127.0.0.1'): + if self._unix_socket: self._reader, self._writer = await \ asyncio.wait_for( _open_unix_connection( @@ -534,6 +535,7 @@ async def _connect(self): timeout=self.connect_timeout) self.host_info = "Localhost via UNIX socket: " + \ self._unix_socket + self._secure = True else: self._reader, self._writer = await \ asyncio.wait_for( @@ -743,7 +745,7 @@ async def _request_authentication(self): if self.user is None: raise ValueError("Did not specify a username") - if self._ssl_context: + if self._ssl_context and self.server_capabilities & CLIENT.SSL: # capablities, max packet, charset data = struct.pack(' Date: Fri, 28 Jan 2022 20:53:26 +0100 Subject: [PATCH 30/92] fix unit socket implementation, most tests should be fine now (#696) configure custom socket path for mysql container, working around implicitly created volume folders being owned by root we should probably just not use service containers for this to avoid having to do this patching --- .github/workflows/ci.yml | 17 ++++- CHANGES.txt | 1 + tests/conftest.py | 67 +++++++++++++------ .../fixtures/{my.cnf.tmpl => my.cnf.tcp.tmpl} | 0 tests/fixtures/my.cnf.unix.tmpl | 16 +++++ tests/sa/test_sa_compiled_cache.py | 11 ++- tests/sa/test_sa_default.py | 11 ++- tests/sa/test_sa_engine.py | 11 ++- tests/ssl_resources/socket.cnf | 2 + tests/test_connection.py | 29 ++++++-- tests/test_issues.py | 2 +- tests/test_sha_connection.py | 7 ++ tests/test_ssl.py | 10 ++- 13 files changed, 146 insertions(+), 38 deletions(-) rename tests/fixtures/{my.cnf.tmpl => my.cnf.tcp.tmpl} (100%) create mode 100644 tests/fixtures/my.cnf.unix.tmpl create mode 100644 tests/ssl_resources/socket.cnf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83465c67..b8db8d57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,8 @@ jobs: image: "${{ join(matrix.db, ':') }}" ports: - 3306:3306 + volumes: + - "/tmp/run-${{ join(matrix.db, '-') }}/:/socket-mount/" options: '--name=mysqld' env: MYSQL_ROOT_PASSWORD: rootpw @@ -104,6 +106,19 @@ jobs: docker container stop mysqld docker container cp "${{ github.workspace }}/tests/ssl_resources/ssl" mysqld:/etc/mysql/ssl docker container cp "${{ github.workspace }}/tests/ssl_resources/tls.cnf" mysqld:/etc/mysql/conf.d/aiomysql-tls.cnf + + # use custom socket path + # we need to ensure that the socket path is writable for the user running the DB process in the container + sudo chmod 0777 /tmp/run-${{ join(matrix.db, '-') }} + + # mysql 5.7 container overrides the socket path in /etc/mysql/mysql.conf.d/mysqld.cnf + if [ "${{ join(matrix.db, '-') }}" = "mysql-5.7" ] + then + docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/mysql.conf.d/zz-aiomysql-socket.cnf + else + docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/conf.d/aiomysql-socket.cnf + fi + docker container start mysqld # ensure server is started up @@ -119,7 +134,7 @@ jobs: run: | # timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs timeout --preserve-status --signal=INT --verbose 5m \ - pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql ./tests --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306" + pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql ./tests --mysql-unix-socket "unix-${{ join(matrix.db, '') }}=/tmp/run-${{ join(matrix.db, '-') }}/mysql.sock" --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306" env: PYTHONUNBUFFERED: 1 DB: '${{ matrix.db[0] }}' diff --git a/CHANGES.txt b/CHANGES.txt index b537f279..9a127dd6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -11,6 +11,7 @@ To be included in 1.0.0 (unreleased) * Ensure connections are properly closed before raising an OperationalError when the server connection is lost #660 * Ensure connections are properly closed before raising an InternalError when packet sequence numbers are out of sync #660 * Unix sockets are now internally considered secure, allowing sha256_password and caching_sha2_password auth methods to be used #695 +* Test suite now also tests unix socket connections #696 0.0.22 (2021-11-14) diff --git a/tests/conftest.py b/tests/conftest.py index a42172c1..d6b0a923 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,17 @@ def pytest_generate_tests(metafunc): mysql_addresses = [] ids = [] + opt_mysql_unix_socket = \ + list(metafunc.config.getoption("mysql_unix_socket")) + for i in range(len(opt_mysql_unix_socket)): + if "=" in opt_mysql_unix_socket[i]: + label, path = opt_mysql_unix_socket[i].split("=", 1) + mysql_addresses.append(path) + ids.append(label) + else: + mysql_addresses.append(opt_mysql_unix_socket[i]) + ids.append("unix{}".format(i)) + opt_mysql_address = list(metafunc.config.getoption("mysql_address")) for i in range(len(opt_mysql_address)): if "=" in opt_mysql_address[i]: @@ -143,6 +154,12 @@ def pytest_addoption(parser): default=[], help="list of addresses to connect to: [name=]host[:port]", ) + parser.addoption( + "--mysql-unix-socket", + action="append", + default=[], + help="list of unix sockets to connect to: [name=]/path/to/socket", + ) @pytest.fixture @@ -250,23 +267,30 @@ def ensure_mysql_version(request, mysql_image, mysql_tag): @pytest.fixture(scope='session') def mysql_server(mysql_image, mysql_tag, mysql_address): - ssl_directory = os.path.join(os.path.dirname(__file__), - 'ssl_resources', 'ssl') - ca_file = os.path.join(ssl_directory, 'ca.pem') + unix_socket = type(mysql_address) is str + + if not unix_socket: + ssl_directory = os.path.join(os.path.dirname(__file__), + 'ssl_resources', 'ssl') + ca_file = os.path.join(ssl_directory, 'ca.pem') - ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - ctx.check_hostname = False - ctx.load_verify_locations(cafile=ca_file) - # ctx.verify_mode = ssl.CERT_NONE + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + ctx.check_hostname = False + ctx.load_verify_locations(cafile=ca_file) + # ctx.verify_mode = ssl.CERT_NONE server_params = { - 'host': mysql_address[0], - 'port': mysql_address[1], 'user': 'root', 'password': os.environ.get("MYSQL_ROOT_PASSWORD"), - 'ssl': ctx, } + if unix_socket: + server_params["unix_socket"] = mysql_address + else: + server_params["host"] = mysql_address[0] + server_params["port"] = mysql_address[1] + server_params["ssl"] = ctx + try: connection = pymysql.connect( db='mysql', @@ -275,21 +299,22 @@ def mysql_server(mysql_image, mysql_tag, mysql_address): **server_params) with connection.cursor() as cursor: - cursor.execute("SHOW VARIABLES LIKE '%ssl%';") + if not unix_socket: + cursor.execute("SHOW VARIABLES LIKE '%ssl%';") - result = cursor.fetchall() - result = {item['Variable_name']: - item['Value'] for item in result} + result = cursor.fetchall() + result = {item['Variable_name']: + item['Value'] for item in result} - assert result['have_ssl'] == "YES", \ - "SSL Not Enabled on MySQL" + assert result['have_ssl'] == "YES", \ + "SSL Not Enabled on MySQL" - cursor.execute("SHOW STATUS LIKE 'Ssl_version%'") + cursor.execute("SHOW STATUS LIKE 'Ssl_version%'") - result = cursor.fetchone() - # As we connected with TLS, it should start with that :D - assert result['Value'].startswith('TLS'), \ - "Not connected to the database with TLS" + result = cursor.fetchone() + # As we connected with TLS, it should start with that :D + assert result['Value'].startswith('TLS'), \ + "Not connected to the database with TLS" # Drop possibly existing old databases cursor.execute('DROP DATABASE IF EXISTS test_pymysql;') diff --git a/tests/fixtures/my.cnf.tmpl b/tests/fixtures/my.cnf.tcp.tmpl similarity index 100% rename from tests/fixtures/my.cnf.tmpl rename to tests/fixtures/my.cnf.tcp.tmpl diff --git a/tests/fixtures/my.cnf.unix.tmpl b/tests/fixtures/my.cnf.unix.tmpl new file mode 100644 index 00000000..2aad4432 --- /dev/null +++ b/tests/fixtures/my.cnf.unix.tmpl @@ -0,0 +1,16 @@ +# +# The MySQL database server configuration file. +# +[client] +user = {user} +socket = {unix_socket} +password = {password} +database = {db} +default-character-set = utf8 + +[client_with_unix_socket] +user = {user} +socket = {unix_socket} +password = {password} +database = {db} +default-character-set = utf8 diff --git a/tests/sa/test_sa_compiled_cache.py b/tests/sa/test_sa_compiled_cache.py index e8c0f5f2..38906551 100644 --- a/tests/sa/test_sa_compiled_cache.py +++ b/tests/sa/test_sa_compiled_cache.py @@ -15,12 +15,19 @@ @pytest.fixture() def make_engine(mysql_params, connection): async def _make_engine(**kwargs): + if "unix_socket" in mysql_params: + conn_args = {"unix_socket": mysql_params["unix_socket"]} + else: + conn_args = { + "host": mysql_params['host'], + "port": mysql_params['port'], + } + return (await sa.create_engine(db=mysql_params['db'], user=mysql_params['user'], password=mysql_params['password'], - host=mysql_params['host'], - port=mysql_params['port'], minsize=10, + **conn_args, **kwargs)) return _make_engine diff --git a/tests/sa/test_sa_default.py b/tests/sa/test_sa_default.py index 42c34f5b..e5f270ec 100644 --- a/tests/sa/test_sa_default.py +++ b/tests/sa/test_sa_default.py @@ -22,12 +22,19 @@ @pytest.fixture() def make_engine(mysql_params, connection): async def _make_engine(**kwargs): + if "unix_socket" in mysql_params: + conn_args = {"unix_socket": mysql_params["unix_socket"]} + else: + conn_args = { + "host": mysql_params['host'], + "port": mysql_params['port'], + } + return (await sa.create_engine(db=mysql_params['db'], user=mysql_params['user'], password=mysql_params['password'], - host=mysql_params['host'], - port=mysql_params['port'], minsize=10, + **conn_args, **kwargs)) return _make_engine diff --git a/tests/sa/test_sa_engine.py b/tests/sa/test_sa_engine.py index e514260d..ed74a96d 100644 --- a/tests/sa/test_sa_engine.py +++ b/tests/sa/test_sa_engine.py @@ -15,12 +15,19 @@ @pytest.fixture() def make_engine(connection, mysql_params): async def _make_engine(**kwargs): + if "unix_socket" in mysql_params: + conn_args = {"unix_socket": mysql_params["unix_socket"]} + else: + conn_args = { + "host": mysql_params['host'], + "port": mysql_params['port'], + } + return (await sa.create_engine(db=mysql_params['db'], user=mysql_params['user'], password=mysql_params['password'], - host=mysql_params['host'], - port=mysql_params['port'], minsize=10, + **conn_args, **kwargs)) return _make_engine diff --git a/tests/ssl_resources/socket.cnf b/tests/ssl_resources/socket.cnf new file mode 100644 index 00000000..32100e93 --- /dev/null +++ b/tests/ssl_resources/socket.cnf @@ -0,0 +1,2 @@ +[mysqld] +socket = /socket-mount/mysql.sock diff --git a/tests/test_connection.py b/tests/test_connection.py index 075039d0..af6788f3 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -10,7 +10,13 @@ @pytest.fixture() def fill_my_cnf(mysql_params): tests_root = os.path.abspath(os.path.dirname(__file__)) - path1 = os.path.join(tests_root, 'fixtures/my.cnf.tmpl') + + if "unix_socket" in mysql_params: + tmpl_path = "fixtures/my.cnf.unix.tmpl" + else: + tmpl_path = "fixtures/my.cnf.tcp.tmpl" + + path1 = os.path.join(tests_root, tmpl_path) path2 = os.path.join(tests_root, 'fixtures/my.cnf') with open(path1) as f1: tmpl = f1.read() @@ -31,8 +37,11 @@ async def test_config_file(fill_my_cnf, connection_creator, mysql_params): path = os.path.join(tests_root, 'fixtures/my.cnf') conn = await connection_creator(read_default_file=path) - assert conn.host == mysql_params['host'] - assert conn.port == mysql_params['port'] + if "unix_socket" in mysql_params: + assert conn.unix_socket == mysql_params["unix_socket"] + else: + assert conn.host == mysql_params['host'] + assert conn.port == mysql_params['port'] assert conn.user, mysql_params['user'] # make sure connection is working @@ -167,12 +176,15 @@ async def test_connection_gone_away(connection_creator): @pytest.mark.run_loop -async def test_connection_info_methods(connection_creator): +async def test_connection_info_methods(connection_creator, mysql_params): conn = await connection_creator() # trhead id is int assert isinstance(conn.thread_id(), int) assert conn.character_set_name() in ('latin1', 'utf8mb4') - assert str(conn.port) in conn.get_host_info() + if "unix_socket" in mysql_params: + assert mysql_params["unix_socket"] in conn.get_host_info() + else: + assert str(conn.port) in conn.get_host_info() assert isinstance(conn.get_server_info(), str) # protocol id is int assert isinstance(conn.get_proto_info(), int) @@ -200,8 +212,11 @@ async def test_connection_ping(connection_creator): @pytest.mark.run_loop async def test_connection_properties(connection_creator, mysql_params): conn = await connection_creator() - assert conn.host == mysql_params['host'] - assert conn.port == mysql_params['port'] + if "unix_socket" in mysql_params: + assert conn.unix_socket == mysql_params["unix_socket"] + else: + assert conn.host == mysql_params['host'] + assert conn.port == mysql_params['port'] assert conn.user == mysql_params['user'] assert conn.db == mysql_params['db'] assert conn.echo is False diff --git a/tests/test_issues.py b/tests/test_issues.py index 942bc8ed..c25e292f 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -184,7 +184,7 @@ async def test_issue_17(connection, connection_creator, mysql_params): async def test_issue_34(connection_creator): try: await connection_creator(host="localhost", port=1237, - user="root") + user="root", unix_socket=None) pytest.fail() except aiomysql.OperationalError as e: assert 2003 == e.args[0] diff --git a/tests/test_sha_connection.py b/tests/test_sha_connection.py index eb57ec3d..0789d162 100644 --- a/tests/test_sha_connection.py +++ b/tests/test_sha_connection.py @@ -39,6 +39,13 @@ async def test_sha256_nopw(mysql_server, loop): @pytest.mark.mysql_version('mysql', '8.0') @pytest.mark.run_loop async def test_sha256_pw(mysql_server, loop): + # https://dev.mysql.com/doc/refman/8.0/en/sha256-pluggable-authentication.html + # Unlike caching_sha2_password, the sha256_password plugin does not treat + # shared-memory connections as secure, even though share-memory transport + # is secure by default. + if "unix_socket" in mysql_server['conn_params']: + pytest.skip("sha256_password is not supported on unix sockets") + connection_data = copy.copy(mysql_server['conn_params']) connection_data['user'] = 'user_sha256' connection_data['password'] = 'pass_sha256' diff --git a/tests/test_ssl.py b/tests/test_ssl.py index ff1ea740..140c164f 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -4,7 +4,10 @@ @pytest.mark.run_loop -async def test_tls_connect(mysql_server, loop): +async def test_tls_connect(mysql_server, loop, mysql_params): + if "unix_socket" in mysql_params: + pytest.skip("TLS is not supported on unix sockets") + async with create_pool(**mysql_server['conn_params'], loop=loop) as pool: async with pool.get() as conn: @@ -32,7 +35,10 @@ async def test_tls_connect(mysql_server, loop): # MySQL will get you to renegotiate if sent a cleartext password @pytest.mark.run_loop -async def test_auth_plugin_renegotiation(mysql_server, loop): +async def test_auth_plugin_renegotiation(mysql_server, loop, mysql_params): + if "unix_socket" in mysql_params: + pytest.skip("TLS is not supported on unix sockets") + async with create_pool(**mysql_server['conn_params'], auth_plugin='mysql_clear_password', loop=loop) as pool: From 95f9db1bf137d2d9a7011f63d4ffcde1aec9a2df Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 28 Jan 2022 21:43:35 +0100 Subject: [PATCH 31/92] don't explicitly pass loop to asyncio methods in tests, fixes python 3.10 test compatibility (#697) --- .github/workflows/ci.yml | 4 ++-- tests/test_pool.py | 24 +++++++++++------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8db8d57..80fc3f96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,8 @@ jobs: - '3.7' - '3.8' - '3.9' - # - '3.10' - # - '3.11.0-alpha.3' + - '3.10' + # - '3.11.0-alpha.4' db: - [mysql, '5.7'] - [mysql, '8.0'] diff --git a/tests/test_pool.py b/tests/test_pool.py index ff1acf5e..fac5c2d2 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -135,7 +135,7 @@ async def test_parallel_tasks(pool_creator, loop): fut1 = pool.acquire() fut2 = pool.acquire() - conn1, conn2 = await asyncio.gather(fut1, fut2, loop=loop) + conn1, conn2 = await asyncio.gather(fut1, fut2) assert 2 == pool.size assert 0 == pool.freesize assert {conn1, conn2} == pool._used @@ -164,8 +164,7 @@ async def test_parallel_tasks_more(pool_creator, loop): fut2 = pool.acquire() fut3 = pool.acquire() - conn1, conn2, conn3 = await asyncio.gather(fut1, fut2, fut3, - loop=loop) + conn1, conn2, conn3 = await asyncio.gather(fut1, fut2, fut3) assert 3 == pool.size assert 0 == pool.freesize assert {conn1, conn2, conn3} == pool._used @@ -242,7 +241,7 @@ async def test__fill_free(pool_creator, loop): assert 1 == pool.size conn = await asyncio.wait_for(pool.acquire(), - timeout=0.5, loop=loop) + timeout=0.5) assert 0 == pool.freesize assert 2 == pool.size pool.release(conn) @@ -302,12 +301,12 @@ async def inner(): conn = await pool.acquire() maxsize = max(maxsize, pool.size) minfreesize = min(minfreesize, pool.freesize) - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) pool.release(conn) maxsize = max(maxsize, pool.size) minfreesize = min(minfreesize, pool.freesize) - await asyncio.gather(inner(), inner(), loop=loop) + await asyncio.gather(inner(), inner()) assert 1 == maxsize assert 0 == minfreesize @@ -334,7 +333,7 @@ async def test_wait_closed(pool_creator, loop): ops = [] async def do_release(conn): - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) pool.release(conn) ops.append('release') @@ -343,8 +342,7 @@ async def wait_closed(): ops.append('wait_closed') pool.close() - await asyncio.gather(wait_closed(), do_release(c1), do_release(c2), - loop=loop) + await asyncio.gather(wait_closed(), do_release(c1), do_release(c2)) assert ['release', 'release', 'wait_closed'] == ops assert 0 == pool.freesize @@ -415,7 +413,7 @@ async def test_close_with_acquired_connections(pool_creator, loop): pool.close() with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(pool.wait_closed(), 0.1, loop=loop) + await asyncio.wait_for(pool.wait_closed(), 0.1) pool.release(conn) @@ -440,7 +438,7 @@ async def test_drop_connection_if_timedout(pool_creator, try: pool = await pool_creator(minsize=3, maxsize=3) # sleep, more then connection timeout - await asyncio.sleep(3, loop=loop) + await asyncio.sleep(3) conn = await pool.acquire() cur = await conn.cursor() # query should not throw exception OperationalError @@ -490,7 +488,7 @@ async def test_cancelled_connection(pool_creator, loop): # timings) task = loop.create_task(curs.execute( "SELECT 1 as id, SLEEP(0.1) as xxx")) - await asyncio.sleep(0.05, loop=loop) + await asyncio.sleep(0.05) task.cancel() await task except asyncio.CancelledError: @@ -516,7 +514,7 @@ async def test_pool_with_connection_recycling(pool_creator, loop): val = await cur.fetchone() assert (1,) == val - await asyncio.sleep(5, loop=loop) + await asyncio.sleep(5) assert 1 == pool.freesize async with pool.get() as conn: From cc1e4bdfb72ddc3a69e6631acafb4932e88ec532 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 28 Jan 2022 22:50:24 +0100 Subject: [PATCH 32/92] Introduce issue forms (#698) Co-authored-by: Anes Abismail --- .github/ISSUE_TEMPLATE/feature_request.yml | 61 ++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000..992c1b64 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,61 @@ +--- +name: ๐Ÿš€ Feature request +description: Suggest an idea for this project. +labels: enhancement +body: +- type: markdown + attributes: + value: | + **Thanks for taking a minute to file a feature for aiomysql!** + + โš  + Verify first that your feature request is not [already reported on + GitHub][issue search]. + + _Please fill out the form below with as many precise + details as possible._ + + [issue search]: ../search?q=is%3Aissue&type=issues + +- type: textarea + attributes: + label: Is your feature request related to a problem? + description: >- + Please add a clear and concise description of what + the problem is. _Ex. I'm always frustrated when [...]_ + +- type: textarea + attributes: + label: Describe the solution you'd like + description: >- + A clear and concise description of what you want to happen. + validations: + required: true + +- type: textarea + attributes: + label: Describe alternatives you've considered + description: >- + A clear and concise description of any alternative solutions + or features you've considered. + validations: + required: true + +- type: textarea + attributes: + label: Additional context + description: >- + Add any other context or screenshots about + the feature request here. + +- type: checkboxes + attributes: + label: Code of Conduct + description: | + Read the [aio-libs Code of Conduct][CoC] first. + + [CoC]: https://github.com/aio-libs/.github/blob/master/CODE_OF_CONDUCT.md + options: + - label: I agree to follow the aio-libs Code of Conduct + required: true +... From 5a9e24064a8e32a7277ae1cb04b4b15999dcbcff Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 28 Jan 2022 23:15:25 +0100 Subject: [PATCH 33/92] now that we have python 3.10 tests we can also advertise it (#699) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6e9c0bf2..7d436d85 100755 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ def read_version(): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Operating System :: POSIX', 'Environment :: Web Environment', 'Development Status :: 3 - Alpha', From 018af447ab15e4167ce49d7031f80af678bae210 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sat, 29 Jan 2022 00:02:59 +0100 Subject: [PATCH 34/92] add xfailing test case for #635 (#700) --- tests/test_sscursor.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_sscursor.py b/tests/test_sscursor.py index c020fe81..203d7d09 100644 --- a/tests/test_sscursor.py +++ b/tests/test_sscursor.py @@ -1,7 +1,7 @@ import asyncio import pytest -from pymysql import NotSupportedError +from pymysql import NotSupportedError, InternalError from aiomysql import ProgrammingError, InterfaceError from aiomysql.cursors import SSCursor @@ -188,3 +188,19 @@ async def read_cursor(): with pytest.raises(InterfaceError): await conn.cursor(SSCursor) + + +@pytest.mark.xfail( + reason="https://github.com/aio-libs/aiomysql/issues/635", + raises=InternalError, + strict=True, +) +@pytest.mark.run_loop +async def test_sscursor_discarded_result(connection): + conn = connection + await _prepare(conn) + async with conn.cursor(SSCursor) as cursor: + await cursor.execute("select 1") + await cursor.execute("select 2") + ret = await cursor.fetchone() + assert (1,) == ret From b47bef691980c22e6077d6ff53587ca99c7c251b Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sat, 29 Jan 2022 00:41:33 +0100 Subject: [PATCH 35/92] ensure coroutine is awaited when discarding results in SSCursor, fixes #635 (#701) also fix typo in associated test case --- CHANGES.txt | 1 + aiomysql/connection.py | 2 +- tests/test_sscursor.py | 9 ++------- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 9a127dd6..ec047173 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,6 +12,7 @@ To be included in 1.0.0 (unreleased) * Ensure connections are properly closed before raising an InternalError when packet sequence numbers are out of sync #660 * Unix sockets are now internally considered secure, allowing sha256_password and caching_sha2_password auth methods to be used #695 * Test suite now also tests unix socket connections #696 +* Fix SSCursor raising InternalError when last result was not fully retrieved #635 0.0.22 (2021-11-14) diff --git a/aiomysql/connection.py b/aiomysql/connection.py index 1bd279d3..af2f5495 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -711,7 +711,7 @@ async def _execute_command(self, command, sql): if self._result is not None: if self._result.unbuffered_active: warnings.warn("Previous unbuffered result was left incomplete") - self._result._finish_unbuffered_query() + await self._result._finish_unbuffered_query() while self._result.has_next: await self.next_result() self._result = None diff --git a/tests/test_sscursor.py b/tests/test_sscursor.py index 203d7d09..2f3a7272 100644 --- a/tests/test_sscursor.py +++ b/tests/test_sscursor.py @@ -1,7 +1,7 @@ import asyncio import pytest -from pymysql import NotSupportedError, InternalError +from pymysql import NotSupportedError from aiomysql import ProgrammingError, InterfaceError from aiomysql.cursors import SSCursor @@ -190,11 +190,6 @@ async def read_cursor(): await conn.cursor(SSCursor) -@pytest.mark.xfail( - reason="https://github.com/aio-libs/aiomysql/issues/635", - raises=InternalError, - strict=True, -) @pytest.mark.run_loop async def test_sscursor_discarded_result(connection): conn = connection @@ -203,4 +198,4 @@ async def test_sscursor_discarded_result(connection): await cursor.execute("select 1") await cursor.execute("select 2") ret = await cursor.fetchone() - assert (1,) == ret + assert (2,) == ret From 1558b4a11a0213f8b44f169a41c6c9a54a18a87a Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sat, 29 Jan 2022 03:30:56 +0100 Subject: [PATCH 36/92] remove deprecated no_delay argument (#702) --- CHANGES.txt | 1 + aiomysql/connection.py | 20 ++++---------------- docs/connection.rst | 3 +-- tests/base.py | 8 ++++---- tests/test_connection.py | 14 -------------- 5 files changed, 10 insertions(+), 36 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ec047173..8f63c7e8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -13,6 +13,7 @@ To be included in 1.0.0 (unreleased) * Unix sockets are now internally considered secure, allowing sha256_password and caching_sha2_password auth methods to be used #695 * Test suite now also tests unix socket connections #696 * Fix SSCursor raising InternalError when last result was not fully retrieved #635 +* Remove deprecated no_delay argument #702 0.0.22 (2021-11-14) diff --git a/aiomysql/connection.py b/aiomysql/connection.py index af2f5495..16a2d480 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -55,7 +55,7 @@ def connect(host="localhost", user=None, password="", read_default_file=None, conv=decoders, use_unicode=None, client_flag=0, cursorclass=Cursor, init_command=None, connect_timeout=None, read_default_group=None, - no_delay=None, autocommit=False, echo=False, + autocommit=False, echo=False, local_infile=False, loop=None, ssl=None, auth_plugin='', program_name='', server_public_key=None): """See connections.Connection.__init__() for information about @@ -68,7 +68,7 @@ def connect(host="localhost", user=None, password="", init_command=init_command, connect_timeout=connect_timeout, read_default_group=read_default_group, - no_delay=no_delay, autocommit=autocommit, echo=echo, + autocommit=autocommit, echo=echo, local_infile=local_infile, loop=loop, ssl=ssl, auth_plugin=auth_plugin, program_name=program_name) return _ConnectionContextManager(coro) @@ -144,7 +144,7 @@ def __init__(self, host="localhost", user=None, password="", read_default_file=None, conv=decoders, use_unicode=None, client_flag=0, cursorclass=Cursor, init_command=None, connect_timeout=None, read_default_group=None, - no_delay=None, autocommit=False, echo=False, + autocommit=False, echo=False, local_infile=False, loop=None, ssl=None, auth_plugin='', program_name='', server_public_key=None): """ @@ -175,7 +175,6 @@ def __init__(self, host="localhost", user=None, password="", when connecting. :param read_default_group: Group to read from in the configuration file. - :param no_delay: Disable Nagle's algorithm on the socket :param autocommit: Autocommit mode. None means use server default. (default: False) :param local_infile: boolean to enable the use of LOAD DATA LOCAL @@ -211,19 +210,11 @@ def __init__(self, host="localhost", user=None, password="", port = int(_config("port", fallback=port)) charset = _config("default-character-set", fallback=charset) - # pymysql port - if no_delay is not None: - warnings.warn("no_delay option is deprecated", DeprecationWarning) - no_delay = bool(no_delay) - else: - no_delay = True - self._host = host self._port = port self._user = user or DEFAULT_USER self._password = password or "" self._db = db - self._no_delay = no_delay self._echo = echo self._last_usage = self._loop.time() self._client_auth_plugin = auth_plugin @@ -544,11 +535,8 @@ async def _connect(self): self._port), timeout=self.connect_timeout) self._set_keep_alive() - self.host_info = "socket %s:%d" % (self._host, self._port) - - # do not set no delay in case of unix_socket - if self._no_delay and not self._unix_socket: self._set_nodelay(True) + self.host_info = "socket %s:%d" % (self._host, self._port) self._next_seq_id = 0 diff --git a/docs/connection.rst b/docs/connection.rst index ef9482ac..49936e01 100644 --- a/docs/connection.rst +++ b/docs/connection.rst @@ -45,7 +45,7 @@ Example:: read_default_file=None, conv=decoders, use_unicode=None, client_flag=0, cursorclass=Cursor, init_command=None, connect_timeout=None, read_default_group=None, - no_delay=False, autocommit=False, echo=False, + autocommit=False, echo=False ssl=None, auth_plugin='', program_name='', server_public_key=None, loop=None) @@ -79,7 +79,6 @@ Example:: when connecting. :param str read_default_group: Group to read from in the configuration file. - :param bool no_delay: disable Nagle's algorithm on the socket :param autocommit: Autocommit mode. None means use server default. (default: ``False``) :param ssl: Optional SSL Context to force SSL diff --git a/tests/base.py b/tests/base.py index 193667c6..be87f535 100644 --- a/tests/base.py +++ b/tests/base.py @@ -44,7 +44,7 @@ def tearDown(self): super(AIOPyMySQLTestCase, self).tearDown() async def connect(self, host=None, user=None, password=None, - db=None, use_unicode=True, no_delay=None, port=None, + db=None, use_unicode=True, port=None, **kwargs): if host is None: host = self.host @@ -59,13 +59,13 @@ async def connect(self, host=None, user=None, password=None, conn = await aiomysql.connect(loop=self.loop, host=host, user=user, password=password, db=db, use_unicode=use_unicode, - no_delay=no_delay, port=port, + port=port, **kwargs) self.addCleanup(conn.close) return conn async def create_pool(self, host=None, user=None, password=None, - db=None, use_unicode=True, no_delay=None, + db=None, use_unicode=True, port=None, **kwargs): if host is None: host = self.host @@ -80,7 +80,7 @@ async def create_pool(self, host=None, user=None, password=None, pool = await aiomysql.create_pool(loop=self.loop, host=host, user=user, password=password, db=db, use_unicode=use_unicode, - no_delay=no_delay, port=port, + port=port, **kwargs) self.addCleanup(pool.close) return pool diff --git a/tests/test_connection.py b/tests/test_connection.py index af6788f3..a0e4e00e 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -242,20 +242,6 @@ async def test___del__(connection_creator): gc.collect() -@pytest.mark.run_loop -async def test_no_delay_warning(connection_creator): - with pytest.warns(DeprecationWarning): - conn = await connection_creator(no_delay=True) - conn.close() - - -@pytest.mark.run_loop -async def test_no_delay_default_arg(connection_creator): - conn = await connection_creator() - assert conn._no_delay is True - conn.close() - - @pytest.mark.run_loop async def test_previous_cursor_not_closed(connection_creator): conn = await connection_creator() From 624de5f16588d7a061695b4ac642ceb50d6bc80b Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 30 Jan 2022 01:36:12 +0100 Subject: [PATCH 37/92] add flake8 config, set max-line-length to 88 (#704) this also provides compatibility with upcoming black formatting --- .flake8 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..2bcd70e3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 88 From 2b790fa4e998b5c26813d04f40411eedbedc8ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Am=C3=A9rico?= <47510020+ghostebony@users.noreply.github.com> Date: Sat, 29 Jan 2022 22:04:27 -0300 Subject: [PATCH 38/92] add support for pymysql 1.0.0+ (#643) * add support to pymysql 1.0.2 * vendor pymysql byte/int utils * Update setup.py * Update requirements-dev.txt * tests fix --- aiomysql/connection.py | 14 +++++--------- aiomysql/utils.py | 26 ++++++++++++++++++++++++++ requirements-dev.txt | 2 +- setup.py | 2 +- tests/test_basic.py | 3 +-- 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/aiomysql/connection.py b/aiomysql/connection.py index 16a2d480..9db34d34 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -17,7 +17,6 @@ from pymysql.constants import COMMAND from pymysql.constants import CR from pymysql.constants import FIELD_TYPE -from pymysql.util import byte2int, int2byte from pymysql.converters import (escape_item, encoders, decoders, escape_string, escape_bytes_prefixed, through) from pymysql.err import (Warning, Error, @@ -29,18 +28,15 @@ from pymysql.connections import TEXT_TYPES, MAX_PACKET_LEN, DEFAULT_CHARSET from pymysql.connections import _auth -from pymysql.connections import pack_int24 - from pymysql.connections import MysqlPacket from pymysql.connections import FieldDescriptorPacket from pymysql.connections import EOFPacketWrapper from pymysql.connections import OKPacketWrapper from pymysql.connections import LoadLocalPacketWrapper -from pymysql.connections import lenenc_int # from aiomysql.utils import _convert_to_str from .cursors import Cursor -from .utils import _ConnectionContextManager, _ContextManager +from .utils import _pack_int24, _lenenc_int, _ConnectionContextManager, _ContextManager from .log import logger try: @@ -349,7 +345,7 @@ async def ensure_closed(self): if self._writer is None: # connection has been closed return - send_data = struct.pack('=0.9,<=0.9.3 +PyMySQL>=0.9,<=1.0.2 sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 sqlalchemy>1.2.12,<=1.3.16 diff --git a/setup.py b/setup.py index 7d436d85..c057e2f4 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages -install_requires = ['PyMySQL>=0.9,<=0.9.3'] +install_requires = ['PyMySQL>=0.9,<=1.0.2'] PY_VER = sys.version_info diff --git a/tests/test_basic.py b/tests/test_basic.py index 9fe71675..19a5abf2 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -4,7 +4,6 @@ import time import pytest -from pymysql import util from pymysql.err import ProgrammingError @@ -42,7 +41,7 @@ async def test_datatypes(connection, cursor, datatype_table): await cursor.execute( "select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") r = await cursor.fetchone() - assert util.int2byte(1) == r[0] + assert bytes([1]) == r[0] # assert v[1:8] == r[1:8]) assert v[1:9] == r[1:9] # mysql throws away microseconds so we need to check datetimes From a030387e631ef52bb732b6b4ec6e13036facfd71 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 30 Jan 2022 02:24:02 +0100 Subject: [PATCH 39/92] add note about PyMySQL 1.0 support in changelog (#705) --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index 8f63c7e8..2921f837 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -14,6 +14,7 @@ To be included in 1.0.0 (unreleased) * Test suite now also tests unix socket connections #696 * Fix SSCursor raising InternalError when last result was not fully retrieved #635 * Remove deprecated no_delay argument #702 +* Support PyMySQL up to version 1.0.2 #643 0.0.22 (2021-11-14) From 01a7c7970f5cb4ae2fe585dc5d5357819f0621cd Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 30 Jan 2022 03:22:40 +0100 Subject: [PATCH 40/92] add missing decorators to async sqlalchemy test methods (#708) previously these methods were skipped in tests due to lack of native pytest async support --- tests/sa/test_sa_default.py | 1 + tests/sa/test_sa_transaction.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/sa/test_sa_default.py b/tests/sa/test_sa_default.py index e5f270ec..2f0c6eb9 100644 --- a/tests/sa/test_sa_default.py +++ b/tests/sa/test_sa_default.py @@ -88,6 +88,7 @@ async def test_default_fields_isnull(make_engine): assert row.created_at == created_at +@pytest.mark.run_loop async def test_default_fields_edit(make_engine): engine = await make_engine() await start(engine) diff --git a/tests/sa/test_sa_transaction.py b/tests/sa/test_sa_transaction.py index 31feaf30..43d9f691 100644 --- a/tests/sa/test_sa_transaction.py +++ b/tests/sa/test_sa_transaction.py @@ -55,6 +55,7 @@ def release(*args): return _connect +@pytest.mark.run_loop async def test_without_transactions(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -71,6 +72,7 @@ async def test_without_transactions(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_connection_attr(sa_connect): conn = await sa_connect() await start(conn) @@ -79,6 +81,7 @@ async def test_connection_attr(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_root_transaction(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -101,6 +104,7 @@ async def test_root_transaction(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_root_transaction_rollback(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -122,6 +126,7 @@ async def test_root_transaction_rollback(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_root_transaction_close(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -143,6 +148,7 @@ async def test_root_transaction_close(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_rollback_on_connection_close(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -163,6 +169,7 @@ async def test_rollback_on_connection_close(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_root_transaction_commit_inactive(sa_connect): conn = await sa_connect() await start(conn) @@ -175,6 +182,7 @@ async def test_root_transaction_commit_inactive(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_root_transaction_rollback_inactive(sa_connect): conn = await sa_connect() await start(conn) @@ -187,6 +195,7 @@ async def test_root_transaction_rollback_inactive(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_root_transaction_double_close(sa_connect): conn = await sa_connect() await start(conn) @@ -199,6 +208,7 @@ async def test_root_transaction_double_close(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_inner_transaction_commit(sa_connect): conn = await sa_connect() await start(conn) @@ -216,6 +226,7 @@ async def test_inner_transaction_commit(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_inner_transaction_rollback(sa_connect): conn = await sa_connect() await start(conn) @@ -233,6 +244,7 @@ async def test_inner_transaction_rollback(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_inner_transaction_close(sa_connect): conn = await sa_connect() await start(conn) @@ -251,6 +263,7 @@ async def test_inner_transaction_close(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_nested_transaction_commit(sa_connect): conn = await sa_connect() await start(conn) @@ -276,6 +289,7 @@ async def test_nested_transaction_commit(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_nested_transaction_commit_twice(sa_connect): conn = await sa_connect() await start(conn) @@ -298,6 +312,7 @@ async def test_nested_transaction_commit_twice(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_nested_transaction_rollback(sa_connect): conn = await sa_connect() await start(conn) @@ -323,6 +338,7 @@ async def test_nested_transaction_rollback(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_nested_transaction_rollback_twice(sa_connect): conn = await sa_connect() await start(conn) @@ -344,6 +360,7 @@ async def test_nested_transaction_rollback_twice(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_twophase_transaction_commit(sa_connect): conn = await sa_connect() await start(conn) @@ -362,6 +379,7 @@ async def test_twophase_transaction_commit(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_twophase_transaction_twice(sa_connect): conn = await sa_connect() await start(conn) @@ -375,6 +393,7 @@ async def test_twophase_transaction_twice(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_transactions_sequence(sa_connect): conn = await sa_connect() await start(conn) From 32cbeac898e06427e1b71132a73d2ffdcc09677d Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 30 Jan 2022 04:06:28 +0100 Subject: [PATCH 41/92] include tests in coverage report (#709) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80fc3f96..663264b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,7 @@ jobs: run: | # timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs timeout --preserve-status --signal=INT --verbose 5m \ - pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql ./tests --mysql-unix-socket "unix-${{ join(matrix.db, '') }}=/tmp/run-${{ join(matrix.db, '-') }}/mysql.sock" --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306" + pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql --cov tests ./tests --mysql-unix-socket "unix-${{ join(matrix.db, '') }}=/tmp/run-${{ join(matrix.db, '-') }}/mysql.sock" --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306" env: PYTHONUNBUFFERED: 1 DB: '${{ matrix.db[0] }}' From 7939aebe393568c3f56270a03eb173b0547fcc72 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 30 Jan 2022 05:30:31 +0100 Subject: [PATCH 42/92] remove unused session_id test fixture (#710) this was introduced in #275 for running the database in a container but is no longer needed --- tests/conftest.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d6b0a923..7580094a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,6 @@ import gc import os import ssl -import uuid import aiomysql import pymysql @@ -247,12 +246,6 @@ def _register_table(table_name): loop.run_until_complete(cursor.execute(sql)) -@pytest.fixture(scope='session') -def session_id(): - """Unique session identifier, random string.""" - return str(uuid.uuid4()) - - @pytest.fixture(autouse=True) def ensure_mysql_version(request, mysql_image, mysql_tag): mysql_version = request.node.get_closest_marker('mysql_version') From 196e325980229753c16b1db3532b0eafe415443c Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 30 Jan 2022 07:08:48 +0100 Subject: [PATCH 43/92] automatically detect db type and version in tests (#711) this removes the need to pass the db type and version from environment variables. it also makes it possible to pass connections to different databases to the same pytest run. note that we can't rely on connection.get_server_info() as this does not yield the actual version on MariaDB. due to the version detection now relying on the parameterized mysql_address fixture it is no longer compatible with tests independent from the connection, especially relevant in test_sa_distil.py. to resolve this the version gate was moved directly into test_sha_connection.py. --- .github/workflows/ci.yml | 2 -- tests/conftest.py | 53 ++++++++++++++++++------------------ tests/test_sha_connection.py | 20 +++++++++++--- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 663264b7..0e3f2c76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,8 +137,6 @@ jobs: pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql --cov tests ./tests --mysql-unix-socket "unix-${{ join(matrix.db, '') }}=/tmp/run-${{ join(matrix.db, '-') }}/mysql.sock" --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306" env: PYTHONUNBUFFERED: 1 - DB: '${{ matrix.db[0] }}' - DBTAG: '${{ matrix.db[1] }}' timeout-minutes: 6 - name: Upload coverage diff --git a/tests/conftest.py b/tests/conftest.py index 7580094a..9d0114e3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ import asyncio import gc import os +import re import ssl import aiomysql @@ -73,17 +74,6 @@ def pytest_generate_tests(metafunc): ) -# This is here unless someone fixes the generate_tests bit -@pytest.fixture(scope='session') -def mysql_tag(): - return os.environ.get('DBTAG', '10.5') - - -@pytest.fixture(scope='session') -def mysql_image(): - return os.environ.get('DB', 'mariadb') - - @pytest.fixture def loop(request, loop_type): loop = asyncio.new_event_loop() @@ -246,20 +236,8 @@ def _register_table(table_name): loop.run_until_complete(cursor.execute(sql)) -@pytest.fixture(autouse=True) -def ensure_mysql_version(request, mysql_image, mysql_tag): - mysql_version = request.node.get_closest_marker('mysql_version') - - if mysql_version and ( - mysql_version.args[0] != mysql_image - or mysql_version.args[1] != mysql_tag): - - pytest.skip('Not applicable for {0} version: {1}' - .format(mysql_image, mysql_tag)) - - @pytest.fixture(scope='session') -def mysql_server(mysql_image, mysql_tag, mysql_address): +def mysql_server(mysql_address): unix_socket = type(mysql_address) is str if not unix_socket: @@ -292,6 +270,23 @@ def mysql_server(mysql_image, mysql_tag, mysql_address): **server_params) with connection.cursor() as cursor: + cursor.execute("SELECT VERSION() AS version") + server_version = cursor.fetchone()["version"] + server_version_tuple = tuple( + (int(dig) if dig is not None else 0) + for dig in + re.match(r"^(\d+)\.(\d+)(?:\.(\d+))?", server_version).group(1, 2, 3) + ) + server_version_tuple_short = (server_version_tuple[0], + server_version_tuple[1]) + if server_version_tuple_short in [(5, 7), (8, 0)]: + db_type = "mysql" + elif server_version_tuple[0] == 10: + db_type = "mariadb" + else: + pytest.fail("Unable to determine database type from {!r}" + .format(server_version_tuple)) + if not unix_socket: cursor.execute("SHOW VARIABLES LIKE '%ssl%';") @@ -322,7 +317,7 @@ def mysql_server(mysql_image, mysql_tag, mysql_address): 'DEFAULT COLLATE utf8_general_ci;') # Do MySQL8+ Specific Setup - if mysql_image == "mysql" and mysql_tag in ('8.0',): + if db_type == "mysql" and server_version_tuple_short == (8, 0): # Drop existing users cursor.execute('DROP USER IF EXISTS user_sha256;') cursor.execute('DROP USER IF EXISTS nopass_sha256;') @@ -347,4 +342,10 @@ def mysql_server(mysql_image, mysql_tag, mysql_address): except Exception: pytest.fail("Cannot initialize MySQL environment") - return {'conn_params': server_params} + return { + "conn_params": server_params, + "server_version": server_version, + "server_version_tuple": server_version_tuple, + "server_version_tuple_short": server_version_tuple_short, + "db_type": db_type, + } diff --git a/tests/test_sha_connection.py b/tests/test_sha_connection.py index 0789d162..c2b81fcc 100644 --- a/tests/test_sha_connection.py +++ b/tests/test_sha_connection.py @@ -21,9 +21,18 @@ # ]) -@pytest.mark.mysql_version('mysql', '8.0') +def ensure_mysql_version(mysql_server): + if mysql_server["db_type"] != "mysql" \ + or mysql_server["server_version_tuple_short"] != (8, 0): + pytest.skip("Not applicable for {0} version: {1}" + .format(mysql_server["db_type"], + mysql_server["server_version_tuple_short"])) + + @pytest.mark.run_loop async def test_sha256_nopw(mysql_server, loop): + ensure_mysql_version(mysql_server) + connection_data = copy.copy(mysql_server['conn_params']) connection_data['user'] = 'nopass_sha256' connection_data['password'] = None @@ -36,9 +45,10 @@ async def test_sha256_nopw(mysql_server, loop): assert conn._auth_plugin_used == 'sha256_password' -@pytest.mark.mysql_version('mysql', '8.0') @pytest.mark.run_loop async def test_sha256_pw(mysql_server, loop): + ensure_mysql_version(mysql_server) + # https://dev.mysql.com/doc/refman/8.0/en/sha256-pluggable-authentication.html # Unlike caching_sha2_password, the sha256_password plugin does not treat # shared-memory connections as secure, even though share-memory transport @@ -58,9 +68,10 @@ async def test_sha256_pw(mysql_server, loop): assert conn._auth_plugin_used == 'sha256_password' -@pytest.mark.mysql_version('mysql', '8.0') @pytest.mark.run_loop async def test_cached_sha256_nopw(mysql_server, loop): + ensure_mysql_version(mysql_server) + connection_data = copy.copy(mysql_server['conn_params']) connection_data['user'] = 'nopass_caching_sha2' connection_data['password'] = None @@ -73,9 +84,10 @@ async def test_cached_sha256_nopw(mysql_server, loop): assert conn._auth_plugin_used == 'caching_sha2_password' -@pytest.mark.mysql_version('mysql', '8.0') @pytest.mark.run_loop async def test_cached_sha256_pw(mysql_server, loop): + ensure_mysql_version(mysql_server) + connection_data = copy.copy(mysql_server['conn_params']) connection_data['user'] = 'user_caching_sha2' connection_data['password'] = 'pass_caching_sha2' From c93fad378a5d9c823218b0b7fb271d65d6b2835b Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 30 Jan 2022 09:07:50 +0100 Subject: [PATCH 44/92] bump minimal PyMySQL version to 1.0.0 (#713) while we do not yet strictly require it this simplifies maintenance. e.g. the xfailing test from #712 gets a different error type starting with PyMySQL 0.10.0. it will also allow us to adopt the backwards incompatible changes from PyMySQL 0.10 and 1.0. as our test suite only runs with the latest supported PyMySQL version we currently cannot reliably test older versions either. --- CHANGES.txt | 1 + requirements-dev.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 2921f837..1720dee9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -15,6 +15,7 @@ To be included in 1.0.0 (unreleased) * Fix SSCursor raising InternalError when last result was not fully retrieved #635 * Remove deprecated no_delay argument #702 * Support PyMySQL up to version 1.0.2 #643 +* Bump minimal PyMySQL version to 1.0.0 #713 0.0.22 (2021-11-14) diff --git a/requirements-dev.txt b/requirements-dev.txt index 389e4211..82305e95 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ ipdb==0.13.9 pytest==6.2.5 pytest-cov==3.0.0 pytest-sugar==0.9.4 -PyMySQL>=0.9,<=1.0.2 +PyMySQL>=1.0.0,<=1.0.2 sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 sqlalchemy>1.2.12,<=1.3.16 diff --git a/setup.py b/setup.py index c057e2f4..0cdacf26 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages -install_requires = ['PyMySQL>=0.9,<=1.0.2'] +install_requires = ['PyMySQL>=1.0.0,<=1.0.2'] PY_VER = sys.version_info From 2aef3c79fed965444dd0566e2ff2287975e2f2d1 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 30 Jan 2022 09:18:50 +0100 Subject: [PATCH 45/92] add xfailing test case for #545 (#712) this is ported from https://github.com/PyMySQL/PyMySQL/pull/450, also adds a test case for Cursor.execute() --- tests/test_cursor.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 23b87a98..91e0d59d 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -2,7 +2,10 @@ import pytest +from pymysql.err import OperationalError + from aiomysql import ProgrammingError, Cursor, InterfaceError +from aiomysql.cursors import RE_INSERT_VALUES async def _prepare(conn): @@ -318,3 +321,43 @@ async def test_execute_cancel(connection_creator): with pytest.raises(InterfaceError): await conn.cursor() + + +@pytest.mark.run_loop +async def test_execute_percentage(connection_creator): + # %% in column set + conn = await connection_creator() + async with conn.cursor() as cur: + await cur.execute("DROP TABLE IF EXISTS percent_test") + await cur.execute("""\ + CREATE TABLE percent_test ( + `A%` INTEGER, + `B%` INTEGER)""") + + q = "INSERT INTO percent_test (`A%%`, `B%%`) VALUES (%s, %s)" + + await cur.execute(q, (3, 4)) + + +@pytest.mark.xfail( + reason="https://github.com/aio-libs/aiomysql/pull/545", + raises=OperationalError, + strict=True, +) +@pytest.mark.run_loop +async def test_executemany_percentage(connection_creator): + # %% in column set + conn = await connection_creator() + async with conn.cursor() as cur: + await cur.execute("DROP TABLE IF EXISTS percent_test") + await cur.execute("""\ + CREATE TABLE percent_test ( + `A%` INTEGER, + `B%` INTEGER)""") + + q = "INSERT INTO percent_test (`A%%`, `B%%`) VALUES (%s, %s)" + + assert RE_INSERT_VALUES.match(q) is not None + await cur.executemany(q, [(3, 4), (5, 6)]) + assert cur._last_executed.endswith("(3, 4),(5, 6)"), \ + "executemany with %% not in one query" From c9e349d4bbd8539036146aad34d77647af121449 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 30 Jan 2022 23:55:21 +0100 Subject: [PATCH 46/92] Align % formatting in Cursor.executemany() with Cursor.execute() (#721) ports the changes from https://github.com/PyMySQL/PyMySQL/pull/450 also, fix the test case for detecting whether executemany() just called execute() multiple times --- CHANGES.txt | 1 + aiomysql/cursors.py | 4 ++-- tests/test_cursor.py | 9 +-------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1720dee9..b086f28a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -16,6 +16,7 @@ To be included in 1.0.0 (unreleased) * Remove deprecated no_delay argument #702 * Support PyMySQL up to version 1.0.2 #643 * Bump minimal PyMySQL version to 1.0.0 #713 +* Align % formatting in Cursor.executemany() with Cursor.execute(), literal % now need to be doubled in Cursor.executemany() #714 0.0.22 (2021-11-14) diff --git a/aiomysql/cursors.py b/aiomysql/cursors.py index 97f0431f..3401bdbf 100644 --- a/aiomysql/cursors.py +++ b/aiomysql/cursors.py @@ -14,7 +14,7 @@ # https://github.com/PyMySQL/PyMySQL/blob/master/pymysql/cursors.py#L11-L18 #: Regular expression for :meth:`Cursor.executemany`. -#: executemany only suports simple bulk insert. +#: executemany only supports simple bulk insert. #: You can use it to load large dataset. RE_INSERT_VALUES = re.compile( r"\s*((?:INSERT|REPLACE)\s.+\sVALUES?\s+)" + @@ -274,7 +274,7 @@ async def executemany(self, query, args): m = RE_INSERT_VALUES.match(query) if m: - q_prefix = m.group(1) + q_prefix = m.group(1) % () q_values = m.group(2).rstrip() q_postfix = m.group(3) or '' assert q_values[0] == '(' and q_values[-1] == ')' diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 91e0d59d..db91f25e 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -2,8 +2,6 @@ import pytest -from pymysql.err import OperationalError - from aiomysql import ProgrammingError, Cursor, InterfaceError from aiomysql.cursors import RE_INSERT_VALUES @@ -339,11 +337,6 @@ async def test_execute_percentage(connection_creator): await cur.execute(q, (3, 4)) -@pytest.mark.xfail( - reason="https://github.com/aio-libs/aiomysql/pull/545", - raises=OperationalError, - strict=True, -) @pytest.mark.run_loop async def test_executemany_percentage(connection_creator): # %% in column set @@ -359,5 +352,5 @@ async def test_executemany_percentage(connection_creator): assert RE_INSERT_VALUES.match(q) is not None await cur.executemany(q, [(3, 4), (5, 6)]) - assert cur._last_executed.endswith("(3, 4),(5, 6)"), \ + assert cur._last_executed.endswith(b"(3, 4),(5, 6)"), \ "executemany with %% not in one query" From c0972d246ea5aeb813c746a8417076101d05b119 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Mon, 31 Jan 2022 02:34:05 +0100 Subject: [PATCH 47/92] add python 3.11 tests (#722) uvloop is disabled on python 3.11 for now due to cython incompatibility. a new cython version has since been released but uvloop needs to be built against this new cython version. see https://github.com/MagicStack/uvloop/issues/450 and https://github.com/MagicStack/uvloop/pull/459 --- .github/workflows/ci.yml | 2 +- requirements-dev.txt | 2 +- tests/conftest.py | 12 +++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e3f2c76..e9e149f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - '3.8' - '3.9' - '3.10' - # - '3.11.0-alpha.4' + - '3.11-dev' db: - [mysql, '5.7'] - [mysql, '8.0'] diff --git a/requirements-dev.txt b/requirements-dev.txt index 82305e95..ae330477 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,5 +8,5 @@ PyMySQL>=1.0.0,<=1.0.2 sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 sqlalchemy>1.2.12,<=1.3.16 -uvloop==0.16.0 +uvloop==0.16.0; python_version < '3.11' pyroma==3.2 diff --git a/tests/conftest.py b/tests/conftest.py index 9d0114e3..5420b456 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,11 +3,21 @@ import os import re import ssl +import sys import aiomysql import pymysql import pytest -import uvloop + + +# version gate can be removed once uvloop supports python 3.11 +# https://github.com/MagicStack/uvloop/issues/450 +# https://github.com/MagicStack/uvloop/pull/459 +PY_311 = sys.version_info >= (3, 11) +if PY_311: + uvloop = None +else: + import uvloop @pytest.fixture From 0824b26254dc20cd59af35b27acd85df07d0e1bf Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 1 Feb 2022 01:07:04 +0100 Subject: [PATCH 48/92] Revert "add python 3.11 tests (#722)" (#723) This reverts commit c0972d246ea5aeb813c746a8417076101d05b119. coverage.py still has issues with Python 3.11, there's not much point in keeping them if they segfault half the time. https://github.com/nedbat/coveragepy/issues/1316 --- .github/workflows/ci.yml | 2 +- requirements-dev.txt | 2 +- tests/conftest.py | 12 +----------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9e149f4..0e3f2c76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - '3.8' - '3.9' - '3.10' - - '3.11-dev' + # - '3.11.0-alpha.4' db: - [mysql, '5.7'] - [mysql, '8.0'] diff --git a/requirements-dev.txt b/requirements-dev.txt index ae330477..82305e95 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,5 +8,5 @@ PyMySQL>=1.0.0,<=1.0.2 sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 sqlalchemy>1.2.12,<=1.3.16 -uvloop==0.16.0; python_version < '3.11' +uvloop==0.16.0 pyroma==3.2 diff --git a/tests/conftest.py b/tests/conftest.py index 5420b456..9d0114e3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,21 +3,11 @@ import os import re import ssl -import sys import aiomysql import pymysql import pytest - - -# version gate can be removed once uvloop supports python 3.11 -# https://github.com/MagicStack/uvloop/issues/450 -# https://github.com/MagicStack/uvloop/pull/459 -PY_311 = sys.version_info >= (3, 11) -if PY_311: - uvloop = None -else: - import uvloop +import uvloop @pytest.fixture From e33f42939786784e8fb947b37d6305511c182c4d Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Wed, 2 Feb 2022 22:57:49 +0100 Subject: [PATCH 49/92] Fix pool maxsize unlimited (#426) * fix: allow pool.maxsize to be unlimited * tests: add test for unlimited pool size * add test for pool minsize=1 maxsize=0 * Skip minsize validation for unlimited size pools * Make maxsize property optional * add changelog note Co-authored-by: Richard Schwab --- CHANGES.txt | 1 + aiomysql/pool.py | 6 +++--- tests/test_pool.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b086f28a..8d76df9a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,6 +17,7 @@ To be included in 1.0.0 (unreleased) * Support PyMySQL up to version 1.0.2 #643 * Bump minimal PyMySQL version to 1.0.0 #713 * Align % formatting in Cursor.executemany() with Cursor.execute(), literal % now need to be doubled in Cursor.executemany() #714 +* Fixed unlimited Pool size not working, this is now working as documented by passing maxsize=0 to create_pool #119 0.0.22 (2021-11-14) diff --git a/aiomysql/pool.py b/aiomysql/pool.py index 3eacb47d..47f14bb5 100644 --- a/aiomysql/pool.py +++ b/aiomysql/pool.py @@ -36,13 +36,13 @@ class Pool(asyncio.AbstractServer): def __init__(self, minsize, maxsize, echo, pool_recycle, loop, **kwargs): if minsize < 0: raise ValueError("minsize should be zero or greater") - if maxsize < minsize: + if maxsize < minsize and maxsize != 0: raise ValueError("maxsize should be not less than minsize") self._minsize = minsize self._loop = loop self._conn_kwargs = kwargs self._acquiring = 0 - self._free = collections.deque(maxlen=maxsize) + self._free = collections.deque(maxlen=maxsize or None) self._cond = asyncio.Condition() self._used = set() self._terminated = set() @@ -182,7 +182,7 @@ async def _fill_free_pool(self, override_min): if self._free: return - if override_min and self.size < self.maxsize: + if override_min and (not self.maxsize or self.size < self.maxsize): self._acquiring += 1 try: conn = await connect(echo=self._echo, loop=self._loop, diff --git a/tests/test_pool.py b/tests/test_pool.py index fac5c2d2..c87d0849 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -538,3 +538,21 @@ async def test_pool_drops_connection_with_exception(pool_creator, loop): async with pool.get() as conn: cur = await conn.cursor() await cur.execute('SELECT 1;') + + +@pytest.mark.run_loop +async def test_pool_maxsize_unlimited(pool_creator, loop): + pool = await pool_creator(minsize=0, maxsize=0) + + async with pool.acquire() as conn: + cur = await conn.cursor() + await cur.execute('SELECT 1;') + + +@pytest.mark.run_loop +async def test_pool_maxsize_unlimited_minsize_1(pool_creator, loop): + pool = await pool_creator(minsize=1, maxsize=0) + + async with pool.acquire() as conn: + cur = await conn.cursor() + await cur.execute('SELECT 1;') From 00d0932070275fbf867a1f5590ec73f5347c88b4 Mon Sep 17 00:00:00 2001 From: "pyup.io bot" Date: Wed, 2 Feb 2022 17:10:17 -0500 Subject: [PATCH 50/92] Update coverage to 6.3.1 (#724) * Update coverage from 6.3 to 6.3.1 * re-enable python 3.11 tests uvloop is disabled on python 3.11 for now due to cython incompatibility. a new cython version has since been released but uvloop needs to be built against this new cython version. see https://github.com/MagicStack/uvloop/issues/450 and https://github.com/MagicStack/uvloop/pull/459 Co-authored-by: Richard Schwab --- .github/workflows/ci.yml | 2 +- requirements-dev.txt | 4 ++-- tests/conftest.py | 12 +++++++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e3f2c76..e9e149f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - '3.8' - '3.9' - '3.10' - # - '3.11.0-alpha.4' + - '3.11-dev' db: - [mysql, '5.7'] - [mysql, '8.0'] diff --git a/requirements-dev.txt b/requirements-dev.txt index 82305e95..e24e1127 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -coverage==6.3 +coverage==6.3.1 flake8==4.0.1 ipdb==0.13.9 pytest==6.2.5 @@ -8,5 +8,5 @@ PyMySQL>=1.0.0,<=1.0.2 sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 sqlalchemy>1.2.12,<=1.3.16 -uvloop==0.16.0 +uvloop==0.16.0; python_version < '3.11' pyroma==3.2 diff --git a/tests/conftest.py b/tests/conftest.py index 9d0114e3..5420b456 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,11 +3,21 @@ import os import re import ssl +import sys import aiomysql import pymysql import pytest -import uvloop + + +# version gate can be removed once uvloop supports python 3.11 +# https://github.com/MagicStack/uvloop/issues/450 +# https://github.com/MagicStack/uvloop/pull/459 +PY_311 = sys.version_info >= (3, 11) +if PY_311: + uvloop = None +else: + import uvloop @pytest.fixture From 30b9ea5261ed398107631497dcc76aa6e1fd79b7 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Wed, 2 Feb 2022 23:35:28 +0100 Subject: [PATCH 51/92] add Pool.closed property as present in aiopg, fixes #463 (#726) --- CHANGES.txt | 1 + aiomysql/pool.py | 9 ++++++++- tests/test_pool.py | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 8d76df9a..83f7ec11 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,6 +18,7 @@ To be included in 1.0.0 (unreleased) * Bump minimal PyMySQL version to 1.0.0 #713 * Align % formatting in Cursor.executemany() with Cursor.execute(), literal % now need to be doubled in Cursor.executemany() #714 * Fixed unlimited Pool size not working, this is now working as documented by passing maxsize=0 to create_pool #119 +* Added Pool.closed property as present in aiopg #463 0.0.22 (2021-11-14) diff --git a/aiomysql/pool.py b/aiomysql/pool.py index 47f14bb5..dc1cbf93 100644 --- a/aiomysql/pool.py +++ b/aiomysql/pool.py @@ -1,4 +1,4 @@ -# copied from aiopg +# based on aiopg pool # https://github.com/aio-libs/aiopg/blob/master/aiopg/pool.py import asyncio @@ -79,6 +79,13 @@ async def clear(self): await conn.ensure_closed() self._cond.notify() + @property + def closed(self): + """ + The readonly property that returns ``True`` if connections is closed. + """ + return self._closed + def close(self): """Close pool. diff --git a/tests/test_pool.py b/tests/test_pool.py index c87d0849..0921e936 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -345,6 +345,7 @@ async def wait_closed(): await asyncio.gather(wait_closed(), do_release(c1), do_release(c2)) assert ['release', 'release', 'wait_closed'] == ops assert 0 == pool.freesize + assert pool.closed @pytest.mark.run_loop From ca4c06452b1460acb110ced8cbc0d482171e9d1c Mon Sep 17 00:00:00 2001 From: Mateusz Czubak <43750226+MateuszCzubak@users.noreply.github.com> Date: Thu, 3 Feb 2022 01:03:15 +0000 Subject: [PATCH 52/92] Fix async iterator protocol in _SAConnectionContextManager (#494) * Fix async iterator protocol in _SAConnectionContextManager * Add changelog entry Co-authored-by: Richard Schwab --- CHANGES.txt | 1 + aiomysql/utils.py | 16 +++++++++++++--- tests/sa/test_sa_connection.py | 11 +++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 83f7ec11..73790bbb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -19,6 +19,7 @@ To be included in 1.0.0 (unreleased) * Align % formatting in Cursor.executemany() with Cursor.execute(), literal % now need to be doubled in Cursor.executemany() #714 * Fixed unlimited Pool size not working, this is now working as documented by passing maxsize=0 to create_pool #119 * Added Pool.closed property as present in aiopg #463 +* Fixed SQLAlchemy connection context iterator #410 0.0.22 (2021-11-14) diff --git a/aiomysql/utils.py b/aiomysql/utils.py index e9e75ebe..74ad99a7 100644 --- a/aiomysql/utils.py +++ b/aiomysql/utils.py @@ -96,9 +96,19 @@ async def __aexit__(self, exc_type, exc, tb): class _SAConnectionContextManager(_ContextManager): - async def __aiter__(self): - result = await self._coro - return result + def __aiter__(self): + return self + + async def __anext__(self): + if self._obj is None: + self._obj = await self._coro + + try: + return await self._obj.__anext__() + except StopAsyncIteration: + await self._obj.close() + self._obj = None + raise class _TransactionContextManager(_ContextManager): diff --git a/tests/sa/test_sa_connection.py b/tests/sa/test_sa_connection.py index 40b72212..fae02553 100644 --- a/tests/sa/test_sa_connection.py +++ b/tests/sa/test_sa_connection.py @@ -455,3 +455,14 @@ async def test_create_table(sa_connect): res = await conn.execute("SELECT * FROM sa_tbl") assert 0 == len(await res.fetchall()) + + +@pytest.mark.run_loop +async def test_async_iter(sa_connect): + conn = await sa_connect() + await conn.execute(tbl.insert().values(name="second")) + + ret = [] + async for row in conn.execute(tbl.select()): + ret.append(row) + assert [(1, "first"), (2, "second")] == ret From d6f1f23cb48f807ddc518a0b16b97eba140ef3d4 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Thu, 3 Feb 2022 02:31:52 +0100 Subject: [PATCH 53/92] test against sqlalchemy up to version 1.3.24 (#727) --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e24e1127..40c6f4cc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,6 +7,6 @@ pytest-sugar==0.9.4 PyMySQL>=1.0.0,<=1.0.2 sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 -sqlalchemy>1.2.12,<=1.3.16 +sqlalchemy>1.2.12,<=1.3.24 uvloop==0.16.0; python_version < '3.11' pyroma==3.2 From 6fd00fdce1ae515d1fa69573c95d6eb907facd70 Mon Sep 17 00:00:00 2001 From: Jakub Czaplicki Date: Thu, 3 Feb 2022 03:08:27 +0100 Subject: [PATCH 54/92] [doc] Update the create_engine usage example (#626) * Update the create_engine usage example * update SA example Co-authored-by: Richard Schwab --- docs/sa.rst | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/docs/sa.rst b/docs/sa.rst index 42b059c5..94c6b1dd 100644 --- a/docs/sa.rst +++ b/docs/sa.rst @@ -30,27 +30,36 @@ Example:: metadata = sa.MetaData() - tbl = sa.Table('tbl', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('val', sa.String(255))) + tbl = sa.Table( + "tbl", + metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("val", sa.String(255)), + ) async def go(): - engine = await create_engine(user='root', - db='test_pymysql', - host='127.0.0.1', - password='') + engine = await create_engine( + user="root", + db="test_pymysql", + host="127.0.0.1", + password="", + ) async with engine.acquire() as conn: - await conn.execute(tbl.insert().values(val='abc')) + async with conn.begin() as transaction: + await conn.execute(tbl.insert().values(val="abc")) + await transaction.commit() - res = await conn.execute(tbl.select()) - for row in res: - print(row.id, row.val) + res = await conn.execute(tbl.select()) + async for row in res: + print(row.id, row.val) - await conn.commit() + engine.close() + await engine.wait_closed() - asyncio.get_event_loop().run_until_complete(go()) + + asyncio.run(go()) So you can execute SQL query built by From 82a6195c2747f7d37fb55f4fc40b4da152856ab9 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 6 Feb 2022 07:02:23 +0100 Subject: [PATCH 55/92] Add query timeout tests for #428 (#728) --- tests/test_cursor.py | 79 ++++++++++++++++++++++++++++- tests/test_sscursor.py | 109 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 186 insertions(+), 2 deletions(-) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index db91f25e..12fd41ff 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -2,7 +2,7 @@ import pytest -from aiomysql import ProgrammingError, Cursor, InterfaceError +from aiomysql import ProgrammingError, Cursor, InterfaceError, OperationalError from aiomysql.cursors import RE_INSERT_VALUES @@ -354,3 +354,80 @@ async def test_executemany_percentage(connection_creator): await cur.executemany(q, [(3, 4), (5, 6)]) assert cur._last_executed.endswith(b"(3, 4),(5, 6)"), \ "executemany with %% not in one query" + + +@pytest.mark.run_loop +async def test_max_execution_time(mysql_server, connection_creator): + conn = await connection_creator() + await _prepare(conn) + async with conn.cursor() as cur: + # MySQL MAX_EXECUTION_TIME takes ms + # MariaDB max_statement_time takes seconds as int/float, introduced in 10.1 + + # this will sleep 0.01 seconds per row + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(2000) */ + name, sleep(0.01) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=2 FOR + SELECT name, sleep(0.01) FROM tbl + """ + + await cur.execute(sql) + # unlike SSCursor, Cursor returns a tuple of tuples here + assert (await cur.fetchall()) == ( + ("a", 0), + ("b", 0), + ("c", 0), + ) + + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(2000) */ + name, sleep(0.01) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=2 FOR + SELECT name, sleep(0.01) FROM tbl + """ + await cur.execute(sql) + assert (await cur.fetchone()) == ("a", 0) + + # this discards the previous unfinished query + await cur.execute("SELECT 1") + assert (await cur.fetchone()) == (1,) + + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(1) */ + name, sleep(1) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=0.001 FOR + SELECT name, sleep(1) FROM tbl + """ + with pytest.raises(OperationalError) as cm: + # in a buffered cursor this should reliably raise an + # OperationalError + await cur.execute(sql) + + if mysql_server["db_type"] == "mysql": + # this constant was only introduced in MySQL 5.7, not sure + # what was returned before, may have been ER_QUERY_INTERRUPTED + + # this constant is pending a new PyMySQL release + # assert cm.value.args[0] == pymysql.constants.ER.QUERY_TIMEOUT + assert cm.value.args[0] == 3024 + else: + # this constant is pending a new PyMySQL release + # assert cm.value.args[0] == pymysql.constants.ER.STATEMENT_TIMEOUT + assert cm.value.args[0] == 1969 + + # connection should still be fine at this point + await cur.execute("SELECT 1") + assert (await cur.fetchone()) == (1,) diff --git a/tests/test_sscursor.py b/tests/test_sscursor.py index 2f3a7272..7266402e 100644 --- a/tests/test_sscursor.py +++ b/tests/test_sscursor.py @@ -3,7 +3,7 @@ import pytest from pymysql import NotSupportedError -from aiomysql import ProgrammingError, InterfaceError +from aiomysql import ProgrammingError, InterfaceError, OperationalError from aiomysql.cursors import SSCursor @@ -199,3 +199,110 @@ async def test_sscursor_discarded_result(connection): await cursor.execute("select 2") ret = await cursor.fetchone() assert (2,) == ret + + +@pytest.mark.skip( + reason="see aio-libs/aiomysql#428, " + "this gets stuck until aio-libs/aiomysql#646 is merged", +) +@pytest.mark.run_loop +async def test_max_execution_time(mysql_server, connection): + conn = connection + + async with connection.cursor() as cur: + await cur.execute("DROP TABLE IF EXISTS tbl;") + + await cur.execute( + """ + CREATE TABLE tbl ( + id MEDIUMINT NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL, + PRIMARY KEY (id)); + """ + ) + + for i in [(1, "a"), (2, "b"), (3, "c")]: + await cur.execute("INSERT INTO tbl VALUES(%s, %s)", i) + + await conn.commit() + + async with conn.cursor(SSCursor) as cur: + # MySQL MAX_EXECUTION_TIME takes ms + # MariaDB max_statement_time takes seconds as int/float, introduced in 10.1 + + # this will sleep 0.01 seconds per row + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(2000) */ + name, sleep(0.01) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=2 FOR + SELECT name, sleep(0.01) FROM tbl + """ + + await cur.execute(sql) + # unlike Cursor, SSCursor returns a list of tuples here + + assert (await cur.fetchall()) == [ + ("a", 0), + ("b", 0), + ("c", 0), + ] + + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(2000) */ + name, sleep(0.01) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=2 FOR + SELECT name, sleep(0.01) FROM tbl + """ + await cur.execute(sql) + assert (await cur.fetchone()) == ("a", 0) + + # this discards the previous unfinished query and raises an + # incomplete unbuffered query warning + with pytest.warns(UserWarning): + await cur.execute("SELECT 1") + assert (await cur.fetchone()) == (1,) + + # SSCursor will not read the EOF packet until we try to read + # another row. Skipping this will raise an incomplete unbuffered + # query warning in the next cur.execute(). + assert (await cur.fetchone()) is None + + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(1) */ + name, sleep(1) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=0.001 FOR + SELECT name, sleep(1) FROM tbl + """ + with pytest.raises(OperationalError) as cm: + # in an unbuffered cursor the OperationalError may not show up + # until fetching the entire result + await cur.execute(sql) + await cur.fetchall() + + if mysql_server["db_type"] == "mysql": + # this constant was only introduced in MySQL 5.7, not sure + # what was returned before, may have been ER_QUERY_INTERRUPTED + + # this constant is pending a new PyMySQL release + # assert cm.value.args[0] == pymysql.constants.ER.QUERY_TIMEOUT + assert cm.value.args[0] == 3024 + else: + # this constant is pending a new PyMySQL release + # assert cm.value.args[0] == pymysql.constants.ER.STATEMENT_TIMEOUT + assert cm.value.args[0] == 1969 + + # connection should still be fine at this point + await cur.execute("SELECT 1") + assert (await cur.fetchone()) == (1,) From b2f9d006a6f1f205c02e8007c89c5186891589b9 Mon Sep 17 00:00:00 2001 From: Ki-Seong Park Date: Sun, 6 Feb 2022 15:54:20 +0900 Subject: [PATCH 56/92] Handle query execution timeout errors when in SSCursor error, partially fixes #428 (#646) Co-authored-by: Richard Schwab --- CHANGES.txt | 1 + aiomysql/connection.py | 6 +++++- tests/test_sscursor.py | 4 ---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 73790bbb..a61887d1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -20,6 +20,7 @@ To be included in 1.0.0 (unreleased) * Fixed unlimited Pool size not working, this is now working as documented by passing maxsize=0 to create_pool #119 * Added Pool.closed property as present in aiopg #463 * Fixed SQLAlchemy connection context iterator #410 +* Fix error packet handling for SSCursor #428 0.0.22 (2021-11-14) diff --git a/aiomysql/connection.py b/aiomysql/connection.py index 9db34d34..07530222 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -634,7 +634,11 @@ async def _read_packet(self, packet_type=MysqlPacket): break packet = packet_type(buff, self._encoding) - packet.check_error() + if packet.is_error_packet(): + if self._result is not None and \ + self._result.unbuffered_active is True: + self._result.unbuffered_active = False + packet.raise_for_error() return packet async def _read_bytes(self, num_bytes): diff --git a/tests/test_sscursor.py b/tests/test_sscursor.py index 7266402e..57106b48 100644 --- a/tests/test_sscursor.py +++ b/tests/test_sscursor.py @@ -201,10 +201,6 @@ async def test_sscursor_discarded_result(connection): assert (2,) == ret -@pytest.mark.skip( - reason="see aio-libs/aiomysql#428, " - "this gets stuck until aio-libs/aiomysql#646 is merged", -) @pytest.mark.run_loop async def test_max_execution_time(mysql_server, connection): conn = connection From 901f2b52e43a16fe1dd2164b9e8fa4cd5f8de43e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Feb 2022 23:11:12 +0100 Subject: [PATCH 57/92] Bump pytest from 6.2.5 to 7.0.0 (#729) Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.2.5 to 7.0.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.5...7.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 40c6f4cc..c8a4cbad 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ coverage==6.3.1 flake8==4.0.1 ipdb==0.13.9 -pytest==6.2.5 +pytest==7.0.0 pytest-cov==3.0.0 pytest-sugar==0.9.4 PyMySQL>=1.0.0,<=1.0.2 From a68c26f9a09b60dd641b66bccbfceea7de249b51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 01:00:57 +0000 Subject: [PATCH 58/92] Bump pytest from 7.0.0 to 7.0.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.0.0 to 7.0.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.0.0...7.0.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c8a4cbad..86243b3c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ coverage==6.3.1 flake8==4.0.1 ipdb==0.13.9 -pytest==7.0.0 +pytest==7.0.1 pytest-cov==3.0.0 pytest-sugar==0.9.4 PyMySQL>=1.0.0,<=1.0.2 From ffd346ef16a2e0ae826739e97ad7bb8e7ec66464 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Thu, 17 Feb 2022 16:31:14 +0100 Subject: [PATCH 59/92] Move static setuptools config to setup.cfg (#732) - implement `python_requires` to replace python version check during setup.py execution - relax PyMySQL dependency to only pin minimum required version - pin SQLAlchemy version to below 1.4, as 1.4 is currently not compatible - add mariadb keyword --- CHANGES.txt | 1 + setup.cfg | 41 +++++++++++++++++++++++++++++++++++++++++ setup.py | 49 ++----------------------------------------------- 3 files changed, 44 insertions(+), 47 deletions(-) create mode 100644 setup.cfg diff --git a/CHANGES.txt b/CHANGES.txt index a61887d1..d29e3a6b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -21,6 +21,7 @@ To be included in 1.0.0 (unreleased) * Added Pool.closed property as present in aiopg #463 * Fixed SQLAlchemy connection context iterator #410 * Fix error packet handling for SSCursor #428 +* Required python version is now properly documented in python_requires instead of failing on setup.py execution #731 0.0.22 (2021-11-14) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..c2ce1f28 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,41 @@ +[metadata] +name = aiomysql +url = https://github.com/aio-libs/aiomysql +download_url = https://pypi.python.org/pypi/aiomysql +description = MySQL driver for asyncio. +author = Nikolay Novik +author_email = nickolainovik@gmail.com +classifiers = + License :: OSI Approved :: MIT License + Intended Audience :: Developers + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Operating System :: POSIX + Environment :: Web Environment + Development Status :: 3 - Alpha + Topic :: Database + Topic :: Database :: Front-Ends + Framework :: AsyncIO +license = MIT +keywords = + mysql + mariadb + asyncio + aiomysql +platforms = + POSIX + +[options] +python_requires = >=3.7 +include_package_data = True + +# runtime requirements +install_requires = + PyMySQL>=1.0 + +[options.extras_require] +sa = + sqlalchemy>=1.0,<1.4 diff --git a/setup.py b/setup.py index 0cdacf26..c46a2c8f 100755 --- a/setup.py +++ b/setup.py @@ -1,25 +1,12 @@ import os import re -import sys from setuptools import setup, find_packages -install_requires = ['PyMySQL>=1.0.0,<=1.0.2'] - -PY_VER = sys.version_info - - -if not PY_VER >= (3, 7, 0): - raise RuntimeError("aiomysql doesn't support Python earlier than 3.7.0") - - def read(f): return open(os.path.join(os.path.dirname(__file__), f)).read().strip() -extras_require = {'sa': ['sqlalchemy>=1.0'], } - - def read_version(): regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+)'") init_py = os.path.join(os.path.dirname(__file__), @@ -33,38 +20,6 @@ def read_version(): raise RuntimeError('Cannot find version in aiomysql/__init__.py') -classifiers = [ - 'License :: OSI Approved :: MIT License', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Operating System :: POSIX', - 'Environment :: Web Environment', - 'Development Status :: 3 - Alpha', - 'Topic :: Database', - 'Topic :: Database :: Front-Ends', - 'Framework :: AsyncIO', -] - -keywords = ["mysql", "asyncio", "aiomysql"] - - -setup(name='aiomysql', - version=read_version(), - description=('MySQL driver for asyncio.'), +setup(version=read_version(), long_description='\n\n'.join((read('README.rst'), read('CHANGES.txt'))), - classifiers=classifiers, - platforms=['POSIX'], - author="Nikolay Novik", - author_email="nickolainovik@gmail.com", - url='https://github.com/aio-libs/aiomysql', - download_url='https://pypi.python.org/pypi/aiomysql', - license='MIT', - packages=find_packages(exclude=['tests', 'tests.*']), - install_requires=install_requires, - extras_require=extras_require, - keywords=keywords, - include_package_data=True) + packages=find_packages(exclude=['tests', 'tests.*'])) From bf8b11e1a74e7239a4a0b63f5fd4b8ead8fb0672 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 18 Feb 2022 01:05:29 +0100 Subject: [PATCH 60/92] Add rsa extras_require depending on PyMySQL[rsa] (#733) fixes #557 --- CHANGES.txt | 1 + setup.cfg | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index d29e3a6b..61187929 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -22,6 +22,7 @@ To be included in 1.0.0 (unreleased) * Fixed SQLAlchemy connection context iterator #410 * Fix error packet handling for SSCursor #428 * Required python version is now properly documented in python_requires instead of failing on setup.py execution #731 +* Add rsa extras_require depending on PyMySQL[rsa] #557 0.0.22 (2021-11-14) diff --git a/setup.cfg b/setup.cfg index c2ce1f28..911d8106 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,3 +39,5 @@ install_requires = [options.extras_require] sa = sqlalchemy>=1.0,<1.4 +rsa = + PyMySQL[rsa]>=1.0 From 3270ca596ca15c0dee4c45846964420d8ce6786b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Feb 2022 05:11:11 +0100 Subject: [PATCH 61/92] Bump coverage from 6.3.1 to 6.3.2 (#736) Bumps [coverage](https://github.com/nedbat/coveragepy) from 6.3.1 to 6.3.2. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/6.3.1...6.3.2) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 86243b3c..79bce51f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -coverage==6.3.1 +coverage==6.3.2 flake8==4.0.1 ipdb==0.13.9 pytest==7.0.1 From a13a4709b8661713fb0752a0ff9b977b81c76d95 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Wed, 23 Feb 2022 05:29:29 +0100 Subject: [PATCH 62/92] Remove pyroma from tests (#737) Upcoming implementation of `setuptools-scm` is incompatible with pyroma: https://github.com/regebro/pyroma/issues/69 --- .github/workflows/lint.yml | 4 ---- Makefile | 5 +---- requirements-dev.txt | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0b000eb7..a7266861 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -41,10 +41,6 @@ jobs: run: | python -m pip install --upgrade --requirement requirements-dev.txt - - name: Run pyroma - run: | - python -m pyroma --min 10 --directory . - - name: flake8 Lint uses: py-actions/flake8@v2.0.0 with: diff --git a/Makefile b/Makefile index 542b2c20..3b0716a8 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,8 @@ FLAGS= checkrst: python setup.py check --restructuredtext -pyroma: - pyroma -d . - -flake:checkrst pyroma +flake:checkrst flake8 aiomysql tests examples test: flake diff --git a/requirements-dev.txt b/requirements-dev.txt index 79bce51f..92494548 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,4 +9,3 @@ sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 sqlalchemy>1.2.12,<=1.3.24 uvloop==0.16.0; python_version < '3.11' -pyroma==3.2 From a02da3b8cda717fd093917a0c37e14712cf9633d Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Wed, 23 Feb 2022 06:03:26 +0100 Subject: [PATCH 63/92] Add project URLs to setup.cfg (#739) --- setup.cfg | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.cfg b/setup.cfg index 911d8106..6a527450 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,12 @@ name = aiomysql url = https://github.com/aio-libs/aiomysql download_url = https://pypi.python.org/pypi/aiomysql +project_urls = + CI: GitHub = https://github.com/aio-libs/aiomysql/actions + Docs: RTD = https://aiomysql.readthedocs.io/ + GitHub: repo = https://github.com/aio-libs/aiomysql + GitHub: issues = https://github.com/aio-libs/aiomysql/issues + GitHub: discussions = https://github.com/aio-libs/aiomysql/discussions description = MySQL driver for asyncio. author = Nikolay Novik author_email = nickolainovik@gmail.com From 82075380c74dc820a49f7fd0b76b09e925532525 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Wed, 23 Feb 2022 08:46:15 +0100 Subject: [PATCH 64/92] Pin exact runtime dependency versions for tests (#740) --- requirements-dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 92494548..db029c12 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,8 +4,8 @@ ipdb==0.13.9 pytest==7.0.1 pytest-cov==3.0.0 pytest-sugar==0.9.4 -PyMySQL>=1.0.0,<=1.0.2 +PyMySQL==1.0.2 sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 -sqlalchemy>1.2.12,<=1.3.24 +SQLAlchemy==1.3.24 uvloop==0.16.0; python_version < '3.11' From 1aa7105d86b35cd110dea6046f184966e6d3f7c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Mar 2022 10:00:25 +0100 Subject: [PATCH 65/92] Bump actions/setup-python from 2 to 3 (#742) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 3. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/lint.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9e149f4..ef49b4e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,7 @@ jobs: uses: actions/checkout@v2.4.0 - name: Setup Python ${{ matrix.py }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.py }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a7266861..361473a3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@v2.4.0 - name: Setup Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: '3.10' From 1ca92a581b6a3e0755e8cbe8c032779c613ad72a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Mar 2022 10:21:18 +0100 Subject: [PATCH 66/92] Bump actions/checkout from 2.4.0 to 3 (#743) Bumps [actions/checkout](https://github.com/actions/checkout) from 2.4.0 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.4.0...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/lint.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef49b4e6..cb29608d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2.4.0 + uses: actions/checkout@v3 - name: Setup Python ${{ matrix.py }} uses: actions/setup-python@v3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 361473a3..363d1746 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2.4.0 + uses: actions/checkout@v3 - name: Setup Python 3.10 uses: actions/setup-python@v3 From a3b7e810dd80237ce141f244f5b279fedd3267d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Mar 2022 02:34:57 +0100 Subject: [PATCH 67/92] Bump pytest from 7.0.1 to 7.1.0 (#745) Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.0.1 to 7.1.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.0.1...7.1.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index db029c12..d47d30ed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ coverage==6.3.2 flake8==4.0.1 ipdb==0.13.9 -pytest==7.0.1 +pytest==7.1.0 pytest-cov==3.0.0 pytest-sugar==0.9.4 PyMySQL==1.0.2 From c8a4334c1c5c0ff6e8b405439310d921d59631b0 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Wed, 16 Mar 2022 00:19:07 +0100 Subject: [PATCH 68/92] Add readthedocs yaml config (#747) --- .readthedocs.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..9ac30d0d --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,23 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +build: + os: ubuntu-20.04 + tools: + python: "3.10" + +sphinx: + configuration: docs/conf.py + fail_on_warning: false # FIXME + +formats: +- pdf +- epub + +python: + install: + - requirements: requirements-dev.txt From 42532c80cc854277f6d8f6161ea57a27c3b09f9e Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 25 Mar 2022 18:30:13 +0100 Subject: [PATCH 69/92] Migrate to PEP 517 build system (#746) - all remaining setuptools configuration is moved from setup.py to setup.cfg - description is now checked with twine rather than `setup.py check` Split from #734, including suggestions from @webknjaz Co-authored-by: Sviatoslav Sydorenko --- .github/workflows/ci.yml | 12 ++++++++---- CHANGES.txt | 1 + pyproject.toml | 6 ++++++ setup.cfg | 10 ++++++++++ setup.py | 25 ------------------------- 5 files changed, 25 insertions(+), 29 deletions(-) create mode 100644 pyproject.toml delete mode 100755 setup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb29608d..f1c73562 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,13 +83,17 @@ jobs: run: | python -m pip install --upgrade --requirement requirements-dev.txt - - name: Install aiomysql + - name: Build distribution packages run: | - python -m pip install . + python -m build + + - name: Check package description + run: | + python -m twine check --strict dist/* - - name: Check rst + - name: Install aiomysql run: | - python setup.py check --restructuredtext + python -m pip install . # this ensures our database is ready. typically by the time the preparations have completed its first start logic. # unfortunately we need this hacky workaround as GitHub Actions service containers can't reference data from our repo. diff --git a/CHANGES.txt b/CHANGES.txt index 61187929..6aaec769 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -23,6 +23,7 @@ To be included in 1.0.0 (unreleased) * Fix error packet handling for SSCursor #428 * Required python version is now properly documented in python_requires instead of failing on setup.py execution #731 * Add rsa extras_require depending on PyMySQL[rsa] #557 +* Migrate to PEP 517 build system #746 0.0.22 (2021-11-14) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..05e4daa9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + # Essentials + "setuptools >= 42", +] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg index 6a527450..5b8c7d34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [metadata] name = aiomysql +version = attr: aiomysql.__version__ url = https://github.com/aio-libs/aiomysql download_url = https://pypi.python.org/pypi/aiomysql project_urls = @@ -9,6 +10,8 @@ project_urls = GitHub: issues = https://github.com/aio-libs/aiomysql/issues GitHub: discussions = https://github.com/aio-libs/aiomysql/discussions description = MySQL driver for asyncio. +long_description = file: README.rst, CHANGES.txt +long_description_content_type = text/x-rst author = Nikolay Novik author_email = nickolainovik@gmail.com classifiers = @@ -38,6 +41,8 @@ platforms = python_requires = >=3.7 include_package_data = True +packages = find: + # runtime requirements install_requires = PyMySQL>=1.0 @@ -47,3 +52,8 @@ sa = sqlalchemy>=1.0,<1.4 rsa = PyMySQL[rsa]>=1.0 + +[options.packages.find] +exclude = + tests + tests.* diff --git a/setup.py b/setup.py deleted file mode 100755 index c46a2c8f..00000000 --- a/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import re -from setuptools import setup, find_packages - - -def read(f): - return open(os.path.join(os.path.dirname(__file__), f)).read().strip() - - -def read_version(): - regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+)'") - init_py = os.path.join(os.path.dirname(__file__), - 'aiomysql', '__init__.py') - with open(init_py) as f: - for line in f: - match = regexp.match(line) - if match is not None: - return match.group(1) - else: - raise RuntimeError('Cannot find version in aiomysql/__init__.py') - - -setup(version=read_version(), - long_description='\n\n'.join((read('README.rst'), read('CHANGES.txt'))), - packages=find_packages(exclude=['tests', 'tests.*'])) From 1697e15e523516bc0e9d3778832769020382f3c3 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Thu, 7 Apr 2022 22:09:11 +0200 Subject: [PATCH 70/92] Use setuptools-scm for versioning (#748) Use setuptools-scm for versioning - generate aiomysql/_scm_version.py at build time, fallback to `unknown` if it doesn't exist - include setuptools-scm-git-archive to support versioning in archives from git tags - install aiomysql package in readthedocs and access generated version to embed in docs - fetch git context in workflow to ensure we have all information for setuptools-scm to work - fetch git context in `make doc` Split from #734, including suggestions from @webknjaz Co-authored-by: Sviatoslav Sydorenko --- .git_archival.txt | 1 + .gitattributes | 5 +++++ .github/workflows/ci.yml | 2 ++ .readthedocs.yaml | 6 ++++-- CHANGES.txt | 1 + MANIFEST.in | 5 ----- Makefile | 9 +++++++++ aiomysql/.gitignore | 1 + aiomysql/__init__.py | 3 ++- aiomysql/_scm_version.pyi | 3 +++ aiomysql/_version.py | 4 ++++ docs/conf.py | 25 ++++--------------------- pyproject.toml | 7 +++++++ 13 files changed, 43 insertions(+), 29 deletions(-) create mode 100644 .git_archival.txt create mode 100644 .gitattributes delete mode 100644 MANIFEST.in create mode 100644 aiomysql/.gitignore create mode 100644 aiomysql/_scm_version.pyi create mode 100644 aiomysql/_version.py diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 00000000..95cb3eea --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1 @@ +ref-names: $Format:%D$ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..ec8c3333 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Force LF line endings for text files +* text=auto eol=lf + +# Needed for setuptools-scm-git-archive +.git_archival.txt export-subst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1c73562..ab2a59f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Setup Python ${{ matrix.py }} uses: actions/setup-python@v3 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 9ac30d0d..5fba73ba 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,5 +19,7 @@ formats: - epub python: - install: - - requirements: requirements-dev.txt + install: + - requirements: requirements-dev.txt + - method: pip + path: . diff --git a/CHANGES.txt b/CHANGES.txt index 6aaec769..5c8d7b73 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -24,6 +24,7 @@ To be included in 1.0.0 (unreleased) * Required python version is now properly documented in python_requires instead of failing on setup.py execution #731 * Add rsa extras_require depending on PyMySQL[rsa] #557 * Migrate to PEP 517 build system #746 +* Self-reported `__version__` now returns version generated by `setuptools-scm` during build, otherwise `'unknown'` #748 0.0.22 (2021-11-14) diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 95727af4..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include LICENSE -include CHANGES.txt -include README.rst -graft aiomysql -global-exclude *.pyc *.swp diff --git a/Makefile b/Makefile index 3b0716a8..ae5ceeb0 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,16 @@ start_mysql: stop_mysql: docker-compose -f docker-compose.yml stop mysql +# TODO: this depends on aiomysql being installed, e.g. in a venv. +# TODO: maybe this can be solved better. doc: + @echo "----------------------------------------------------------------" + @echo "Doc builds require installing the aiomysql package in the" + @echo "environment. Make sure you've installed your current dev version" + @echo "into your environment, e.g. using venv, then run this command in" + @echo "the virtual environment." + @echo "----------------------------------------------------------------" + git fetch --tags --all make -C docs html @echo "open file://`pwd`/docs/_build/html/index.html" diff --git a/aiomysql/.gitignore b/aiomysql/.gitignore new file mode 100644 index 00000000..91cb299f --- /dev/null +++ b/aiomysql/.gitignore @@ -0,0 +1 @@ +/_scm_version.py diff --git a/aiomysql/__init__.py b/aiomysql/__init__.py index 1b08ab56..a367fcd2 100644 --- a/aiomysql/__init__.py +++ b/aiomysql/__init__.py @@ -32,8 +32,9 @@ from .connection import Connection, connect from .cursors import Cursor, SSCursor, DictCursor, SSDictCursor from .pool import create_pool, Pool +from ._version import version -__version__ = '0.0.22' +__version__ = version __all__ = [ diff --git a/aiomysql/_scm_version.pyi b/aiomysql/_scm_version.pyi new file mode 100644 index 00000000..deb2e36a --- /dev/null +++ b/aiomysql/_scm_version.pyi @@ -0,0 +1,3 @@ +# This stub file is necessary because `_scm_version.py` +# autogenerated on build and absent on mypy checks time +version: str diff --git a/aiomysql/_version.py b/aiomysql/_version.py new file mode 100644 index 00000000..80af3cd2 --- /dev/null +++ b/aiomysql/_version.py @@ -0,0 +1,4 @@ +try: + from ._scm_version import version +except ImportError: + version = "unknown" diff --git a/docs/conf.py b/docs/conf.py index 4a4019e1..027c81c5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,25 +30,8 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -import re, os.path - -def get_release(): - regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+)'") - here = os.path.dirname(__file__) - root = os.path.dirname(here) - init_py = os.path.join(root, 'aiomysql', '__init__.py') - with open(init_py) as f: - for line in f: - match = regexp.match(line) - if match is not None: - return match.group(1) - else: - raise RuntimeError('Cannot find version in aiomysql/__init__.py') - - -def get_version(release): - parts = release.split('.') - return '.'.join(parts[:2]) +from aiomysql import __version__ + extensions = [ 'sphinx.ext.autodoc', @@ -82,8 +65,8 @@ def get_version(release): # |version| and |release|, also used in various other places throughout the # built documents. # -release = get_release() -version = get_version(release) +release = __version__ +version = '.'.join(__version__.split('.')[:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pyproject.toml b/pyproject.toml index 05e4daa9..4e903b7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,5 +2,12 @@ requires = [ # Essentials "setuptools >= 42", + + # Plugins + "setuptools_scm[toml] >= 6.4", + "setuptools_scm_git_archive >= 1.1", ] build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "aiomysql/_scm_version.py" From f12aa64022b055018826c89ef255e83e87dc74bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Apr 2022 22:27:10 +0200 Subject: [PATCH 71/92] Bump pytest from 7.1.0 to 7.1.1 (#749) Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.1.0 to 7.1.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.1.0...7.1.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d47d30ed..79bb620e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ coverage==6.3.2 flake8==4.0.1 ipdb==0.13.9 -pytest==7.1.0 +pytest==7.1.1 pytest-cov==3.0.0 pytest-sugar==0.9.4 PyMySQL==1.0.2 From dc031e294fb13a887405f316799e5481f6b1c50a Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 10 Apr 2022 03:36:17 +0200 Subject: [PATCH 72/92] Implement release workflow in GitHub actions (#734) - heavily based on @webknjaz's work on https://github.com/cherrypy/cheroot/blob/b2c619f3e60682b9405a87cea48e8d30801b6048/.github/workflows/ci-cd.yml - create package before running tests, run tests against same package that will be published - run nightly tests - publish package to test pypi for commits to master - publish package to pypi using workflow_dispatch - create git tag and github release after successful tests on workflow_dispatch - add twine as explicit dev requirement Co-authored-by: Sviatoslav Sydorenko --- .github/workflows/ci-cd.yml | 761 ++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 153 -------- .github/workflows/lint.yml | 49 --- requirements-dev.txt | 1 + 4 files changed, 762 insertions(+), 202 deletions(-) create mode 100644 .github/workflows/ci-cd.yml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 00000000..e421705f --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,761 @@ +name: CI/CD + +on: + push: + branches-ignore: + - dependabot/** + pull_request: + workflow_dispatch: + inputs: + release-version: + # github.event_name == 'workflow_dispatch' + # && github.event.inputs.release-version + description: >- + Target PEP440-compliant version to release. + Please, don't prepend `v`. + required: true + release-commitish: + # github.event_name == 'workflow_dispatch' + # && github.event.inputs.release-commitish + default: '' + description: >- + The commit to be released to PyPI and tagged + in Git as `release-version`. Normally, you + should keep this empty. + required: false + YOLO: + default: false + description: >- + Flag whether test results should block the + release (true/false). Only use this under + extraordinary circumstances to ignore the + test failures and cut the release regardless. + required: false + schedule: + - cron: 1 0 * * * # Run daily at 0:01 UTC + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + pre-setup: + name: โš™๏ธ Pre-set global build settings + runs-on: ubuntu-latest + defaults: + run: + shell: python + outputs: + dist-version: >- + ${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }} + is-untagged-devel: >- + ${{ steps.untagged-check.outputs.is-untagged-devel || false }} + release-requested: >- + ${{ + steps.request-check.outputs.release-requested || false + }} + cache-key-files: >- + ${{ steps.calc-cache-key-files.outputs.files-hash-key }} + git-tag: ${{ steps.git-tag.outputs.tag }} + sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }} + wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }} + steps: + - name: Switch to using Python 3.10 by default + uses: actions/setup-python@v3 + with: + python-version: >- + 3.10 + - name: >- + Mark the build as untagged '${{ + github.event.repository.default_branch + }}' branch build + id: untagged-check + if: >- + github.event_name == 'push' && + github.ref == format( + 'refs/heads/{0}', github.event.repository.default_branch + ) + run: >- + print('::set-output name=is-untagged-devel::true') + - name: Mark the build as "release request" + id: request-check + if: github.event_name == 'workflow_dispatch' + run: >- + print('::set-output name=release-requested::true') + - name: Check out src from Git + if: >- + steps.request-check.outputs.release-requested != 'true' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.release-commitish }} + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + if: >- + steps.request-check.outputs.release-requested != 'true' + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + - name: >- + Calculate dependency files' combined hash value + for use in the cache key + if: >- + steps.request-check.outputs.release-requested != 'true' + id: calc-cache-key-files + run: | + print( + "::set-output name=files-hash-key::${{ + hashFiles( + 'requirements-dev.txt', + 'setup.cfg', + 'pyproject.toml' + ) + }}", + ) + - name: Get pip cache dir + id: pip-cache-dir + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + echo "::set-output name=dir::$(python -m pip cache dir)" + shell: bash + - name: Set up pip cache + if: >- + steps.request-check.outputs.release-requested != 'true' + uses: actions/cache@v2.1.7 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + steps.calc-cache-key-files.outputs.files-hash-key }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + ${{ runner.os }}- + - name: Drop Git tags from HEAD for non-release requests + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + git tag --points-at HEAD + | + xargs git tag --delete + shell: bash + - name: Set up versioning prerequisites + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + python -m + pip install + --user + --upgrade + setuptools-scm + shell: bash + - name: Set the current dist version from Git + if: steps.request-check.outputs.release-requested != 'true' + id: scm-version + run: | + import setuptools_scm + ver = setuptools_scm.get_version( + ${{ + steps.untagged-check.outputs.is-untagged-devel == 'true' + && 'local_scheme="no-local-version"' || '' + }} + ) + print('::set-output name=dist-version::{ver}'.format(ver=ver)) + - name: Set the target Git tag + id: git-tag + run: >- + print('::set-output name=tag::v${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }}') + - name: Set the expected dist artifact names + id: artifact-name + run: | + print('::set-output name=sdist::aiomysql-${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }}.tar.gz') + print('::set-output name=wheel::aiomysql-${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }}-py3-none-any.whl') + + build: + name: >- + ๐Ÿ‘ท dists ${{ needs.pre-setup.outputs.git-tag }} + [mode: ${{ + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + && 'nightly' || '' + }}${{ + fromJSON(needs.pre-setup.outputs.release-requested) + && 'release' || '' + }}${{ + ( + !fromJSON(needs.pre-setup.outputs.is-untagged-devel) + && !fromJSON(needs.pre-setup.outputs.release-requested) + ) && 'test' || '' + }}] + needs: + - pre-setup # transitive, for accessing settings + + runs-on: ubuntu-latest + + env: + PY_COLORS: 1 + + steps: + - name: Switch to using Python v3.10 + uses: actions/setup-python@v3 + with: + python-version: >- + 3.10 + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + shell: python + - name: Get pip cache dir + id: pip-cache-dir + run: >- + echo "::set-output name=dir::$(python -m pip cache dir)" + - name: Set up pip cache + uses: actions/cache@v2.1.7 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + needs.pre-setup.outputs.cache-key-files }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + - name: Install build tools + run: >- + python -m + pip install + --user + --upgrade + build + + - name: Grab the source from Git + uses: actions/checkout@v3 + with: + fetch-depth: >- + ${{ + steps.request-check.outputs.release-requested == 'true' + && 1 || 0 + }} + ref: ${{ github.event.inputs.release-commitish }} + + - name: Setup git user as [bot] + if: >- + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + || fromJSON(needs.pre-setup.outputs.release-requested) + uses: fregante/setup-git-user@6cef8bf084d00360a293e0cc3c56e1b45d6502b8 + - name: >- + Tag the release in the local Git repo + as ${{ needs.pre-setup.outputs.git-tag }} + for setuptools-scm to set the desired version + if: >- + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + || fromJSON(needs.pre-setup.outputs.release-requested) + run: >- + git tag + -m '${{ needs.pre-setup.outputs.git-tag }}' + '${{ needs.pre-setup.outputs.git-tag }}' + -- + ${{ github.event.inputs.release-commitish }} + - name: Build dists + run: >- + python + -m + build + - name: Verify that the artifacts with expected names got created + run: >- + ls -1 + 'dist/${{ needs.pre-setup.outputs.sdist-artifact-name }}' + 'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' + - name: Store the distribution packages + uses: actions/upload-artifact@v2 + with: + name: python-package-distributions + # NOTE: Exact expected file names are specified here + # NOTE: as a safety measure โ€” if anything weird ends + # NOTE: up being in this dir or not all dists will be + # NOTE: produced, this will fail the workflow. + path: | + dist/${{ needs.pre-setup.outputs.sdist-artifact-name }} + dist/${{ needs.pre-setup.outputs.wheel-artifact-name }} + retention-days: 30 + + lint: + name: ๐Ÿงน Lint + + needs: + - build + - pre-setup # transitive, for accessing settings + + runs-on: ubuntu-latest + + env: + PY_COLORS: 1 + + steps: + - name: Switch to using Python 3.10 by default + uses: actions/setup-python@v3 + with: + python-version: >- + 3.10 + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + shell: python + - name: Get pip cache dir + id: pip-cache-dir + run: >- + echo "::set-output name=dir::$(python -m pip cache dir)" + - name: Set up pip cache + uses: actions/cache@v2.1.7 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + needs.pre-setup.outputs.cache-key-files }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + + - name: Grab the source from Git + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.release-commitish }} + + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + + - name: Install build tools + run: >- + python -m + pip install + --user + --requirement requirements-dev.txt + + - name: flake8 Lint + uses: py-actions/flake8@v2.0.0 + with: + flake8-version: 4.0.1 + path: aiomysql + args: tests examples + + - name: Check package description + run: | + python -m twine check --strict dist/* + + tests: + name: >- + ๐Ÿงช ๐Ÿ${{ + matrix.py + }} @ ${{ + matrix.os + }} on ${{ + join(matrix.db, '-') + }} + needs: + - build + - pre-setup # transitive, for accessing settings + strategy: + matrix: + # service containers are only supported on ubuntu currently + os: + - ubuntu-latest + py: + - '3.7' + - '3.8' + - '3.9' + - '3.10' + - '3.11-dev' + db: + - [mysql, '5.7'] + - [mysql, '8.0'] + - [mariadb, '10.2'] + - [mariadb, '10.3'] + - [mariadb, '10.4'] + - [mariadb, '10.5'] + - [mariadb, '10.6'] + - [mariadb, '10.7'] + + fail-fast: false + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + + continue-on-error: >- + ${{ + ( + ( + needs.pre-setup.outputs.release-requested == 'true' && + !toJSON(github.event.inputs.YOLO) + ) || + contains(matrix.py, '-dev') + ) && true || false + }} + + env: + MYSQL_ROOT_PASSWORD: rootpw + PY_COLORS: 1 + + services: + mysql: + image: "${{ join(matrix.db, ':') }}" + ports: + - 3306:3306 + volumes: + - "/tmp/run-${{ join(matrix.db, '-') }}/:/socket-mount/" + options: '--name=mysqld' + env: + MYSQL_ROOT_PASSWORD: rootpw + + steps: + - name: Setup Python ${{ matrix.py }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.py }} + + - name: Figure out if the interpreter ABI is stable + id: py-abi + run: | + from sys import version_info + is_stable_abi = version_info.releaselevel == 'final' + print( + '::set-output name=is-stable-abi::{is_stable_abi}'. + format(is_stable_abi=str(is_stable_abi).lower()) + ) + shell: python + + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + if: fromJSON(steps.py-abi.outputs.is-stable-abi) + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + hash = sha512(version.encode()).hexdigest() + print('::set-output name=py-hash-key::{hash}'.format(hash=hash)) + shell: python + + - name: Get pip cache dir + if: fromJSON(steps.py-abi.outputs.is-stable-abi) + id: pip-cache-dir + run: >- + echo "::set-output name=dir::$(python -m pip cache dir)" + + - name: Set up pip cache + if: fromJSON(steps.py-abi.outputs.is-stable-abi) + uses: actions/cache@v2.1.7 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + needs.pre-setup.outputs.cache-key-files }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + + - name: Update pip + run: >- + python -m + pip install + --user + pip + + - name: Grab the source from Git + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.release-commitish }} + + - name: Remove aiomysql source to avoid accidentally using it + run: >- + rm -rf aiomysql + + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + + - name: Install dependencies + run: >- + python -m + pip install + --user + --requirement requirements-dev.txt + + - name: Install previously built wheel + run: >- + python -m + pip install + --user + 'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' + + - name: >- + Log platform.platform() + run: >- + python -m platform + - name: >- + Log platform.version() + run: >- + python -c "import platform; + print(platform.version())" + - name: >- + Log platform.uname() + run: >- + python -c "import platform; + print(platform.uname())" + - name: >- + Log platform.release() + run: >- + python -c "import platform; + print(platform.release())" + - name: Log stdlib OpenSSL version + run: >- + python -c + "import ssl; print('\nOPENSSL_VERSION: ' + + ssl.OPENSSL_VERSION + '\nOPENSSL_VERSION_INFO: ' + + repr(ssl.OPENSSL_VERSION_INFO) + + '\nOPENSSL_VERSION_NUMBER: ' + + repr(ssl.OPENSSL_VERSION_NUMBER))" + + # this ensures our database is ready. typically by the time the preparations have completed its first start logic. + # unfortunately we need this hacky workaround as GitHub Actions service containers can't reference data from our repo. + - name: Prepare mysql + run: | + # ensure server is started up + while : + do + sleep 1 + mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break + done + + # inject tls configuration + docker container stop mysqld + docker container cp "${{ github.workspace }}/tests/ssl_resources/ssl" mysqld:/etc/mysql/ssl + docker container cp "${{ github.workspace }}/tests/ssl_resources/tls.cnf" mysqld:/etc/mysql/conf.d/aiomysql-tls.cnf + + # use custom socket path + # we need to ensure that the socket path is writable for the user running the DB process in the container + sudo chmod 0777 /tmp/run-${{ join(matrix.db, '-') }} + + # mysql 5.7 container overrides the socket path in /etc/mysql/mysql.conf.d/mysqld.cnf + if [ "${{ join(matrix.db, '-') }}" = "mysql-5.7" ] + then + docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/mysql.conf.d/zz-aiomysql-socket.cnf + else + docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/conf.d/aiomysql-socket.cnf + fi + + docker container start mysqld + + # ensure server is started up + while : + do + sleep 1 + mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break + done + + mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e "SET GLOBAL local_infile=on" + + - name: Run tests + run: | + # timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs + timeout --preserve-status --signal=INT --verbose 5m \ + pytest --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql --cov tests ./tests --mysql-unix-socket "unix-${{ join(matrix.db, '') }}=/tmp/run-${{ join(matrix.db, '-') }}/mysql.sock" --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306" + env: + PYTHONUNBUFFERED: 1 + timeout-minutes: 6 + + - name: Upload coverage + uses: codecov/codecov-action@v2.1.0 + with: + file: ./coverage.xml + flags: "${{ matrix.os }}_${{ matrix.py }}_${{ join(matrix.db, '-') }}" + fail_ci_if_error: true + + check: # This job does nothing and is only used for the branch protection + if: always() + + needs: + - lint + - tests + + runs-on: ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@v1.1.0 + with: + jobs: ${{ toJSON(needs) }} + + publish-pypi: + name: Publish ๐Ÿ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} to PyPI + needs: + - check + - pre-setup # transitive, for accessing settings + if: >- + fromJSON(needs.pre-setup.outputs.release-requested) + runs-on: ubuntu-latest + + environment: + name: pypi + url: >- + https://pypi.org/project/aiomysql/${{ + needs.pre-setup.outputs.dist-version + }} + + steps: + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + - name: >- + Publish ๐Ÿ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} to PyPI + uses: pypa/gh-action-pypi-publish@v1.5.0 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + print_hash: true + + publish-testpypi: + name: Publish ๐Ÿ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI + needs: + - check + - pre-setup # transitive, for accessing settings + if: >- + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + || fromJSON(needs.pre-setup.outputs.release-requested) + runs-on: ubuntu-latest + + environment: + name: testpypi + url: >- + https://test.pypi.org/project/aiomysql/${{ + needs.pre-setup.outputs.dist-version + }} + + steps: + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + - name: >- + Publish ๐Ÿ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI + uses: pypa/gh-action-pypi-publish@v1.5.0 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + print_hash: true + + post-release-repo-update: + name: >- + Publish post-release Git tag + for ${{ needs.pre-setup.outputs.git-tag }} + needs: + - publish-pypi + - pre-setup # transitive, for accessing settings + runs-on: ubuntu-latest + + steps: + - name: Fetch the src snapshot + uses: actions/checkout@v3 + with: + fetch-depth: 1 + ref: ${{ github.event.inputs.release-commitish }} + - name: Setup git user as [bot] + uses: fregante/setup-git-user@6cef8bf084d00360a293e0cc3c56e1b45d6502b8 + + - name: >- + Tag the release in the local Git repo + as v${{ needs.pre-setup.outputs.git-tag }} + run: >- + git tag + -m '${{ needs.pre-setup.outputs.git-tag }}' + '${{ needs.pre-setup.outputs.git-tag }}' + -- + ${{ github.event.inputs.release-commitish }} + - name: >- + Push ${{ needs.pre-setup.outputs.git-tag }} tag corresponding + to the just published release back to GitHub + run: >- + git push --atomic origin '${{ needs.pre-setup.outputs.git-tag }}' + + publish-github-release: + name: >- + Publish a tag and GitHub release for + ${{ needs.pre-setup.outputs.git-tag }} + needs: + - post-release-repo-update + - pre-setup # transitive, for accessing settings + runs-on: ubuntu-latest + + permissions: + contents: write + discussions: write + + steps: + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ + + - name: >- + Publish a GitHub Release for + ${{ needs.pre-setup.outputs.git-tag }} + uses: ncipollo/release-action@40bb172bd05f266cf9ba4ff965cb61e9ee5f6d01 + with: + artifacts: | + dist/${{ needs.pre-setup.outputs.sdist-artifact-name }} + dist/${{ needs.pre-setup.outputs.wheel-artifact-name }} + artifactContentType: raw # Because whl and tgz are of different types + # FIXME: Use Towncrier once it is integrated. + bodyFile: CHANGES.txt + discussionCategory: Announcements + name: ${{ needs.pre-setup.outputs.git-tag }} + tag: ${{ needs.pre-setup.outputs.git-tag }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index ab2a59f5..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: CI - -on: - push: - branches: - - 'master' - tags: - - 'v*' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - - -jobs: - test: - name: Test - strategy: - matrix: - # service containers are only supported on ubuntu currently - os: - - ubuntu-latest - py: - - '3.7' - - '3.8' - - '3.9' - - '3.10' - - '3.11-dev' - db: - - [mysql, '5.7'] - - [mysql, '8.0'] - - [mariadb, '10.2'] - - [mariadb, '10.3'] - - [mariadb, '10.4'] - - [mariadb, '10.5'] - - [mariadb, '10.6'] - - [mariadb, '10.7'] - - fail-fast: false - runs-on: ${{ matrix.os }} - timeout-minutes: 15 - - env: - MYSQL_ROOT_PASSWORD: rootpw - - services: - mysql: - image: "${{ join(matrix.db, ':') }}" - ports: - - 3306:3306 - volumes: - - "/tmp/run-${{ join(matrix.db, '-') }}/:/socket-mount/" - options: '--name=mysqld' - env: - MYSQL_ROOT_PASSWORD: rootpw - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Setup Python ${{ matrix.py }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.py }} - - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" # - name: Cache - - - name: Cache PyPI - uses: actions/cache@v2.1.7 - with: - key: pip-ci-${{ runner.os }}-${{ matrix.py }} - path: ${{ steps.pip-cache.outputs.dir }} - - - name: Update pip, wheel, setuptools, build, twine, codecov - run: | - python -m pip install -U pip wheel setuptools build twine codecov - - - name: Install dependencies - run: | - python -m pip install --upgrade --requirement requirements-dev.txt - - - name: Build distribution packages - run: | - python -m build - - - name: Check package description - run: | - python -m twine check --strict dist/* - - - name: Install aiomysql - run: | - python -m pip install . - - # this ensures our database is ready. typically by the time the preparations have completed its first start logic. - # unfortunately we need this hacky workaround as GitHub Actions service containers can't reference data from our repo. - - name: Prepare mysql - run: | - # ensure server is started up - while : - do - sleep 1 - mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break - done - - # inject tls configuration - docker container stop mysqld - docker container cp "${{ github.workspace }}/tests/ssl_resources/ssl" mysqld:/etc/mysql/ssl - docker container cp "${{ github.workspace }}/tests/ssl_resources/tls.cnf" mysqld:/etc/mysql/conf.d/aiomysql-tls.cnf - - # use custom socket path - # we need to ensure that the socket path is writable for the user running the DB process in the container - sudo chmod 0777 /tmp/run-${{ join(matrix.db, '-') }} - - # mysql 5.7 container overrides the socket path in /etc/mysql/mysql.conf.d/mysqld.cnf - if [ "${{ join(matrix.db, '-') }}" = "mysql-5.7" ] - then - docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/mysql.conf.d/zz-aiomysql-socket.cnf - else - docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/conf.d/aiomysql-socket.cnf - fi - - docker container start mysqld - - # ensure server is started up - while : - do - sleep 1 - mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break - done - - mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e "SET GLOBAL local_infile=on" - - - name: Run tests - run: | - # timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs - timeout --preserve-status --signal=INT --verbose 5m \ - pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql --cov tests ./tests --mysql-unix-socket "unix-${{ join(matrix.db, '') }}=/tmp/run-${{ join(matrix.db, '-') }}/mysql.sock" --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306" - env: - PYTHONUNBUFFERED: 1 - timeout-minutes: 6 - - - name: Upload coverage - uses: codecov/codecov-action@v2.1.0 - with: - file: ./coverage.xml - flags: "${{ matrix.os }}_${{ matrix.py }}_${{ join(matrix.db, '-') }}" - fail_ci_if_error: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 363d1746..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: lint - -on: - push: - branches: - - 'master' - tags: - - 'v*' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -jobs: - lint: - name: Lint - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" # - name: Cache - - - name: Cache PyPI - uses: actions/cache@v2.1.7 - with: - key: pip-lint - path: ${{ steps.pip-cache.outputs.dir }} - - - name: Install dependencies - run: | - python -m pip install --upgrade --requirement requirements-dev.txt - - - name: flake8 Lint - uses: py-actions/flake8@v2.0.0 - with: - flake8-version: 4.0.1 - path: aiomysql - args: tests examples diff --git a/requirements-dev.txt b/requirements-dev.txt index 79bb620e..d1067624 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,3 +9,4 @@ sphinx>=1.8.1, <4.4.1 sphinxcontrib-asyncio==0.3.0 SQLAlchemy==1.3.24 uvloop==0.16.0; python_version < '3.11' +twine==4.0.0 From 50613513f366062f286b91f048a40c5ccf44b5c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 Apr 2022 03:53:27 +0200 Subject: [PATCH 73/92] Bump actions/cache from 2.1.7 to 3.0.1 (#752) Bumps [actions/cache](https://github.com/actions/cache) from 2.1.7 to 3.0.1. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v2.1.7...v3.0.1) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-cd.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index e421705f..0d42436b 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -130,7 +130,7 @@ jobs: - name: Set up pip cache if: >- steps.request-check.outputs.release-requested != 'true' - uses: actions/cache@v2.1.7 + uses: actions/cache@v3.0.1 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: >- @@ -239,7 +239,7 @@ jobs: run: >- echo "::set-output name=dir::$(python -m pip cache dir)" - name: Set up pip cache - uses: actions/cache@v2.1.7 + uses: actions/cache@v3.0.1 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: >- @@ -343,7 +343,7 @@ jobs: run: >- echo "::set-output name=dir::$(python -m pip cache dir)" - name: Set up pip cache - uses: actions/cache@v2.1.7 + uses: actions/cache@v3.0.1 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: >- @@ -485,7 +485,7 @@ jobs: - name: Set up pip cache if: fromJSON(steps.py-abi.outputs.is-stable-abi) - uses: actions/cache@v2.1.7 + uses: actions/cache@v3.0.1 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: >- From 0255cfdcf7e94a67146f000f6b9baffbfc300b46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 Apr 2022 04:15:15 +0200 Subject: [PATCH 74/92] Bump codecov/codecov-action from 2.1.0 to 3.0.0 (#758) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 2.1.0 to 3.0.0. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v2.1.0...v3.0.0) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 0d42436b..eadc43af 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -611,7 +611,7 @@ jobs: timeout-minutes: 6 - name: Upload coverage - uses: codecov/codecov-action@v2.1.0 + uses: codecov/codecov-action@v3.0.0 with: file: ./coverage.xml flags: "${{ matrix.os }}_${{ matrix.py }}_${{ join(matrix.db, '-') }}" From a0b7a82c27130bccb067f5060a22ddba0138d688 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 Apr 2022 04:58:07 +0200 Subject: [PATCH 75/92] Update sphinx requirement from <4.4.1,>=1.8.1 to 4.5.0 (#751) Updates the requirements on [Sphinx](https://github.com/sphinx-doc/sphinx) to the latest version. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/4.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v1.8.1...v4.5.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development ... Co-authored-by: Richard Schwab --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d1067624..51cf82bb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,7 @@ pytest==7.1.1 pytest-cov==3.0.0 pytest-sugar==0.9.4 PyMySQL==1.0.2 -sphinx>=1.8.1, <4.4.1 +sphinx>=1.8.1, <4.5.1 sphinxcontrib-asyncio==0.3.0 SQLAlchemy==1.3.24 uvloop==0.16.0; python_version < '3.11' From 89011c9797a171cfc9699af078e933c09ca9952f Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 10 Apr 2022 07:28:58 +0200 Subject: [PATCH 76/92] fix `make checkrst` (#759) fixes #756 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ae5ceeb0..798790cf 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ FLAGS= checkrst: - python setup.py check --restructuredtext + python -m twine check --strict dist/* flake:checkrst From ef42007d965871ecab0cfcb7d7c7c3fdcbc6e97f Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 10 Apr 2022 09:25:58 +0200 Subject: [PATCH 77/92] Specify explicit language for literal blocks in CONTRIBUTING.rst to reduce warnings in doc builds (#760) --- CONTRIBUTING.rst | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8d0b4e0b..ec2b3f9c 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -51,25 +51,33 @@ We expect you to use a python virtual environment to run our tests. There are several ways to make a virtual environment. -If you like to use *virtualenv* please run:: +If you like to use *virtualenv* please run: + +.. code-block:: sh $ cd aiomysql - $ virtualenv --python=`which python3` venv + $ virtualenv --python="$(which python3)" venv + +For standard python *venv*: -For standard python *venv*:: +.. code-block:: sh $ cd aiomysql $ python3 -m venv venv -For *virtualenvwrapper*:: +For *virtualenvwrapper*: + +.. code-block:: sh $ cd aiomysql - $ mkvirtualenv --python=`which python3` aiomysql + $ mkvirtualenv --python="$(which python3)" aiomysql There are other tools like *pyvenv* but you know the rule of thumb now: create a python3 virtual environment and activate it. -After that please install libraries required for development:: +After that please install libraries required for development: + +.. code-block:: sh $ pip install -r requirements-dev.txt @@ -83,7 +91,7 @@ use this values by default. But you always can override host/port, user and password in `aiomysql/tests/base.py` file or install corresponding environment variables. Tests require two databases to be created before running suit: -:: +.. code-block:: sh $ mysql -u root mysql> CREATE DATABASE test_pymysql DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; @@ -94,7 +102,9 @@ Run aiomysql test suite ----------------------- After all the preconditions are met you can run tests typing the next -command:: +command: + +.. code-block:: sh $ make test @@ -113,7 +123,9 @@ Tests coverage We are trying hard to have good test coverage; please don't make it worse. -Use:: +Use: + +.. code-block:: sh $ make cov @@ -130,7 +142,9 @@ Documentation We encourage documentation improvements. -Please before making a Pull Request about documentation changes run:: +Please before making a Pull Request about documentation changes run: + +.. code-block:: sh $ make doc From 1e474e66a3fe7d64f5f133529f1d4877267b1fbd Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Mon, 11 Apr 2022 02:09:44 +0200 Subject: [PATCH 78/92] Fix SSCursor raising query timeout error on wrong query on MySQL DB (#761) ports https://github.com/PyMySQL/PyMySQL/pull/1035/commits/dd0f854f9307f3ce4040eb63096c6bdbc7f72746 from https://github.com/PyMySQL/PyMySQL/pull/1035 fixes #428 --- CHANGES.txt | 1 + aiomysql/connection.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5c8d7b73..d47f50d0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -25,6 +25,7 @@ To be included in 1.0.0 (unreleased) * Add rsa extras_require depending on PyMySQL[rsa] #557 * Migrate to PEP 517 build system #746 * Self-reported `__version__` now returns version generated by `setuptools-scm` during build, otherwise `'unknown'` #748 +* Fix SSCursor raising query timeout error on wrong query #428 0.0.22 (2021-11-14) diff --git a/aiomysql/connection.py b/aiomysql/connection.py index 07530222..42573f2f 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -1247,7 +1247,22 @@ async def _finish_unbuffered_query(self): # in fact, no way to stop MySQL from sending all the data after # executing a query, so we just spin, and wait for an EOF packet. while self.unbuffered_active: - packet = await self.connection._read_packet() + try: + packet = await self.connection._read_packet() + except OperationalError as e: + # TODO: replace these numbers with constants when available + # TODO: in a new PyMySQL release + if e.args[0] in ( + 3024, # ER.QUERY_TIMEOUT + 1969, # ER.STATEMENT_TIMEOUT + ): + # if the query timed out we can simply ignore this error + self.unbuffered_active = False + self.connection = None + return + + raise + if self._check_packet_is_eof(packet): self.unbuffered_active = False # release reference to kill cyclic reference. From 4c1c325db36e84e968dba5d663f3a8427470726c Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Mon, 11 Apr 2022 02:29:30 +0200 Subject: [PATCH 79/92] Fix GHA badge in readme after #734 (#762) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 97bb9c34..b5e52c1d 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ aiomysql ======== -.. image:: https://github.com/aio-libs/aiomysql/actions/workflows/ci.yml/badge.svg?branch=master - :target: https://github.com/aio-libs/aiomysql/actions/workflows/ci.yml +.. image:: https://github.com/aio-libs/aiomysql/actions/workflows/ci-cd.yml/badge.svg?branch=master + :target: https://github.com/aio-libs/aiomysql/actions/workflows/ci-cd.yml .. image:: https://codecov.io/gh/aio-libs/aiomysql/branch/master/graph/badge.svg :target: https://codecov.io/gh/aio-libs/aiomysql :alt: Code coverage From f990ebfc9631fc219bb952631c48a9301d107ffb Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Mon, 11 Apr 2022 02:15:31 +0200 Subject: [PATCH 80/92] Change next version to 0.1.0 in changelog --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index d47f50d0..8a9f5bf4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,7 @@ Changes ------- -To be included in 1.0.0 (unreleased) +To be included in 0.1.0 (unreleased) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * Don't send sys.argv[0] as program_name to MySQL server by default #620 From f6b50a1d282fc501d7a570d7136bd5d2643ed440 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Mon, 11 Apr 2022 03:25:07 +0200 Subject: [PATCH 81/92] Checkout git source to access changelog for GitHub release creation --- .github/workflows/ci-cd.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index eadc43af..8cf7d2e7 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -739,6 +739,12 @@ jobs: discussions: write steps: + - name: Fetch the src snapshot + uses: actions/checkout@v3 + with: + fetch-depth: 1 + ref: ${{ github.event.inputs.release-commitish }} + - name: Download all the dists uses: actions/download-artifact@v2 with: From 63deeccae39da4d213c21e3bb93d7efa3619e7a4 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Mon, 11 Apr 2022 21:16:13 +0200 Subject: [PATCH 82/92] Update changelog for release (#769) adjust line spacing to match previous changelog entries --- CHANGES.txt | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 8a9f5bf4..79ec144c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,30 +1,51 @@ Changes ------- -To be included in 0.1.0 (unreleased) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +0.1.0 (2022-04-11) +^^^^^^^^^^^^^^^^^^ * Don't send sys.argv[0] as program_name to MySQL server by default #620 + * Allow running process as anonymous uid #587 + * Fix timed out MySQL 8.0 connections raising InternalError rather than OperationalError #660 + * Fix timed out MySQL 8.0 connections being returned from Pool #660 + * Ensure connections are properly closed before raising an OperationalError when the server connection is lost #660 + * Ensure connections are properly closed before raising an InternalError when packet sequence numbers are out of sync #660 + * Unix sockets are now internally considered secure, allowing sha256_password and caching_sha2_password auth methods to be used #695 + * Test suite now also tests unix socket connections #696 + * Fix SSCursor raising InternalError when last result was not fully retrieved #635 + * Remove deprecated no_delay argument #702 + * Support PyMySQL up to version 1.0.2 #643 + * Bump minimal PyMySQL version to 1.0.0 #713 + * Align % formatting in Cursor.executemany() with Cursor.execute(), literal % now need to be doubled in Cursor.executemany() #714 + * Fixed unlimited Pool size not working, this is now working as documented by passing maxsize=0 to create_pool #119 + * Added Pool.closed property as present in aiopg #463 + * Fixed SQLAlchemy connection context iterator #410 + * Fix error packet handling for SSCursor #428 + * Required python version is now properly documented in python_requires instead of failing on setup.py execution #731 + * Add rsa extras_require depending on PyMySQL[rsa] #557 + * Migrate to PEP 517 build system #746 + * Self-reported `__version__` now returns version generated by `setuptools-scm` during build, otherwise `'unknown'` #748 + * Fix SSCursor raising query timeout error on wrong query #428 From 766e70ed1e7dad4a6e5dbf4391db3567f9067173 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 22:31:43 +0200 Subject: [PATCH 83/92] Bump actions/upload-artifact from 2 to 3 (#767) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 2 to 3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 8cf7d2e7..4b55b39b 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -298,7 +298,7 @@ jobs: 'dist/${{ needs.pre-setup.outputs.sdist-artifact-name }}' 'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' - name: Store the distribution packages - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: python-package-distributions # NOTE: Exact expected file names are specified here From aad6121a56cc358a24da96c64365cd61bd639b48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 23:02:10 +0200 Subject: [PATCH 84/92] Bump actions/download-artifact from 2 to 3 (#765) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 2 to 3. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-cd.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 4b55b39b..0bae4888 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -362,7 +362,7 @@ jobs: ref: ${{ github.event.inputs.release-commitish }} - name: Download all the dists - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: python-package-distributions path: dist/ @@ -515,7 +515,7 @@ jobs: rm -rf aiomysql - name: Download all the dists - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: python-package-distributions path: dist/ @@ -650,7 +650,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: python-package-distributions path: dist/ @@ -680,7 +680,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: python-package-distributions path: dist/ @@ -746,7 +746,7 @@ jobs: ref: ${{ github.event.inputs.release-commitish }} - name: Download all the dists - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: python-package-distributions path: dist/ From 5a42bc1722be0d34c5212a7c742f9531eff7ac09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Apr 2022 10:20:19 +0200 Subject: [PATCH 85/92] Bump actions/cache from 3.0.1 to 3.0.2 (#771) Bumps [actions/cache](https://github.com/actions/cache) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3.0.1...v3.0.2) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-cd.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 0bae4888..33917e58 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -130,7 +130,7 @@ jobs: - name: Set up pip cache if: >- steps.request-check.outputs.release-requested != 'true' - uses: actions/cache@v3.0.1 + uses: actions/cache@v3.0.2 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: >- @@ -239,7 +239,7 @@ jobs: run: >- echo "::set-output name=dir::$(python -m pip cache dir)" - name: Set up pip cache - uses: actions/cache@v3.0.1 + uses: actions/cache@v3.0.2 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: >- @@ -343,7 +343,7 @@ jobs: run: >- echo "::set-output name=dir::$(python -m pip cache dir)" - name: Set up pip cache - uses: actions/cache@v3.0.1 + uses: actions/cache@v3.0.2 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: >- @@ -485,7 +485,7 @@ jobs: - name: Set up pip cache if: fromJSON(steps.py-abi.outputs.is-stable-abi) - uses: actions/cache@v3.0.1 + uses: actions/cache@v3.0.2 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: >- From fe0120b580ae8822e8a126d152dc3037597327f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Apr 2022 19:21:04 +0200 Subject: [PATCH 86/92] Bump ncipollo/release-action from 1.9.0 to 1.10.0 (#766) Bumps [ncipollo/release-action](https://github.com/ncipollo/release-action) from 1.9.0 to 1.10.0. - [Release notes](https://github.com/ncipollo/release-action/releases) - [Commits](https://github.com/ncipollo/release-action/compare/40bb172bd05f266cf9ba4ff965cb61e9ee5f6d01...58ae73b360456532aafd58ee170c045abbeaee37) --- updated-dependencies: - dependency-name: ncipollo/release-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 33917e58..4def26e8 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -754,7 +754,7 @@ jobs: - name: >- Publish a GitHub Release for ${{ needs.pre-setup.outputs.git-tag }} - uses: ncipollo/release-action@40bb172bd05f266cf9ba4ff965cb61e9ee5f6d01 + uses: ncipollo/release-action@58ae73b360456532aafd58ee170c045abbeaee37 with: artifacts: | dist/${{ needs.pre-setup.outputs.sdist-artifact-name }} From 23b0feaadaa33b1b33fec4e9050aede976279f1c Mon Sep 17 00:00:00 2001 From: Alvie Zhang Date: Wed, 20 Apr 2022 17:40:54 +0800 Subject: [PATCH 87/92] Fix SSL connection handshake charset not respecting client configuration (#776) fixes #775, closes #648 --- CHANGES.txt | 5 +++++ aiomysql/connection.py | 14 +++++--------- tests/test_basic.py | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 79ec144c..56fc3463 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,11 @@ Changes ------- +next (unreleased) +^^^^^^^^^^^^^^^^^ + +* Fix SSL connection handshake charset not respecting client configuration #776 + 0.1.0 (2022-04-11) ^^^^^^^^^^^^^^^^^^ diff --git a/aiomysql/connection.py b/aiomysql/connection.py index 42573f2f..022315a9 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -733,12 +733,12 @@ async def _request_authentication(self): if self.user is None: raise ValueError("Did not specify a username") - if self._ssl_context and self.server_capabilities & CLIENT.SSL: - # capablities, max packet, charset - data = struct.pack(' Date: Fri, 22 Apr 2022 03:45:29 +0200 Subject: [PATCH 88/92] Bump codecov/codecov-action from 3.0.0 to 3.1.0 (#777) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3.0.0...v3.1.0) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 4def26e8..a2f484d6 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -611,7 +611,7 @@ jobs: timeout-minutes: 6 - name: Upload coverage - uses: codecov/codecov-action@v3.0.0 + uses: codecov/codecov-action@v3.1.0 with: file: ./coverage.xml flags: "${{ matrix.os }}_${{ matrix.py }}_${{ join(matrix.db, '-') }}" From cd06af8e437369bfb3d519b5960762a817d737ad Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 24 Apr 2022 16:10:45 +0200 Subject: [PATCH 89/92] Skip codecov uploads in scheduled daily tests (#778) We can only upload a limited number of coverage reports for any commit anyway, this avoids turning these into test failures. --- .github/workflows/ci-cd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index a2f484d6..9665d579 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -611,6 +611,7 @@ jobs: timeout-minutes: 6 - name: Upload coverage + if: ${{ github.event_name != 'schedule' }} uses: codecov/codecov-action@v3.1.0 with: file: ./coverage.xml From 4c766f0f51eac33546cfae6c70ab5e76e40962fb Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 24 Apr 2022 17:34:15 +0200 Subject: [PATCH 90/92] XFail test_issue_36 for now to avoid failing for unrelated changes (#779) See #714 --- tests/test_issues.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_issues.py b/tests/test_issues.py index c25e292f..07fa4944 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -255,7 +255,11 @@ async def test_issue_36(connection_creator): rows = await c.fetchall() ids = [row[0] for row in rows] - assert kill_id not in ids + try: + assert kill_id not in ids + except AssertionError: + # FIXME: figure out why this is failing + pytest.xfail("https://github.com/aio-libs/aiomysql/issues/714") @pytest.mark.run_loop From 36a6962660f13d2c7a06d314ee650a394929fa88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 04:55:04 +0200 Subject: [PATCH 91/92] Bump pytest from 7.1.1 to 7.1.2 (#781) Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.1.1 to 7.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.1.1...7.1.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 51cf82bb..716749a7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ coverage==6.3.2 flake8==4.0.1 ipdb==0.13.9 -pytest==7.1.1 +pytest==7.1.2 pytest-cov==3.0.0 pytest-sugar==0.9.4 PyMySQL==1.0.2 From 8a32f052a16dc3886af54b98f4d91d95862bfb8e Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 8 May 2022 20:50:12 +0200 Subject: [PATCH 92/92] Update changelog for 0.1.1 release (#784) --- CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 56fc3463..c717378f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,8 +1,8 @@ Changes ------- -next (unreleased) -^^^^^^^^^^^^^^^^^ +0.1.1 (2022-05-08) +^^^^^^^^^^^^^^^^^^ * Fix SSL connection handshake charset not respecting client configuration #776