diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..2bcd70e3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 88 diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 00000000..95cb3eea --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1 @@ +ref-names: $Format:%D$ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..ec8c3333 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Force LF line endings for text files +* text=auto eol=lf + +# Needed for setuptools-scm-git-archive +.git_archival.txt export-subst diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..99aa53df --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,130 @@ +--- +name: Bug Report +description: Create a report to help us improve. +labels: [bug] +body: +- type: markdown + attributes: + value: | + **Thanks for taking a minute to file a bug report!** + + โš  + Verify first that your issue is not [already reported on + GitHub][issue search]. + + _Please fill out the form below with as many precise + details as possible._ + + [issue search]: ../search?q=is%3Aissue&type=issues + +- type: textarea + attributes: + label: Describe the bug + description: >- + A clear and concise description of what the bug is. + validations: + required: true + +- type: textarea + attributes: + label: To Reproduce + description: >- + Describe the steps to reproduce this bug. + placeholder: | + 1. Have certain environment + 2. Then run '...' + 3. An error occurs. + validations: + required: true + +- type: textarea + attributes: + label: Expected behavior + description: >- + A clear and concise description of what you expected to happen. + validations: + required: true + +- type: textarea + attributes: + label: Logs/tracebacks + description: | + If applicable, add logs/tracebacks to help explain your problem. + Paste the output of the steps above, including the commands + themselves and their output/traceback etc. + render: python-traceback + validations: + required: true + +- type: textarea + attributes: + label: Python Version + description: Attach your version of Python. + render: console + value: | + $ python --version + validations: + required: true +- type: textarea + attributes: + label: aiomysql Version + description: Attach your version of aiomysql. + render: console + value: | + $ python -m pip show aiomysql + validations: + required: true +- type: textarea + attributes: + label: PyMySQL Version + description: Attach your version of PyMySQL. + render: console + value: | + $ python -m pip show PyMySQL + validations: + required: true +- type: textarea + attributes: + label: SQLAlchemy Version + description: Attach your version of SQLAlchemy if you're using it. + render: console + value: | + $ python -m pip show sqlalchemy + +- type: textarea + attributes: + label: OS + placeholder: >- + For example, Arch Linux, Windows, macOS, etc. + validations: + required: true + +- type: textarea + attributes: + label: Database type and version + description: Attach your version of MariaDB/MySQL. + render: console + value: | + SELECT VERSION(); + validations: + required: true + +- type: textarea + attributes: + label: Additional context + description: | + Add any other context about the problem here. + + Describe the environment you have that lead to your issue. + +- type: checkboxes + attributes: + label: Code of Conduct + description: | + Read the [aio-libs Code of Conduct][CoC] first. + + [CoC]: https://github.com/aio-libs/.github/blob/master/CODE_OF_CONDUCT.md + options: + - label: I agree to follow the aio-libs Code of Conduct + required: true +... diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000..992c1b64 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,61 @@ +--- +name: ๐Ÿš€ Feature request +description: Suggest an idea for this project. +labels: enhancement +body: +- type: markdown + attributes: + value: | + **Thanks for taking a minute to file a feature for aiomysql!** + + โš  + Verify first that your feature request is not [already reported on + GitHub][issue search]. + + _Please fill out the form below with as many precise + details as possible._ + + [issue search]: ../search?q=is%3Aissue&type=issues + +- type: textarea + attributes: + label: Is your feature request related to a problem? + description: >- + Please add a clear and concise description of what + the problem is. _Ex. I'm always frustrated when [...]_ + +- type: textarea + attributes: + label: Describe the solution you'd like + description: >- + A clear and concise description of what you want to happen. + validations: + required: true + +- type: textarea + attributes: + label: Describe alternatives you've considered + description: >- + A clear and concise description of any alternative solutions + or features you've considered. + validations: + required: true + +- type: textarea + attributes: + label: Additional context + description: >- + Add any other context or screenshots about + the feature request here. + +- type: checkboxes + attributes: + label: Code of Conduct + description: | + Read the [aio-libs Code of Conduct][CoC] first. + + [CoC]: https://github.com/aio-libs/.github/blob/master/CODE_OF_CONDUCT.md + options: + - label: I agree to follow the aio-libs Code of Conduct + required: true +... diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..10c63edc --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + target-branch: master +- package-ecosystem: github-actions + directory: / + schedule: + interval: daily + open-pull-requests-limit: 10 diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 00000000..9665d579 --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,768 @@ +name: CI/CD + +on: + push: + branches-ignore: + - dependabot/** + pull_request: + workflow_dispatch: + inputs: + release-version: + # github.event_name == 'workflow_dispatch' + # && github.event.inputs.release-version + description: >- + Target PEP440-compliant version to release. + Please, don't prepend `v`. + required: true + release-commitish: + # github.event_name == 'workflow_dispatch' + # && github.event.inputs.release-commitish + default: '' + description: >- + The commit to be released to PyPI and tagged + in Git as `release-version`. Normally, you + should keep this empty. + required: false + YOLO: + default: false + description: >- + Flag whether test results should block the + release (true/false). Only use this under + extraordinary circumstances to ignore the + test failures and cut the release regardless. + required: false + schedule: + - cron: 1 0 * * * # Run daily at 0:01 UTC + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + pre-setup: + name: โš™๏ธ Pre-set global build settings + runs-on: ubuntu-latest + defaults: + run: + shell: python + outputs: + dist-version: >- + ${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }} + is-untagged-devel: >- + ${{ steps.untagged-check.outputs.is-untagged-devel || false }} + release-requested: >- + ${{ + steps.request-check.outputs.release-requested || false + }} + cache-key-files: >- + ${{ steps.calc-cache-key-files.outputs.files-hash-key }} + git-tag: ${{ steps.git-tag.outputs.tag }} + sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }} + wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }} + steps: + - name: Switch to using Python 3.10 by default + uses: actions/setup-python@v3 + with: + python-version: >- + 3.10 + - name: >- + Mark the build as untagged '${{ + github.event.repository.default_branch + }}' branch build + id: untagged-check + if: >- + github.event_name == 'push' && + github.ref == format( + 'refs/heads/{0}', github.event.repository.default_branch + ) + run: >- + print('::set-output name=is-untagged-devel::true') + - name: Mark the build as "release request" + id: request-check + if: github.event_name == 'workflow_dispatch' + run: >- + print('::set-output name=release-requested::true') + - name: Check out src from Git + if: >- + steps.request-check.outputs.release-requested != 'true' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.release-commitish }} + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + if: >- + steps.request-check.outputs.release-requested != 'true' + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + - name: >- + Calculate dependency files' combined hash value + for use in the cache key + if: >- + steps.request-check.outputs.release-requested != 'true' + id: calc-cache-key-files + run: | + print( + "::set-output name=files-hash-key::${{ + hashFiles( + 'requirements-dev.txt', + 'setup.cfg', + 'pyproject.toml' + ) + }}", + ) + - name: Get pip cache dir + id: pip-cache-dir + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + echo "::set-output name=dir::$(python -m pip cache dir)" + shell: bash + - name: Set up pip cache + if: >- + steps.request-check.outputs.release-requested != 'true' + uses: actions/cache@v3.0.2 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + steps.calc-cache-key-files.outputs.files-hash-key }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + ${{ runner.os }}- + - name: Drop Git tags from HEAD for non-release requests + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + git tag --points-at HEAD + | + xargs git tag --delete + shell: bash + - name: Set up versioning prerequisites + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + python -m + pip install + --user + --upgrade + setuptools-scm + shell: bash + - name: Set the current dist version from Git + if: steps.request-check.outputs.release-requested != 'true' + id: scm-version + run: | + import setuptools_scm + ver = setuptools_scm.get_version( + ${{ + steps.untagged-check.outputs.is-untagged-devel == 'true' + && 'local_scheme="no-local-version"' || '' + }} + ) + print('::set-output name=dist-version::{ver}'.format(ver=ver)) + - name: Set the target Git tag + id: git-tag + run: >- + print('::set-output name=tag::v${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }}') + - name: Set the expected dist artifact names + id: artifact-name + run: | + print('::set-output name=sdist::aiomysql-${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }}.tar.gz') + print('::set-output name=wheel::aiomysql-${{ + steps.request-check.outputs.release-requested == 'true' + && github.event.inputs.release-version + || steps.scm-version.outputs.dist-version + }}-py3-none-any.whl') + + build: + name: >- + ๐Ÿ‘ท dists ${{ needs.pre-setup.outputs.git-tag }} + [mode: ${{ + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + && 'nightly' || '' + }}${{ + fromJSON(needs.pre-setup.outputs.release-requested) + && 'release' || '' + }}${{ + ( + !fromJSON(needs.pre-setup.outputs.is-untagged-devel) + && !fromJSON(needs.pre-setup.outputs.release-requested) + ) && 'test' || '' + }}] + needs: + - pre-setup # transitive, for accessing settings + + runs-on: ubuntu-latest + + env: + PY_COLORS: 1 + + steps: + - name: Switch to using Python v3.10 + uses: actions/setup-python@v3 + with: + python-version: >- + 3.10 + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + shell: python + - name: Get pip cache dir + id: pip-cache-dir + run: >- + echo "::set-output name=dir::$(python -m pip cache dir)" + - name: Set up pip cache + uses: actions/cache@v3.0.2 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + needs.pre-setup.outputs.cache-key-files }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + - name: Install build tools + run: >- + python -m + pip install + --user + --upgrade + build + + - name: Grab the source from Git + uses: actions/checkout@v3 + with: + fetch-depth: >- + ${{ + steps.request-check.outputs.release-requested == 'true' + && 1 || 0 + }} + ref: ${{ github.event.inputs.release-commitish }} + + - name: Setup git user as [bot] + if: >- + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + || fromJSON(needs.pre-setup.outputs.release-requested) + uses: fregante/setup-git-user@6cef8bf084d00360a293e0cc3c56e1b45d6502b8 + - name: >- + Tag the release in the local Git repo + as ${{ needs.pre-setup.outputs.git-tag }} + for setuptools-scm to set the desired version + if: >- + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + || fromJSON(needs.pre-setup.outputs.release-requested) + run: >- + git tag + -m '${{ needs.pre-setup.outputs.git-tag }}' + '${{ needs.pre-setup.outputs.git-tag }}' + -- + ${{ github.event.inputs.release-commitish }} + - name: Build dists + run: >- + python + -m + build + - name: Verify that the artifacts with expected names got created + run: >- + ls -1 + 'dist/${{ needs.pre-setup.outputs.sdist-artifact-name }}' + 'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' + - name: Store the distribution packages + uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + # NOTE: Exact expected file names are specified here + # NOTE: as a safety measure โ€” if anything weird ends + # NOTE: up being in this dir or not all dists will be + # NOTE: produced, this will fail the workflow. + path: | + dist/${{ needs.pre-setup.outputs.sdist-artifact-name }} + dist/${{ needs.pre-setup.outputs.wheel-artifact-name }} + retention-days: 30 + + lint: + name: ๐Ÿงน Lint + + needs: + - build + - pre-setup # transitive, for accessing settings + + runs-on: ubuntu-latest + + env: + PY_COLORS: 1 + + steps: + - name: Switch to using Python 3.10 by default + uses: actions/setup-python@v3 + with: + python-version: >- + 3.10 + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + hash = sha512(version.encode()).hexdigest() + print(f'::set-output name=py-hash-key::{hash}') + shell: python + - name: Get pip cache dir + id: pip-cache-dir + run: >- + echo "::set-output name=dir::$(python -m pip cache dir)" + - name: Set up pip cache + uses: actions/cache@v3.0.2 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + needs.pre-setup.outputs.cache-key-files }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + + - name: Grab the source from Git + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.release-commitish }} + + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + - name: Install build tools + run: >- + python -m + pip install + --user + --requirement requirements-dev.txt + + - name: flake8 Lint + uses: py-actions/flake8@v2.0.0 + with: + flake8-version: 4.0.1 + path: aiomysql + args: tests examples + + - name: Check package description + run: | + python -m twine check --strict dist/* + + tests: + name: >- + ๐Ÿงช ๐Ÿ${{ + matrix.py + }} @ ${{ + matrix.os + }} on ${{ + join(matrix.db, '-') + }} + needs: + - build + - pre-setup # transitive, for accessing settings + strategy: + matrix: + # service containers are only supported on ubuntu currently + os: + - ubuntu-latest + py: + - '3.7' + - '3.8' + - '3.9' + - '3.10' + - '3.11-dev' + db: + - [mysql, '5.7'] + - [mysql, '8.0'] + - [mariadb, '10.2'] + - [mariadb, '10.3'] + - [mariadb, '10.4'] + - [mariadb, '10.5'] + - [mariadb, '10.6'] + - [mariadb, '10.7'] + + fail-fast: false + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + + continue-on-error: >- + ${{ + ( + ( + needs.pre-setup.outputs.release-requested == 'true' && + !toJSON(github.event.inputs.YOLO) + ) || + contains(matrix.py, '-dev') + ) && true || false + }} + + env: + MYSQL_ROOT_PASSWORD: rootpw + PY_COLORS: 1 + + services: + mysql: + image: "${{ join(matrix.db, ':') }}" + ports: + - 3306:3306 + volumes: + - "/tmp/run-${{ join(matrix.db, '-') }}/:/socket-mount/" + options: '--name=mysqld' + env: + MYSQL_ROOT_PASSWORD: rootpw + + steps: + - name: Setup Python ${{ matrix.py }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.py }} + + - name: Figure out if the interpreter ABI is stable + id: py-abi + run: | + from sys import version_info + is_stable_abi = version_info.releaselevel == 'final' + print( + '::set-output name=is-stable-abi::{is_stable_abi}'. + format(is_stable_abi=str(is_stable_abi).lower()) + ) + shell: python + + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + if: fromJSON(steps.py-abi.outputs.is-stable-abi) + id: calc-cache-key-py + run: | + from hashlib import sha512 + from sys import version + hash = sha512(version.encode()).hexdigest() + print('::set-output name=py-hash-key::{hash}'.format(hash=hash)) + shell: python + + - name: Get pip cache dir + if: fromJSON(steps.py-abi.outputs.is-stable-abi) + id: pip-cache-dir + run: >- + echo "::set-output name=dir::$(python -m pip cache dir)" + + - name: Set up pip cache + if: fromJSON(steps.py-abi.outputs.is-stable-abi) + uses: actions/cache@v3.0.2 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + needs.pre-setup.outputs.cache-key-files }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + + - name: Update pip + run: >- + python -m + pip install + --user + pip + + - name: Grab the source from Git + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.release-commitish }} + + - name: Remove aiomysql source to avoid accidentally using it + run: >- + rm -rf aiomysql + + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + - name: Install dependencies + run: >- + python -m + pip install + --user + --requirement requirements-dev.txt + + - name: Install previously built wheel + run: >- + python -m + pip install + --user + 'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' + + - name: >- + Log platform.platform() + run: >- + python -m platform + - name: >- + Log platform.version() + run: >- + python -c "import platform; + print(platform.version())" + - name: >- + Log platform.uname() + run: >- + python -c "import platform; + print(platform.uname())" + - name: >- + Log platform.release() + run: >- + python -c "import platform; + print(platform.release())" + - name: Log stdlib OpenSSL version + run: >- + python -c + "import ssl; print('\nOPENSSL_VERSION: ' + + ssl.OPENSSL_VERSION + '\nOPENSSL_VERSION_INFO: ' + + repr(ssl.OPENSSL_VERSION_INFO) + + '\nOPENSSL_VERSION_NUMBER: ' + + repr(ssl.OPENSSL_VERSION_NUMBER))" + + # this ensures our database is ready. typically by the time the preparations have completed its first start logic. + # unfortunately we need this hacky workaround as GitHub Actions service containers can't reference data from our repo. + - name: Prepare mysql + run: | + # ensure server is started up + while : + do + sleep 1 + mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break + done + + # inject tls configuration + docker container stop mysqld + docker container cp "${{ github.workspace }}/tests/ssl_resources/ssl" mysqld:/etc/mysql/ssl + docker container cp "${{ github.workspace }}/tests/ssl_resources/tls.cnf" mysqld:/etc/mysql/conf.d/aiomysql-tls.cnf + + # use custom socket path + # we need to ensure that the socket path is writable for the user running the DB process in the container + sudo chmod 0777 /tmp/run-${{ join(matrix.db, '-') }} + + # mysql 5.7 container overrides the socket path in /etc/mysql/mysql.conf.d/mysqld.cnf + if [ "${{ join(matrix.db, '-') }}" = "mysql-5.7" ] + then + docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/mysql.conf.d/zz-aiomysql-socket.cnf + else + docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/conf.d/aiomysql-socket.cnf + fi + + docker container start mysqld + + # ensure server is started up + while : + do + sleep 1 + mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break + done + + mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e "SET GLOBAL local_infile=on" + + - name: Run tests + run: | + # timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs + timeout --preserve-status --signal=INT --verbose 5m \ + pytest --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql --cov tests ./tests --mysql-unix-socket "unix-${{ join(matrix.db, '') }}=/tmp/run-${{ join(matrix.db, '-') }}/mysql.sock" --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306" + env: + PYTHONUNBUFFERED: 1 + timeout-minutes: 6 + + - name: Upload coverage + if: ${{ github.event_name != 'schedule' }} + uses: codecov/codecov-action@v3.1.0 + with: + file: ./coverage.xml + flags: "${{ matrix.os }}_${{ matrix.py }}_${{ join(matrix.db, '-') }}" + fail_ci_if_error: true + + check: # This job does nothing and is only used for the branch protection + if: always() + + needs: + - lint + - tests + + runs-on: ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@v1.1.0 + with: + jobs: ${{ toJSON(needs) }} + + publish-pypi: + name: Publish ๐Ÿ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} to PyPI + needs: + - check + - pre-setup # transitive, for accessing settings + if: >- + fromJSON(needs.pre-setup.outputs.release-requested) + runs-on: ubuntu-latest + + environment: + name: pypi + url: >- + https://pypi.org/project/aiomysql/${{ + needs.pre-setup.outputs.dist-version + }} + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: >- + Publish ๐Ÿ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} to PyPI + uses: pypa/gh-action-pypi-publish@v1.5.0 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + print_hash: true + + publish-testpypi: + name: Publish ๐Ÿ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI + needs: + - check + - pre-setup # transitive, for accessing settings + if: >- + fromJSON(needs.pre-setup.outputs.is-untagged-devel) + || fromJSON(needs.pre-setup.outputs.release-requested) + runs-on: ubuntu-latest + + environment: + name: testpypi + url: >- + https://test.pypi.org/project/aiomysql/${{ + needs.pre-setup.outputs.dist-version + }} + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: >- + Publish ๐Ÿ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} to TestPyPI + uses: pypa/gh-action-pypi-publish@v1.5.0 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + print_hash: true + + post-release-repo-update: + name: >- + Publish post-release Git tag + for ${{ needs.pre-setup.outputs.git-tag }} + needs: + - publish-pypi + - pre-setup # transitive, for accessing settings + runs-on: ubuntu-latest + + steps: + - name: Fetch the src snapshot + uses: actions/checkout@v3 + with: + fetch-depth: 1 + ref: ${{ github.event.inputs.release-commitish }} + - name: Setup git user as [bot] + uses: fregante/setup-git-user@6cef8bf084d00360a293e0cc3c56e1b45d6502b8 + + - name: >- + Tag the release in the local Git repo + as v${{ needs.pre-setup.outputs.git-tag }} + run: >- + git tag + -m '${{ needs.pre-setup.outputs.git-tag }}' + '${{ needs.pre-setup.outputs.git-tag }}' + -- + ${{ github.event.inputs.release-commitish }} + - name: >- + Push ${{ needs.pre-setup.outputs.git-tag }} tag corresponding + to the just published release back to GitHub + run: >- + git push --atomic origin '${{ needs.pre-setup.outputs.git-tag }}' + + publish-github-release: + name: >- + Publish a tag and GitHub release for + ${{ needs.pre-setup.outputs.git-tag }} + needs: + - post-release-repo-update + - pre-setup # transitive, for accessing settings + runs-on: ubuntu-latest + + permissions: + contents: write + discussions: write + + steps: + - name: Fetch the src snapshot + uses: actions/checkout@v3 + with: + fetch-depth: 1 + ref: ${{ github.event.inputs.release-commitish }} + + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + - name: >- + Publish a GitHub Release for + ${{ needs.pre-setup.outputs.git-tag }} + uses: ncipollo/release-action@58ae73b360456532aafd58ee170c045abbeaee37 + with: + artifacts: | + dist/${{ needs.pre-setup.outputs.sdist-artifact-name }} + dist/${{ needs.pre-setup.outputs.wheel-artifact-name }} + artifactContentType: raw # Because whl and tgz are of different types + # FIXME: Use Towncrier once it is integrated. + bodyFile: CHANGES.txt + discussionCategory: Announcements + name: ${{ needs.pre-setup.outputs.git-tag }} + tag: ${{ needs.pre-setup.outputs.git-tag }} diff --git a/.pyup.yml b/.pyup.yml deleted file mode 100644 index 75f97116..00000000 --- a/.pyup.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Label PRs with `deps-update` label -label_prs: deps-update - -schedule: every week diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..5fba73ba --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,25 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +build: + os: ubuntu-20.04 + tools: + python: "3.10" + +sphinx: + configuration: docs/conf.py + fail_on_warning: false # FIXME + +formats: +- pdf +- epub + +python: + install: + - requirements: requirements-dev.txt + - method: pip + path: . diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cc2076e9..00000000 --- a/.travis.yml +++ /dev/null @@ -1,68 +0,0 @@ -language: python - -python: - - 3.6 - - 3.7 - - 3.8 - -env: - matrix: - - PYTHONASYNCIODEBUG=1 - - PYTHONASYNCIODEBUG= - -services: - - docker - -matrix: - include: - - python: 3.6 - env: - - PYTHONASYNCIODEBUG= - - DB=mariadb - - DBTAG=5.5 - - python: 3.6 - env: - - PYTHONASYNCIODEBUG=1 - - DB=mariadb - - DBTAG=10.0 - addons: - mariadb: '10.0' - - python: 3.6 - env: - - PYTHONASYNCIODEBUG= - - DB=mariadb - - DBTAG=10.5 - - python: 3.6 - env: - - PYTHONASYNCIODEBUG= - - DB=mysql - - DBTAG=5.7 - - -#before_script: -# - "mysql -e 'SELECT VERSION()'" -# - "mysql -e 'DROP DATABASE IF EXISTS test_pymysql; create database test_pymysql DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'" -# - "mysql -e 'DROP DATABASE IF EXISTS test_pymysql2; create database test_pymysql2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'" - -install: - - pip install -Ur requirements-dev.txt - - pip install . - - pip install codecov - -deploy: - provider: pypi - user: aio-libs-bot - distributions: "sdist bdist_wheel" - password: - secure: G9vr3UOuK7tJifGEzO1kForcz+DCq1IdtIVKr/e4gPenvENinCFrRRWw206BQf1ba1+EjvRqc/yJFfZrgFUJlxgrtahvFmTCzKiLHCwr2vJsEYyr6JLWKRE81//RKTOykDWwvCAjk6sgV8lYtKL6R5sTtjzLJq8CKsLoKZ97yniPKmNu/+7IJvp+vSOA9gIL+GWbDTP8lmNDLWwphFxq6mm4WQ0VWqwDb0SF3FG/QPGDYU19mPLsLqgf1cxaBuRtb/epRDLLG70M9l9/aBPbtAHdbxY+O+/Fv5RPxfo2xFB7ry7yQEsIGrOwot/TxsZDRwRnfPm8N4OV9AfnKjiF5sBpidRR5kQr2pgFP2xq7LEROOPydMbY+YbSHCRBGCHWsHusjwCCL1veVTZ10EB9j3j0O9C6rAaF6Ssdlfq3kzhbWUItQfIZ/h7C0Z0ucVqFB8uBug7jNxT8hD3pR4ftM6Y94HY4BFOlkmSUH9u7owCeUoV9WQT/QAZHOLswRpp1wjsu2c0zKh3wiiuzCYJ64cD/BQW8rQMp0QiGqsNebqR+3L6yNLLSMpDWp3q/pnVbjsI/yepvpVbpp3PltSfZJL0uUfE0OR+xU67JP4npVS8B/A7aTARcM/ljx7IYNYYf/peXQ6UmBOrR2OgPnRPv5LfG9NGdEy0WpQKCVuSydxk= - on: - tags: true - repo: aio-libs/aiomysql - all_branches: true - python: 3.6 - -script: - - make cov - -after_success: - - codecov diff --git a/CHANGES.txt b/CHANGES.txt index d0f6b604..c717378f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,65 @@ Changes ------- +0.1.1 (2022-05-08) +^^^^^^^^^^^^^^^^^^ + +* Fix SSL connection handshake charset not respecting client configuration #776 + +0.1.0 (2022-04-11) +^^^^^^^^^^^^^^^^^^ + +* Don't send sys.argv[0] as program_name to MySQL server by default #620 + +* Allow running process as anonymous uid #587 + +* Fix timed out MySQL 8.0 connections raising InternalError rather than OperationalError #660 + +* Fix timed out MySQL 8.0 connections being returned from Pool #660 + +* Ensure connections are properly closed before raising an OperationalError when the server connection is lost #660 + +* Ensure connections are properly closed before raising an InternalError when packet sequence numbers are out of sync #660 + +* Unix sockets are now internally considered secure, allowing sha256_password and caching_sha2_password auth methods to be used #695 + +* Test suite now also tests unix socket connections #696 + +* Fix SSCursor raising InternalError when last result was not fully retrieved #635 + +* Remove deprecated no_delay argument #702 + +* Support PyMySQL up to version 1.0.2 #643 + +* Bump minimal PyMySQL version to 1.0.0 #713 + +* Align % formatting in Cursor.executemany() with Cursor.execute(), literal % now need to be doubled in Cursor.executemany() #714 + +* Fixed unlimited Pool size not working, this is now working as documented by passing maxsize=0 to create_pool #119 + +* Added Pool.closed property as present in aiopg #463 + +* Fixed SQLAlchemy connection context iterator #410 + +* Fix error packet handling for SSCursor #428 + +* Required python version is now properly documented in python_requires instead of failing on setup.py execution #731 + +* Add rsa extras_require depending on PyMySQL[rsa] #557 + +* Migrate to PEP 517 build system #746 + +* Self-reported `__version__` now returns version generated by `setuptools-scm` during build, otherwise `'unknown'` #748 + +* Fix SSCursor raising query timeout error on wrong query #428 + + +0.0.22 (2021-11-14) +^^^^^^^^^^^^^^^^^^^ + +* Support python 3.10 #505 + + 0.0.21 (2020-11-26) ^^^^^^^^^^^^^^^^^^^ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8d0b4e0b..ec2b3f9c 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -51,25 +51,33 @@ We expect you to use a python virtual environment to run our tests. There are several ways to make a virtual environment. -If you like to use *virtualenv* please run:: +If you like to use *virtualenv* please run: + +.. code-block:: sh $ cd aiomysql - $ virtualenv --python=`which python3` venv + $ virtualenv --python="$(which python3)" venv + +For standard python *venv*: -For standard python *venv*:: +.. code-block:: sh $ cd aiomysql $ python3 -m venv venv -For *virtualenvwrapper*:: +For *virtualenvwrapper*: + +.. code-block:: sh $ cd aiomysql - $ mkvirtualenv --python=`which python3` aiomysql + $ mkvirtualenv --python="$(which python3)" aiomysql There are other tools like *pyvenv* but you know the rule of thumb now: create a python3 virtual environment and activate it. -After that please install libraries required for development:: +After that please install libraries required for development: + +.. code-block:: sh $ pip install -r requirements-dev.txt @@ -83,7 +91,7 @@ use this values by default. But you always can override host/port, user and password in `aiomysql/tests/base.py` file or install corresponding environment variables. Tests require two databases to be created before running suit: -:: +.. code-block:: sh $ mysql -u root mysql> CREATE DATABASE test_pymysql DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; @@ -94,7 +102,9 @@ Run aiomysql test suite ----------------------- After all the preconditions are met you can run tests typing the next -command:: +command: + +.. code-block:: sh $ make test @@ -113,7 +123,9 @@ Tests coverage We are trying hard to have good test coverage; please don't make it worse. -Use:: +Use: + +.. code-block:: sh $ make cov @@ -130,7 +142,9 @@ Documentation We encourage documentation improvements. -Please before making a Pull Request about documentation changes run:: +Please before making a Pull Request about documentation changes run: + +.. code-block:: sh $ make doc diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 95727af4..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include LICENSE -include CHANGES.txt -include README.rst -graft aiomysql -global-exclude *.pyc *.swp diff --git a/Makefile b/Makefile index 542b2c20..798790cf 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,10 @@ FLAGS= checkrst: - python setup.py check --restructuredtext + python -m twine check --strict dist/* -pyroma: - pyroma -d . - -flake:checkrst pyroma +flake:checkrst flake8 aiomysql tests examples test: flake @@ -47,7 +44,16 @@ start_mysql: stop_mysql: docker-compose -f docker-compose.yml stop mysql +# TODO: this depends on aiomysql being installed, e.g. in a venv. +# TODO: maybe this can be solved better. doc: + @echo "----------------------------------------------------------------" + @echo "Doc builds require installing the aiomysql package in the" + @echo "environment. Make sure you've installed your current dev version" + @echo "into your environment, e.g. using venv, then run this command in" + @echo "the virtual environment." + @echo "----------------------------------------------------------------" + git fetch --tags --all make -C docs html @echo "open file://`pwd`/docs/_build/html/index.html" diff --git a/README.rst b/README.rst index 388252e1..b5e52c1d 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ aiomysql ======== -.. image:: https://travis-ci.com/aio-libs/aiomysql.svg?branch=master - :target: https://travis-ci.com/aio-libs/aiomysql +.. image:: https://github.com/aio-libs/aiomysql/actions/workflows/ci-cd.yml/badge.svg?branch=master + :target: https://github.com/aio-libs/aiomysql/actions/workflows/ci-cd.yml .. image:: https://codecov.io/gh/aio-libs/aiomysql/branch/master/graph/badge.svg :target: https://codecov.io/gh/aio-libs/aiomysql :alt: Code coverage @@ -29,12 +29,6 @@ Documentation ------------- https://aiomysql.readthedocs.io/ - -Mailing List ------------- -https://groups.google.com/forum/#!forum/aio-libs - - Basic Example ------------- @@ -110,7 +104,7 @@ for aiopg_ user.: Requirements ------------ -* Python_ 3.5.3+ +* Python_ 3.7+ * PyMySQL_ diff --git a/aiomysql/.gitignore b/aiomysql/.gitignore new file mode 100644 index 00000000..91cb299f --- /dev/null +++ b/aiomysql/.gitignore @@ -0,0 +1 @@ +/_scm_version.py diff --git a/aiomysql/__init__.py b/aiomysql/__init__.py index f5e74aee..a367fcd2 100644 --- a/aiomysql/__init__.py +++ b/aiomysql/__init__.py @@ -32,8 +32,9 @@ from .connection import Connection, connect from .cursors import Cursor, SSCursor, DictCursor, SSDictCursor from .pool import create_pool, Pool +from ._version import version -__version__ = '0.0.21' +__version__ = version __all__ = [ diff --git a/aiomysql/_scm_version.pyi b/aiomysql/_scm_version.pyi new file mode 100644 index 00000000..deb2e36a --- /dev/null +++ b/aiomysql/_scm_version.pyi @@ -0,0 +1,3 @@ +# This stub file is necessary because `_scm_version.py` +# autogenerated on build and absent on mypy checks time +version: str diff --git a/aiomysql/_version.py b/aiomysql/_version.py new file mode 100644 index 00000000..80af3cd2 --- /dev/null +++ b/aiomysql/_version.py @@ -0,0 +1,4 @@ +try: + from ._scm_version import version +except ImportError: + version = "unknown" diff --git a/aiomysql/connection.py b/aiomysql/connection.py index f5866171..3dce75a7 100644 --- a/aiomysql/connection.py +++ b/aiomysql/connection.py @@ -15,8 +15,8 @@ from pymysql.constants import SERVER_STATUS from pymysql.constants import CLIENT from pymysql.constants import COMMAND +from pymysql.constants import CR from pymysql.constants import FIELD_TYPE -from pymysql.util import byte2int, int2byte from pymysql.converters import (escape_item, encoders, decoders, escape_string, escape_bytes_prefixed, through) from pymysql.err import (Warning, Error, @@ -28,21 +28,21 @@ from pymysql.connections import TEXT_TYPES, MAX_PACKET_LEN, DEFAULT_CHARSET from pymysql.connections import _auth -from pymysql.connections import pack_int24 - from pymysql.connections import MysqlPacket from pymysql.connections import FieldDescriptorPacket from pymysql.connections import EOFPacketWrapper from pymysql.connections import OKPacketWrapper from pymysql.connections import LoadLocalPacketWrapper -from pymysql.connections import lenenc_int # from aiomysql.utils import _convert_to_str from .cursors import Cursor -from .utils import _ConnectionContextManager, _ContextManager +from .utils import _pack_int24, _lenenc_int, _ConnectionContextManager, _ContextManager from .log import logger -DEFAULT_USER = getpass.getuser() +try: + DEFAULT_USER = getpass.getuser() +except KeyError: + DEFAULT_USER = "unknown" def connect(host="localhost", user=None, password="", @@ -51,7 +51,7 @@ def connect(host="localhost", user=None, password="", read_default_file=None, conv=decoders, use_unicode=None, client_flag=0, cursorclass=Cursor, init_command=None, connect_timeout=None, read_default_group=None, - no_delay=None, autocommit=False, echo=False, + autocommit=False, echo=False, local_infile=False, loop=None, ssl=None, auth_plugin='', program_name='', server_public_key=None): """See connections.Connection.__init__() for information about @@ -64,7 +64,7 @@ def connect(host="localhost", user=None, password="", init_command=init_command, connect_timeout=connect_timeout, read_default_group=read_default_group, - no_delay=no_delay, autocommit=autocommit, echo=echo, + autocommit=autocommit, echo=echo, local_infile=local_infile, loop=loop, ssl=ssl, auth_plugin=auth_plugin, program_name=program_name) return _ConnectionContextManager(coro) @@ -76,6 +76,57 @@ async def _connect(*args, **kwargs): return conn +async def _open_connection(host=None, port=None, **kwds): + """This is based on asyncio.open_connection, allowing us to use a custom + StreamReader. + + `limit` arg has been removed as we don't currently use it. + """ + loop = asyncio.events.get_running_loop() + reader = _StreamReader(loop=loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=loop) + transport, _ = await loop.create_connection( + lambda: protocol, host, port, **kwds) + writer = asyncio.StreamWriter(transport, protocol, reader, loop) + return reader, writer + + +async def _open_unix_connection(path=None, **kwds): + """This is based on asyncio.open_unix_connection, allowing us to use a custom + StreamReader. + + `limit` arg has been removed as we don't currently use it. + """ + loop = asyncio.events.get_running_loop() + + reader = _StreamReader(loop=loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=loop) + transport, _ = await loop.create_unix_connection( + lambda: protocol, path, **kwds) + writer = asyncio.StreamWriter(transport, protocol, reader, loop) + return reader, writer + + +class _StreamReader(asyncio.StreamReader): + """This StreamReader exposes whether EOF was received, allowing us to + discard the associated connection instead of returning it from the pool + when checking free connections in Pool._fill_free_pool(). + + `limit` arg has been removed as we don't currently use it. + """ + def __init__(self, loop=None): + self._eof_received = False + super().__init__(loop=loop) + + def feed_eof(self) -> None: + self._eof_received = True + super().feed_eof() + + @property + def eof_received(self): + return self._eof_received + + class Connection: """Representation of a socket with a mysql server. @@ -89,7 +140,7 @@ def __init__(self, host="localhost", user=None, password="", read_default_file=None, conv=decoders, use_unicode=None, client_flag=0, cursorclass=Cursor, init_command=None, connect_timeout=None, read_default_group=None, - no_delay=None, autocommit=False, echo=False, + autocommit=False, echo=False, local_infile=False, loop=None, ssl=None, auth_plugin='', program_name='', server_public_key=None): """ @@ -120,7 +171,6 @@ def __init__(self, host="localhost", user=None, password="", when connecting. :param read_default_group: Group to read from in the configuration file. - :param no_delay: Disable Nagle's algorithm on the socket :param autocommit: Autocommit mode. None means use server default. (default: False) :param local_infile: boolean to enable the use of LOAD DATA LOCAL @@ -131,7 +181,7 @@ def __init__(self, host="localhost", user=None, password="", when using IAM authentication with Amazon RDS. (default: Server Default) :param program_name: Program name string to provide when - handshaking with MySQL. (default: sys.argv[0]) + handshaking with MySQL. (omitted by default) :param server_public_key: SHA256 authentication plugin public key value. :param loop: asyncio loop @@ -156,24 +206,17 @@ def __init__(self, host="localhost", user=None, password="", port = int(_config("port", fallback=port)) charset = _config("default-character-set", fallback=charset) - # pymysql port - if no_delay is not None: - warnings.warn("no_delay option is deprecated", DeprecationWarning) - no_delay = bool(no_delay) - else: - no_delay = True - self._host = host self._port = port self._user = user or DEFAULT_USER self._password = password or "" self._db = db - self._no_delay = no_delay self._echo = echo self._last_usage = self._loop.time() self._client_auth_plugin = auth_plugin self._server_auth_plugin = "" self._auth_plugin_used = "" + self._secure = False self.server_public_key = server_public_key self.salt = None @@ -185,8 +228,6 @@ def __init__(self, host="localhost", user=None, password="", } if program_name: self._connect_attrs["program_name"] = program_name - elif sys.argv: - self._connect_attrs["program_name"] = sys.argv[0] self._unix_socket = unix_socket if charset: @@ -304,7 +345,7 @@ async def ensure_closed(self): if self._writer is None: # connection has been closed return - send_data = struct.pack(' -1 and self._loop.time() - conn.last_usage > self._recycle): self._free.pop() @@ -174,7 +189,7 @@ async def _fill_free_pool(self, override_min): if self._free: return - if override_min and self.size < self.maxsize: + if override_min and (not self.maxsize or self.size < self.maxsize): self._acquiring += 1 try: conn = await connect(echo=self._echo, loop=self._loop, diff --git a/aiomysql/utils.py b/aiomysql/utils.py index 9b74b9df..74ad99a7 100644 --- a/aiomysql/utils.py +++ b/aiomysql/utils.py @@ -1,5 +1,31 @@ from collections.abc import Coroutine +import struct + + +def _pack_int24(n): + return struct.pack("` returns all rows of a query result set:: - yield from cursor.execute("SELECT * FROM test;") - r = yield from cursor.fetchall() + await cursor.execute("SELECT * FROM test;") + r = await cursor.fetchall() print(r) # [(1, 100, "abc'def"), (2, None, 'dada'), (3, 42, 'bar')] @@ -274,7 +273,7 @@ Cursor probably to catch both exceptions in your code:: try: - yield from cur.scroll(1000 * 1000) + await cur.scroll(1000 * 1000) except (ProgrammingError, IndexError), exc: deal_with_it(exc) @@ -292,21 +291,20 @@ Cursor loop = asyncio.get_event_loop() - @asyncio.coroutine - def test_example(): - conn = yield from aiomysql.connect(host='127.0.0.1', port=3306, - user='root', password='', - db='mysql', loop=loop) + async def test_example(): + conn = await aiomysql.connect(host='127.0.0.1', port=3306, + user='root', password='', + db='mysql', loop=loop) # create dict cursor - cursor = yield from conn.cursor(aiomysql.DictCursor) + cursor = await conn.cursor(aiomysql.DictCursor) # execute sql query - yield from cursor.execute( + await cursor.execute( "SELECT * from people where name='bob'") # fetch all results - r = yield from cursor.fetchone() + r = await cursor.fetchone() print(r) # {'age': 20, 'DOB': datetime.datetime(1990, 2, 6, 23, 4, 56), # 'name': 'bob'} @@ -332,21 +330,20 @@ Cursor loop = asyncio.get_event_loop() - @asyncio.coroutine - def test_example(): - conn = yield from aiomysql.connect(host='127.0.0.1', port=3306, - user='root', password='', - db='mysql', loop=loop) + async def test_example(): + conn = await aiomysql.connect(host='127.0.0.1', port=3306, + user='root', password='', + db='mysql', loop=loop) # create your dict cursor - cursor = yield from conn.cursor(AttrDictCursor) + cursor = await conn.cursor(AttrDictCursor) # execute sql query - yield from cursor.execute( + await cursor.execute( "SELECT * from people where name='bob'") # fetch all results - r = yield from cursor.fetchone() + r = await cursor.fetchone() print(r) # {'age': 20, 'DOB': datetime.datetime(1990, 2, 6, 23, 4, 56), # 'name': 'bob'} diff --git a/docs/index.rst b/docs/index.rst index 684d08c1..8dc92185 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -96,16 +96,16 @@ Please feel free to file an issue on `bug tracker `_ if you have found a bug or have some suggestion for library improvement. -The library uses `Travis `_ for -Continious Integration and `Coveralls -`_ for +The library uses `GitHub Actions +`_ for Continuous Integration +and `Codecov `_ for coverage reports. Dependencies ------------ -- Python 3.5.3+ +- Python 3.7+ - :term:`PyMySQL` - aiomysql.sa requires :term:`sqlalchemy`. diff --git a/docs/pool.rst b/docs/pool.rst index c8323db0..b2683715 100644 --- a/docs/pool.rst +++ b/docs/pool.rst @@ -14,20 +14,19 @@ The basic usage is:: loop = asyncio.get_event_loop() - @asyncio.coroutine - def go(): - pool = yield from aiomysql.create_pool(host='127.0.0.1', port=3306, - user='root', password='', - db='mysql', loop=loop, autocommit=False) - - with (yield from pool) as conn: - cur = yield from conn.cursor() - yield from cur.execute("SELECT 10") + async def go(): + pool = await aiomysql.create_pool(host='127.0.0.1', port=3306, + user='root', password='', + db='mysql', loop=loop, autocommit=False) + + async with pool.acquire() as conn: + cur = await conn.cursor() + await cur.execute("SELECT 10") # print(cur.description) - (r,) = yield from cur.fetchone() + (r,) = await cur.fetchone() assert r == 10 pool.close() - yield from pool.wait_closed() + await pool.wait_closed() loop.run_until_complete(go()) @@ -45,6 +44,9 @@ The basic usage is:: :param kwargs: The function accepts all parameters that :func:`aiomysql.connect` does plus optional keyword-only parameters *loop*, *minsize*, *maxsize*. + :param float pool_recycle: number of seconds after which connection is + recycled, helps to deal with stale connections in pool, default + value is -1, means recycling logic is disabled. :returns: :class:`Pool` instance. @@ -62,8 +64,8 @@ The basic usage is:: The most important way to use it is getting connection in *with statement*:: - with (yield from pool) as conn: - cur = yield from conn.cursor() + async with pool.acquire() as conn: + cur = await conn.cursor() See also :meth:`Pool.acquire` and :meth:`Pool.release` for acquring diff --git a/docs/sa.rst b/docs/sa.rst index 339e2f81..94c6b1dd 100644 --- a/docs/sa.rst +++ b/docs/sa.rst @@ -30,28 +30,36 @@ Example:: metadata = sa.MetaData() - tbl = sa.Table('tbl', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('val', sa.String(255))) + tbl = sa.Table( + "tbl", + metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("val", sa.String(255)), + ) - @asyncio.coroutine - def go(): - engine = yield from create_engine(user='root', - db='test_pymysql', - host='127.0.0.1', - password='') + async def go(): + engine = await create_engine( + user="root", + db="test_pymysql", + host="127.0.0.1", + password="", + ) - with (yield from engine) as conn: - yield from conn.execute(tbl.insert().values(val='abc')) + async with engine.acquire() as conn: + async with conn.begin() as transaction: + await conn.execute(tbl.insert().values(val="abc")) + await transaction.commit() - res = yield from conn.execute(tbl.select()) - for row in res: - print(row.id, row.val) + res = await conn.execute(tbl.select()) + async for row in res: + print(row.id, row.val) - await conn.commit() + engine.close() + await engine.wait_closed() - asyncio.get_event_loop().run_until_complete(go()) + + asyncio.run(go()) So you can execute SQL query built by @@ -202,26 +210,26 @@ Connection to be used in the execution. Typically, the format is either a dictionary passed to \*multiparams:: - yield from conn.execute( + await conn.execute( table.insert(), {"id":1, "value":"v1"} ) ...or individual key/values interpreted by \**params:: - yield from conn.execute( + await conn.execute( table.insert(), id=1, value="v1" ) In the case that a plain SQL string is passed, a tuple or individual values in \*multiparams may be passed:: - yield from conn.execute( + await conn.execute( "INSERT INTO table (id, value) VALUES (%d, %s)", (1, "v1") ) - yield from conn.execute( + await conn.execute( "INSERT INTO table (id, value) VALUES (%s, %s)", 1, "v1" ) @@ -257,10 +265,10 @@ Connection an emulated transaction within the scope of the enclosing transaction, that is:: - trans = yield from conn.begin() # outermost transaction - trans2 = yield from conn.begin() # "inner" - yield from trans2.commit() # does nothing - yield from trans.commit() # actually commits + trans = await conn.begin() # outermost transaction + trans2 = await conn.begin() # "inner" + await trans2.commit() # does nothing + await trans.commit() # actually commits Calls to :meth:`.Transaction.commit` only have an effect when invoked via the outermost :class:`.Transaction` object, though the @@ -364,7 +372,7 @@ ResultProxy case-sensitive column name, or by :class:`sqlalchemy.schema.Column`` object. e.g.:: - for row in (yield from conn.execute(...)): + async for row in conn.execute(...): col1 = row[0] # access via integer position col2 = row['col2'] # access via name col3 = row[mytable.c.mycol] # access via Column object. @@ -531,14 +539,14 @@ Transaction objects calling the :meth:`SAConnection.begin` method of :class:`SAConnection`:: - with (yield from engine) as conn: - trans = yield from conn.begin() + async with engine.acquire() as conn: + trans = await conn.begin() try: - yield from conn.execute("insert into x (a, b) values (1, 2)") + await conn.execute("insert into x (a, b) values (1, 2)") except Exception: - yield from trans.rollback() + await trans.rollback() else: - yield from trans.commit() + await trans.commit() The object provides :meth:`.rollback` and :meth:`.commit` methods in order to control transaction boundaries. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 17a4d2d2..c8c04d11 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -6,7 +6,7 @@ Tutorial Python database access modules all have similar interfaces, described by the :term:`DBAPI`. Most relational databases use the same synchronous interface, *aiomysql* tries to provide same api you just need -to use ``yield from conn.f()`` instead of just call ``conn.f()`` for +to use ``await conn.f()`` instead of just call ``conn.f()`` for every method. Installation @@ -29,18 +29,17 @@ Lets start from basic example:: loop = asyncio.get_event_loop() - @asyncio.coroutine - def test_example(): - conn = yield from aiomysql.connect(host='127.0.0.1', port=3306, + async def test_example(): + conn = await aiomysql.connect(host='127.0.0.1', port=3306, user='root', password='', db='mysql', loop=loop) - cur = yield from conn.cursor() - yield from cur.execute("SELECT Host,User FROM user") + cur = await conn.cursor() + await cur.execute("SELECT Host,User FROM user") print(cur.description) - r = yield from cur.fetchall() + r = await cur.fetchall() print(r) - yield from cur.close() + await cur.close() conn.close() loop.run_until_complete(test_example()) @@ -60,10 +59,10 @@ processing statements. Example uses cursor to issue a ``SELECT Host,User FROM user;`` statement, which returns a list of `host` and `user` from :term:`MySQL` system table ``user``:: - cur = yield from conn.cursor() - yield from cur.execute("SELECT Host,User FROM user") + cur = await conn.cursor() + await cur.execute("SELECT Host,User FROM user") print(cur.description) - r = yield from cur.fetchall() + r = await cur.fetchall() The cursor object's :meth:`Cursor.execute()` method sends the query the server and :meth:`Cursor.fetchall()` retrieves rows. @@ -72,7 +71,7 @@ Finally, the script invokes :meth:`Cursor.close()` coroutine and connection object's :meth:`Connection.close()` method to disconnect from the server:: - yield from cur.close() + await cur.close() conn.close() After that, ``conn`` becomes invalid and should not be used to access the diff --git a/examples/example_executemany.py b/examples/example_executemany.py index 30ff1a30..087b456e 100644 --- a/examples/example_executemany.py +++ b/examples/example_executemany.py @@ -22,7 +22,7 @@ async def test_example_executemany(loop): await cur.execute("INSERT INTO music_style VALUES(3,'power metal');") await conn.commit() - # insert 3 row by one long query using *executemane* method + # insert 3 row by one long query using *executemany* method data = [(4, 'gothic metal'), (5, 'doom metal'), (6, 'post metal')] await cur.executemany( "INSERT INTO music_style (id, name)" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..4e903b7d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = [ + # Essentials + "setuptools >= 42", + + # Plugins + "setuptools_scm[toml] >= 6.4", + "setuptools_scm_git_archive >= 1.1", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "aiomysql/_scm_version.py" diff --git a/requirements-dev.txt b/requirements-dev.txt index 70e6fd13..716749a7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,14 +1,12 @@ -coverage>=4.5.1,<=5.1 -flake8>=3.5.0,<=3.7.9 -ipdb>=0.11,<=0.13.2 -ipython>=7.0.1,<=7.13.0 -pytest>=3.9.1,<=5.4.1 -pytest-cov>=2.6.0,<=2.8.1 -pytest-sugar>=0.9.1,<=0.9.3 -PyMySQL>=0.9,<=0.9.3 -docker>=3.5.1,<=4.2.0 -sphinx>=1.8.1, <=3.0.3 -sphinxcontrib-asyncio==0.2.0 -sqlalchemy>1.2.12,<=1.3.16 -uvloop>=0.11.2,<=0.14.0; python_version >= '3.5' -pyroma==2.6 +coverage==6.3.2 +flake8==4.0.1 +ipdb==0.13.9 +pytest==7.1.2 +pytest-cov==3.0.0 +pytest-sugar==0.9.4 +PyMySQL==1.0.2 +sphinx>=1.8.1, <4.5.1 +sphinxcontrib-asyncio==0.3.0 +SQLAlchemy==1.3.24 +uvloop==0.16.0; python_version < '3.11' +twine==4.0.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..5b8c7d34 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,59 @@ +[metadata] +name = aiomysql +version = attr: aiomysql.__version__ +url = https://github.com/aio-libs/aiomysql +download_url = https://pypi.python.org/pypi/aiomysql +project_urls = + CI: GitHub = https://github.com/aio-libs/aiomysql/actions + Docs: RTD = https://aiomysql.readthedocs.io/ + GitHub: repo = https://github.com/aio-libs/aiomysql + GitHub: issues = https://github.com/aio-libs/aiomysql/issues + GitHub: discussions = https://github.com/aio-libs/aiomysql/discussions +description = MySQL driver for asyncio. +long_description = file: README.rst, CHANGES.txt +long_description_content_type = text/x-rst +author = Nikolay Novik +author_email = nickolainovik@gmail.com +classifiers = + License :: OSI Approved :: MIT License + Intended Audience :: Developers + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Operating System :: POSIX + Environment :: Web Environment + Development Status :: 3 - Alpha + Topic :: Database + Topic :: Database :: Front-Ends + Framework :: AsyncIO +license = MIT +keywords = + mysql + mariadb + asyncio + aiomysql +platforms = + POSIX + +[options] +python_requires = >=3.7 +include_package_data = True + +packages = find: + +# runtime requirements +install_requires = + PyMySQL>=1.0 + +[options.extras_require] +sa = + sqlalchemy>=1.0,<1.4 +rsa = + PyMySQL[rsa]>=1.0 + +[options.packages.find] +exclude = + tests + tests.* diff --git a/setup.py b/setup.py deleted file mode 100755 index 290c2075..00000000 --- a/setup.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import re -import sys -from setuptools import setup, find_packages - - -install_requires = ['PyMySQL>=0.9,<=0.9.3'] - -PY_VER = sys.version_info - - -if not PY_VER >= (3, 5, 3): - raise RuntimeError("aiomysql doesn't support Python earlier than 3.5.3") - - -def read(f): - return open(os.path.join(os.path.dirname(__file__), f)).read().strip() - - -extras_require = {'sa': ['sqlalchemy>=1.0'], } - - -def read_version(): - regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+)'") - init_py = os.path.join(os.path.dirname(__file__), - 'aiomysql', '__init__.py') - with open(init_py) as f: - for line in f: - match = regexp.match(line) - if match is not None: - return match.group(1) - else: - raise RuntimeError('Cannot find version in aiomysql/__init__.py') - - -classifiers = [ - 'License :: OSI Approved :: MIT License', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Operating System :: POSIX', - 'Environment :: Web Environment', - 'Development Status :: 3 - Alpha', - 'Topic :: Database', - 'Topic :: Database :: Front-Ends', - 'Framework :: AsyncIO', -] - -keywords = ["mysql", "asyncio", "aiomysql"] - - -setup(name='aiomysql', - version=read_version(), - description=('MySQL driver for asyncio.'), - long_description='\n\n'.join((read('README.rst'), read('CHANGES.txt'))), - classifiers=classifiers, - platforms=['POSIX'], - author="Nikolay Novik", - author_email="nickolainovik@gmail.com", - url='https://github.com/aio-libs/aiomysql', - download_url='https://pypi.python.org/pypi/aiomysql', - license='MIT', - packages=find_packages(exclude=['tests', 'tests.*']), - install_requires=install_requires, - extras_require=extras_require, - keywords=keywords, - include_package_data=True) diff --git a/tests/_testutils.py b/tests/_testutils.py index 3ccb4c8a..c3761a54 100644 --- a/tests/_testutils.py +++ b/tests/_testutils.py @@ -1,21 +1,6 @@ import asyncio import unittest -from functools import wraps - - -def run_until_complete(fun): - if not asyncio.iscoroutinefunction(fun): - fun = asyncio.coroutine(fun) - - @wraps(fun) - def wrapper(test, *args, **kw): - loop = test.loop - ret = loop.run_until_complete( - asyncio.wait_for(fun(test, *args, **kw), 15, loop=loop)) - return ret - return wrapper - class BaseTest(unittest.TestCase): """Base test case for unittests. diff --git a/tests/base.py b/tests/base.py index ba306a87..be87f535 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,4 +1,3 @@ -import asyncio import os import aiomysql from tests._testutils import BaseTest @@ -6,24 +5,23 @@ class AIOPyMySQLTestCase(BaseTest): - @asyncio.coroutine - def _connect_all(self): - conn1 = yield from aiomysql.connect(loop=self.loop, host=self.host, - port=self.port, user=self.user, - db=self.db, - password=self.password, - use_unicode=True, echo=True) - conn2 = yield from aiomysql.connect(loop=self.loop, host=self.host, - port=self.port, user=self.user, - db=self.other_db, - password=self.password, - use_unicode=False, echo=True) - conn3 = yield from aiomysql.connect(loop=self.loop, host=self.host, - port=self.port, user=self.user, - db=self.db, - password=self.password, - use_unicode=True, echo=True, - local_infile=True) + async def _connect_all(self): + conn1 = await aiomysql.connect(loop=self.loop, host=self.host, + port=self.port, user=self.user, + db=self.db, + password=self.password, + use_unicode=True, echo=True) + conn2 = await aiomysql.connect(loop=self.loop, host=self.host, + port=self.port, user=self.user, + db=self.other_db, + password=self.password, + use_unicode=False, echo=True) + conn3 = await aiomysql.connect(loop=self.loop, host=self.host, + port=self.port, user=self.user, + db=self.db, + password=self.password, + use_unicode=True, echo=True, + local_infile=True) self.connections = [conn1, conn2, conn3] @@ -45,9 +43,9 @@ def tearDown(self): self.doCleanups() super(AIOPyMySQLTestCase, self).tearDown() - @asyncio.coroutine - def connect(self, host=None, user=None, password=None, - db=None, use_unicode=True, no_delay=None, port=None, **kwargs): + async def connect(self, host=None, user=None, password=None, + db=None, use_unicode=True, port=None, + **kwargs): if host is None: host = self.host if user is None: @@ -58,18 +56,17 @@ def connect(self, host=None, user=None, password=None, db = self.db if port is None: port = self.port - conn = yield from aiomysql.connect(loop=self.loop, host=host, - user=user, password=password, - db=db, use_unicode=use_unicode, - no_delay=no_delay, port=port, - **kwargs) + conn = await aiomysql.connect(loop=self.loop, host=host, + user=user, password=password, + db=db, use_unicode=use_unicode, + port=port, + **kwargs) self.addCleanup(conn.close) return conn - @asyncio.coroutine - def create_pool(self, host=None, user=None, password=None, - db=None, use_unicode=True, no_delay=None, - port=None, **kwargs): + async def create_pool(self, host=None, user=None, password=None, + db=None, use_unicode=True, + port=None, **kwargs): if host is None: host = self.host if user is None: @@ -80,10 +77,10 @@ def create_pool(self, host=None, user=None, password=None, db = self.db if port is None: port = self.port - pool = yield from aiomysql.create_pool(loop=self.loop, host=host, - user=user, password=password, - db=db, use_unicode=use_unicode, - no_delay=no_delay, port=port, - **kwargs) + pool = await aiomysql.create_pool(loop=self.loop, host=host, + user=user, password=password, + db=db, use_unicode=use_unicode, + port=port, + **kwargs) self.addCleanup(pool.close) return pool diff --git a/tests/conftest.py b/tests/conftest.py index 5381498d..5420b456 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,25 +1,23 @@ import asyncio import gc import os +import re import ssl -import socket import sys -import time -import uuid - -from distutils.version import StrictVersion -from docker import APIClient import aiomysql import pymysql import pytest -PY_35 = sys.version_info >= (3, 5) -if PY_35: - import uvloop -else: +# version gate can be removed once uvloop supports python 3.11 +# https://github.com/MagicStack/uvloop/issues/450 +# https://github.com/MagicStack/uvloop/pull/459 +PY_311 = sys.version_info >= (3, 11) +if PY_311: uvloop = None +else: + import uvloop @pytest.fixture @@ -34,33 +32,59 @@ def disable_gc(): gc.enable() -@pytest.fixture(scope='session') -def unused_port(): - def f(): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(('127.0.0.1', 0)) - return s.getsockname()[1] - return f - - def pytest_generate_tests(metafunc): if 'loop_type' in metafunc.fixturenames: loop_type = ['asyncio', 'uvloop'] if uvloop else ['asyncio'] metafunc.parametrize("loop_type", loop_type) + if "mysql_address" in metafunc.fixturenames: + mysql_addresses = [] + ids = [] + + opt_mysql_unix_socket = \ + list(metafunc.config.getoption("mysql_unix_socket")) + for i in range(len(opt_mysql_unix_socket)): + if "=" in opt_mysql_unix_socket[i]: + label, path = opt_mysql_unix_socket[i].split("=", 1) + mysql_addresses.append(path) + ids.append(label) + else: + mysql_addresses.append(opt_mysql_unix_socket[i]) + ids.append("unix{}".format(i)) + + opt_mysql_address = list(metafunc.config.getoption("mysql_address")) + for i in range(len(opt_mysql_address)): + if "=" in opt_mysql_address[i]: + label, addr = opt_mysql_address[i].split("=", 1) + ids.append(label) + else: + addr = opt_mysql_address[i] + ids.append("tcp{}".format(i)) + + if ":" in addr: + addr = addr.split(":", 1) + mysql_addresses.append((addr[0], int(addr[1]))) + else: + mysql_addresses.append((addr, 3306)) + + # default to connecting to localhost + if len(mysql_addresses) == 0: + mysql_addresses = [("127.0.0.1", 3306)] + ids = ["tcp-local"] + + assert len(mysql_addresses) == len(set(mysql_addresses)), \ + "mysql targets are not unique" + assert len(ids) == len(set(ids)), \ + "mysql target names are not unique" + + metafunc.parametrize("mysql_address", + mysql_addresses, + ids=ids, + scope="session", + ) -# This is here unless someone fixes the generate_tests bit -@pytest.yield_fixture(scope='session') -def mysql_tag(): - return os.environ.get('DBTAG', '10.5') - -@pytest.yield_fixture(scope='session') -def mysql_image(): - return os.environ.get('DB', 'mariadb') - - -@pytest.yield_fixture +@pytest.fixture def loop(request, loop_type): loop = asyncio.new_event_loop() asyncio.set_event_loop(None) @@ -85,7 +109,7 @@ def pytest_pycollect_makeitem(collector, name, obj): if collector.funcnamefilter(name): if not callable(obj): return - item = pytest.Function(name, parent=collector) + item = pytest.Function.from_parent(collector, name=name) if 'run_loop' in item.keywords: return list(collector._genfunctions(name, obj)) @@ -111,26 +135,36 @@ def pytest_runtest_setup(item): item.fixturenames.append('loop') -def pytest_ignore_collect(path, config): - if 'pep492' in str(path): - if sys.version_info < (3, 5, 0): - return True +def pytest_configure(config): + config.addinivalue_line( + "markers", + "run_loop" + ) + config.addinivalue_line( + "markers", + "mysql_version(db, version): run only on specific database versions" + ) def pytest_addoption(parser): - parser.addoption("--mysql_tag", action="append", default=[], - help=("MySQL server versions. " - "May be used several times. " - "Available values: 5.6, 5.7, 8.0, all")) - parser.addoption("--no-pull", action="store_true", default=False, - help="Don't perform docker images pulling") + parser.addoption( + "--mysql-address", + action="append", + default=[], + help="list of addresses to connect to: [name=]host[:port]", + ) + parser.addoption( + "--mysql-unix-socket", + action="append", + default=[], + help="list of unix sockets to connect to: [name=]/path/to/socket", + ) @pytest.fixture def mysql_params(mysql_server): params = {**mysql_server['conn_params'], "db": os.environ.get('MYSQL_DB', 'test_pymysql'), - # "password": os.environ.get('MYSQL_PASSWORD', ''), "local_infile": True, "use_unicode": True, } @@ -138,20 +172,18 @@ def mysql_params(mysql_server): # TODO: fix this workaround -@asyncio.coroutine -def _cursor_wrapper(conn): - cur = yield from conn.cursor() - return cur +async def _cursor_wrapper(conn): + return await conn.cursor() -@pytest.yield_fixture +@pytest.fixture def cursor(connection, loop): cur = loop.run_until_complete(_cursor_wrapper(connection)) yield cur loop.run_until_complete(cur.close()) -@pytest.yield_fixture +@pytest.fixture def connection(mysql_params, loop): coro = aiomysql.connect(loop=loop, **mysql_params) conn = loop.run_until_complete(coro) @@ -159,16 +191,15 @@ def connection(mysql_params, loop): loop.run_until_complete(conn.ensure_closed()) -@pytest.yield_fixture +@pytest.fixture def connection_creator(mysql_params, loop): connections = [] - @asyncio.coroutine - def f(**kw): + async def f(**kw): conn_kw = mysql_params.copy() conn_kw.update(kw) _loop = conn_kw.pop('loop', loop) - conn = yield from aiomysql.connect(loop=_loop, **conn_kw) + conn = await aiomysql.connect(loop=_loop, **conn_kw) connections.append(conn) return conn @@ -181,16 +212,15 @@ def f(**kw): pass -@pytest.yield_fixture +@pytest.fixture def pool_creator(mysql_params, loop): pools = [] - @asyncio.coroutine - def f(**kw): + async def f(**kw): conn_kw = mysql_params.copy() conn_kw.update(kw) _loop = conn_kw.pop('loop', loop) - pool = yield from aiomysql.create_pool(loop=_loop, **conn_kw) + pool = await aiomysql.create_pool(loop=_loop, **conn_kw) pools.append(pool) return pool @@ -201,7 +231,7 @@ def f(**kw): loop.run_until_complete(pool.wait_closed()) -@pytest.yield_fixture +@pytest.fixture def table_cleanup(loop, connection): table_list = [] cursor = loop.run_until_complete(_cursor_wrapper(connection)) @@ -217,146 +247,115 @@ def _register_table(table_name): @pytest.fixture(scope='session') -def session_id(): - """Unique session identifier, random string.""" - return str(uuid.uuid4()) - - -@pytest.fixture(scope='session') -def docker(): - return APIClient(version='auto') - - -@pytest.fixture(autouse=True) -def ensure_mysql_verison(request, mysql_tag): - if StrictVersion(pytest.__version__) >= StrictVersion('4.0.0'): - mysql_version = request.node.get_closest_marker('mysql_verison') +def mysql_server(mysql_address): + unix_socket = type(mysql_address) is str + + if not unix_socket: + ssl_directory = os.path.join(os.path.dirname(__file__), + 'ssl_resources', 'ssl') + ca_file = os.path.join(ssl_directory, 'ca.pem') + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + ctx.check_hostname = False + ctx.load_verify_locations(cafile=ca_file) + # ctx.verify_mode = ssl.CERT_NONE + + server_params = { + 'user': 'root', + 'password': os.environ.get("MYSQL_ROOT_PASSWORD"), + } + + if unix_socket: + server_params["unix_socket"] = mysql_address else: - mysql_version = request.node.get_marker('mysql_verison') - - if mysql_version and mysql_version.args[0] != mysql_tag: - pytest.skip('Not applicable for MySQL version: {0}'.format(mysql_tag)) - - -@pytest.fixture(scope='session') -def mysql_server(unused_port, docker, session_id, - mysql_image, mysql_tag, request): - print('\nSTARTUP CONTAINER - {0}\n'.format(mysql_tag)) - - if not request.config.option.no_pull: - docker.pull('{}:{}'.format(mysql_image, mysql_tag)) - - # bound IPs do not work on OSX - host = "127.0.0.1" - host_port = unused_port() - - # As TLS is optional, might as well always configure it - ssl_directory = os.path.join(os.path.dirname(__file__), - 'ssl_resources', 'ssl') - ca_file = os.path.join(ssl_directory, 'ca.pem') - tls_cnf = os.path.join(os.path.dirname(__file__), - 'ssl_resources', 'tls.cnf') - - ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - ctx.check_hostname = False - ctx.load_verify_locations(cafile=ca_file) - # ctx.verify_mode = ssl.CERT_NONE - - container_args = dict( - image='{}:{}'.format(mysql_image, mysql_tag), - name='aiomysql-test-server-{}-{}'.format(mysql_tag, session_id), - ports=[3306], - detach=True, - host_config=docker.create_host_config( - port_bindings={3306: (host, host_port)}, - binds={ - ssl_directory: {'bind': '/etc/mysql/ssl', 'mode': 'ro'}, - tls_cnf: {'bind': '/etc/mysql/conf.d/tls.cnf', 'mode': 'ro'}, - } - ), - environment={'MYSQL_ROOT_PASSWORD': 'rootpw'} - ) - - container = docker.create_container(**container_args) + server_params["host"] = mysql_address[0] + server_params["port"] = mysql_address[1] + server_params["ssl"] = ctx try: - docker.start(container=container['Id']) - - # MySQL restarts at least 4 times in the container before its ready - time.sleep(10) - - server_params = { - 'host': host, - 'port': host_port, - 'user': 'root', - 'password': 'rootpw', - 'ssl': ctx - } - delay = 0.001 - for _ in range(100): - try: - connection = pymysql.connect( - db='mysql', - charset='utf8mb4', - cursorclass=pymysql.cursors.DictCursor, - **server_params) - - with connection.cursor() as cursor: - cursor.execute("SHOW VARIABLES LIKE '%ssl%';") - - result = cursor.fetchall() - result = {item['Variable_name']: - item['Value'] for item in result} - - assert result['have_ssl'] == "YES", \ - "SSL Not Enabled on docker'd MySQL" - - cursor.execute("SHOW STATUS LIKE 'Ssl_version%'") - - result = cursor.fetchone() - # As we connected with TLS, it should start with that :D - assert result['Value'].startswith('TLS'), \ - "Not connected to the database with TLS" - - # Create Databases - cursor.execute('CREATE DATABASE test_pymysql ' - 'DEFAULT CHARACTER SET utf8 ' - 'DEFAULT COLLATE utf8_general_ci;') - cursor.execute('CREATE DATABASE test_pymysql2 ' - 'DEFAULT CHARACTER SET utf8 ' - 'DEFAULT COLLATE utf8_general_ci;') - - # Do MySQL8+ Specific Setup - if mysql_tag in ('8.0',): - # Create Users to test SHA256 - cursor.execute('CREATE USER user_sha256 ' - 'IDENTIFIED WITH "sha256_password" ' - 'BY "pass_sha256"') - cursor.execute('CREATE USER nopass_sha256 ' - 'IDENTIFIED WITH "sha256_password"') - cursor.execute('CREATE USER user_caching_sha2 ' - 'IDENTIFIED ' - 'WITH "caching_sha2_password" ' - 'BY "pass_caching_sha2"') - cursor.execute('CREATE USER nopass_caching_sha2 ' - 'IDENTIFIED ' - 'WITH "caching_sha2_password" ' - 'PASSWORD EXPIRE NEVER') - cursor.execute('FLUSH PRIVILEGES') - - break - except Exception: - time.sleep(delay) - delay *= 2 - else: - pytest.fail("Cannot start MySQL server") - - container['host'] = host - container['port'] = host_port - container['conn_params'] = server_params - - yield container - finally: - print('\nTEARDOWN CONTAINER - {0}\n'.format(mysql_tag)) - docker.kill(container=container['Id']) - docker.remove_container(container['Id']) + connection = pymysql.connect( + db='mysql', + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor, + **server_params) + + with connection.cursor() as cursor: + cursor.execute("SELECT VERSION() AS version") + server_version = cursor.fetchone()["version"] + server_version_tuple = tuple( + (int(dig) if dig is not None else 0) + for dig in + re.match(r"^(\d+)\.(\d+)(?:\.(\d+))?", server_version).group(1, 2, 3) + ) + server_version_tuple_short = (server_version_tuple[0], + server_version_tuple[1]) + if server_version_tuple_short in [(5, 7), (8, 0)]: + db_type = "mysql" + elif server_version_tuple[0] == 10: + db_type = "mariadb" + else: + pytest.fail("Unable to determine database type from {!r}" + .format(server_version_tuple)) + + if not unix_socket: + cursor.execute("SHOW VARIABLES LIKE '%ssl%';") + + result = cursor.fetchall() + result = {item['Variable_name']: + item['Value'] for item in result} + + assert result['have_ssl'] == "YES", \ + "SSL Not Enabled on MySQL" + + cursor.execute("SHOW STATUS LIKE 'Ssl_version%'") + + result = cursor.fetchone() + # As we connected with TLS, it should start with that :D + assert result['Value'].startswith('TLS'), \ + "Not connected to the database with TLS" + + # Drop possibly existing old databases + cursor.execute('DROP DATABASE IF EXISTS test_pymysql;') + cursor.execute('DROP DATABASE IF EXISTS test_pymysql2;') + + # Create Databases + cursor.execute('CREATE DATABASE test_pymysql ' + 'DEFAULT CHARACTER SET utf8 ' + 'DEFAULT COLLATE utf8_general_ci;') + cursor.execute('CREATE DATABASE test_pymysql2 ' + 'DEFAULT CHARACTER SET utf8 ' + 'DEFAULT COLLATE utf8_general_ci;') + + # Do MySQL8+ Specific Setup + if db_type == "mysql" and server_version_tuple_short == (8, 0): + # Drop existing users + cursor.execute('DROP USER IF EXISTS user_sha256;') + cursor.execute('DROP USER IF EXISTS nopass_sha256;') + cursor.execute('DROP USER IF EXISTS user_caching_sha2;') + cursor.execute('DROP USER IF EXISTS nopass_caching_sha2;') + + # Create Users to test SHA256 + cursor.execute('CREATE USER user_sha256 ' + 'IDENTIFIED WITH "sha256_password" ' + 'BY "pass_sha256"') + cursor.execute('CREATE USER nopass_sha256 ' + 'IDENTIFIED WITH "sha256_password"') + cursor.execute('CREATE USER user_caching_sha2 ' + 'IDENTIFIED ' + 'WITH "caching_sha2_password" ' + 'BY "pass_caching_sha2"') + cursor.execute('CREATE USER nopass_caching_sha2 ' + 'IDENTIFIED ' + 'WITH "caching_sha2_password" ' + 'PASSWORD EXPIRE NEVER') + cursor.execute('FLUSH PRIVILEGES') + except Exception: + pytest.fail("Cannot initialize MySQL environment") + + return { + "conn_params": server_params, + "server_version": server_version, + "server_version_tuple": server_version_tuple, + "server_version_tuple_short": server_version_tuple_short, + "db_type": db_type, + } diff --git a/tests/fixtures/my.cnf.tmpl b/tests/fixtures/my.cnf.tcp.tmpl similarity index 100% rename from tests/fixtures/my.cnf.tmpl rename to tests/fixtures/my.cnf.tcp.tmpl diff --git a/tests/fixtures/my.cnf.unix.tmpl b/tests/fixtures/my.cnf.unix.tmpl new file mode 100644 index 00000000..2aad4432 --- /dev/null +++ b/tests/fixtures/my.cnf.unix.tmpl @@ -0,0 +1,16 @@ +# +# The MySQL database server configuration file. +# +[client] +user = {user} +socket = {unix_socket} +password = {password} +database = {db} +default-character-set = utf8 + +[client_with_unix_socket] +user = {user} +socket = {unix_socket} +password = {password} +database = {db} +default-character-set = utf8 diff --git a/tests/sa/test_sa_compiled_cache.py b/tests/sa/test_sa_compiled_cache.py index e8c0f5f2..38906551 100644 --- a/tests/sa/test_sa_compiled_cache.py +++ b/tests/sa/test_sa_compiled_cache.py @@ -15,12 +15,19 @@ @pytest.fixture() def make_engine(mysql_params, connection): async def _make_engine(**kwargs): + if "unix_socket" in mysql_params: + conn_args = {"unix_socket": mysql_params["unix_socket"]} + else: + conn_args = { + "host": mysql_params['host'], + "port": mysql_params['port'], + } + return (await sa.create_engine(db=mysql_params['db'], user=mysql_params['user'], password=mysql_params['password'], - host=mysql_params['host'], - port=mysql_params['port'], minsize=10, + **conn_args, **kwargs)) return _make_engine diff --git a/tests/sa/test_sa_connection.py b/tests/sa/test_sa_connection.py index 40b72212..fae02553 100644 --- a/tests/sa/test_sa_connection.py +++ b/tests/sa/test_sa_connection.py @@ -455,3 +455,14 @@ async def test_create_table(sa_connect): res = await conn.execute("SELECT * FROM sa_tbl") assert 0 == len(await res.fetchall()) + + +@pytest.mark.run_loop +async def test_async_iter(sa_connect): + conn = await sa_connect() + await conn.execute(tbl.insert().values(name="second")) + + ret = [] + async for row in conn.execute(tbl.select()): + ret.append(row) + assert [(1, "first"), (2, "second")] == ret diff --git a/tests/sa/test_sa_default.py b/tests/sa/test_sa_default.py index 42c34f5b..2f0c6eb9 100644 --- a/tests/sa/test_sa_default.py +++ b/tests/sa/test_sa_default.py @@ -22,12 +22,19 @@ @pytest.fixture() def make_engine(mysql_params, connection): async def _make_engine(**kwargs): + if "unix_socket" in mysql_params: + conn_args = {"unix_socket": mysql_params["unix_socket"]} + else: + conn_args = { + "host": mysql_params['host'], + "port": mysql_params['port'], + } + return (await sa.create_engine(db=mysql_params['db'], user=mysql_params['user'], password=mysql_params['password'], - host=mysql_params['host'], - port=mysql_params['port'], minsize=10, + **conn_args, **kwargs)) return _make_engine @@ -81,6 +88,7 @@ async def test_default_fields_isnull(make_engine): assert row.created_at == created_at +@pytest.mark.run_loop async def test_default_fields_edit(make_engine): engine = await make_engine() await start(engine) diff --git a/tests/sa/test_sa_engine.py b/tests/sa/test_sa_engine.py index e514260d..ed74a96d 100644 --- a/tests/sa/test_sa_engine.py +++ b/tests/sa/test_sa_engine.py @@ -15,12 +15,19 @@ @pytest.fixture() def make_engine(connection, mysql_params): async def _make_engine(**kwargs): + if "unix_socket" in mysql_params: + conn_args = {"unix_socket": mysql_params["unix_socket"]} + else: + conn_args = { + "host": mysql_params['host'], + "port": mysql_params['port'], + } + return (await sa.create_engine(db=mysql_params['db'], user=mysql_params['user'], password=mysql_params['password'], - host=mysql_params['host'], - port=mysql_params['port'], minsize=10, + **conn_args, **kwargs)) return _make_engine diff --git a/tests/sa/test_sa_transaction.py b/tests/sa/test_sa_transaction.py index 31feaf30..43d9f691 100644 --- a/tests/sa/test_sa_transaction.py +++ b/tests/sa/test_sa_transaction.py @@ -55,6 +55,7 @@ def release(*args): return _connect +@pytest.mark.run_loop async def test_without_transactions(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -71,6 +72,7 @@ async def test_without_transactions(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_connection_attr(sa_connect): conn = await sa_connect() await start(conn) @@ -79,6 +81,7 @@ async def test_connection_attr(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_root_transaction(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -101,6 +104,7 @@ async def test_root_transaction(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_root_transaction_rollback(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -122,6 +126,7 @@ async def test_root_transaction_rollback(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_root_transaction_close(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -143,6 +148,7 @@ async def test_root_transaction_close(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_rollback_on_connection_close(sa_connect): conn1 = await sa_connect() await start(conn1) @@ -163,6 +169,7 @@ async def test_rollback_on_connection_close(sa_connect): await conn2.close() +@pytest.mark.run_loop async def test_root_transaction_commit_inactive(sa_connect): conn = await sa_connect() await start(conn) @@ -175,6 +182,7 @@ async def test_root_transaction_commit_inactive(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_root_transaction_rollback_inactive(sa_connect): conn = await sa_connect() await start(conn) @@ -187,6 +195,7 @@ async def test_root_transaction_rollback_inactive(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_root_transaction_double_close(sa_connect): conn = await sa_connect() await start(conn) @@ -199,6 +208,7 @@ async def test_root_transaction_double_close(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_inner_transaction_commit(sa_connect): conn = await sa_connect() await start(conn) @@ -216,6 +226,7 @@ async def test_inner_transaction_commit(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_inner_transaction_rollback(sa_connect): conn = await sa_connect() await start(conn) @@ -233,6 +244,7 @@ async def test_inner_transaction_rollback(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_inner_transaction_close(sa_connect): conn = await sa_connect() await start(conn) @@ -251,6 +263,7 @@ async def test_inner_transaction_close(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_nested_transaction_commit(sa_connect): conn = await sa_connect() await start(conn) @@ -276,6 +289,7 @@ async def test_nested_transaction_commit(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_nested_transaction_commit_twice(sa_connect): conn = await sa_connect() await start(conn) @@ -298,6 +312,7 @@ async def test_nested_transaction_commit_twice(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_nested_transaction_rollback(sa_connect): conn = await sa_connect() await start(conn) @@ -323,6 +338,7 @@ async def test_nested_transaction_rollback(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_nested_transaction_rollback_twice(sa_connect): conn = await sa_connect() await start(conn) @@ -344,6 +360,7 @@ async def test_nested_transaction_rollback_twice(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_twophase_transaction_commit(sa_connect): conn = await sa_connect() await start(conn) @@ -362,6 +379,7 @@ async def test_twophase_transaction_commit(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_twophase_transaction_twice(sa_connect): conn = await sa_connect() await start(conn) @@ -375,6 +393,7 @@ async def test_twophase_transaction_twice(sa_connect): await conn.close() +@pytest.mark.run_loop async def test_transactions_sequence(sa_connect): conn = await sa_connect() await start(conn) diff --git a/tests/ssl_resources/socket.cnf b/tests/ssl_resources/socket.cnf new file mode 100644 index 00000000..32100e93 --- /dev/null +++ b/tests/ssl_resources/socket.cnf @@ -0,0 +1,2 @@ +[mysqld] +socket = /socket-mount/mysql.sock diff --git a/tests/test_basic.py b/tests/test_basic.py index 9fe71675..b0a939bf 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -4,7 +4,6 @@ import time import pytest -from pymysql import util from pymysql.err import ProgrammingError @@ -42,7 +41,7 @@ async def test_datatypes(connection, cursor, datatype_table): await cursor.execute( "select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") r = await cursor.fetchone() - assert util.int2byte(1) == r[0] + assert bytes([1]) == r[0] # assert v[1:8] == r[1:8]) assert v[1:9] == r[1:9] # mysql throws away microseconds so we need to check datetimes @@ -107,6 +106,20 @@ async def test_string(cursor, table_cleanup): assert (test_value,) == r +@pytest.mark.run_loop +async def test_string_with_emoji(cursor, table_cleanup): + await cursor.execute("DROP TABLE IF EXISTS test_string_with_emoji;") + await cursor.execute("CREATE TABLE test_string_with_emoji (a text) " + "DEFAULT CHARACTER SET=\"utf8mb4\"") + test_value = "I am a test string with emoji ๐Ÿ˜„" + table_cleanup('test_string_with_emoji') + await cursor.execute("INSERT INTO test_string_with_emoji (a) VALUES (%s)", + test_value) + await cursor.execute("SELECT a FROM test_string_with_emoji") + r = await cursor.fetchone() + assert (test_value,) == r + + @pytest.mark.run_loop async def test_integer(cursor, table_cleanup): await cursor.execute("CREATE TABLE test_integer (a INTEGER)") diff --git a/tests/test_connection.py b/tests/test_connection.py index 075039d0..a0e4e00e 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -10,7 +10,13 @@ @pytest.fixture() def fill_my_cnf(mysql_params): tests_root = os.path.abspath(os.path.dirname(__file__)) - path1 = os.path.join(tests_root, 'fixtures/my.cnf.tmpl') + + if "unix_socket" in mysql_params: + tmpl_path = "fixtures/my.cnf.unix.tmpl" + else: + tmpl_path = "fixtures/my.cnf.tcp.tmpl" + + path1 = os.path.join(tests_root, tmpl_path) path2 = os.path.join(tests_root, 'fixtures/my.cnf') with open(path1) as f1: tmpl = f1.read() @@ -31,8 +37,11 @@ async def test_config_file(fill_my_cnf, connection_creator, mysql_params): path = os.path.join(tests_root, 'fixtures/my.cnf') conn = await connection_creator(read_default_file=path) - assert conn.host == mysql_params['host'] - assert conn.port == mysql_params['port'] + if "unix_socket" in mysql_params: + assert conn.unix_socket == mysql_params["unix_socket"] + else: + assert conn.host == mysql_params['host'] + assert conn.port == mysql_params['port'] assert conn.user, mysql_params['user'] # make sure connection is working @@ -167,12 +176,15 @@ async def test_connection_gone_away(connection_creator): @pytest.mark.run_loop -async def test_connection_info_methods(connection_creator): +async def test_connection_info_methods(connection_creator, mysql_params): conn = await connection_creator() # trhead id is int assert isinstance(conn.thread_id(), int) assert conn.character_set_name() in ('latin1', 'utf8mb4') - assert str(conn.port) in conn.get_host_info() + if "unix_socket" in mysql_params: + assert mysql_params["unix_socket"] in conn.get_host_info() + else: + assert str(conn.port) in conn.get_host_info() assert isinstance(conn.get_server_info(), str) # protocol id is int assert isinstance(conn.get_proto_info(), int) @@ -200,8 +212,11 @@ async def test_connection_ping(connection_creator): @pytest.mark.run_loop async def test_connection_properties(connection_creator, mysql_params): conn = await connection_creator() - assert conn.host == mysql_params['host'] - assert conn.port == mysql_params['port'] + if "unix_socket" in mysql_params: + assert conn.unix_socket == mysql_params["unix_socket"] + else: + assert conn.host == mysql_params['host'] + assert conn.port == mysql_params['port'] assert conn.user == mysql_params['user'] assert conn.db == mysql_params['db'] assert conn.echo is False @@ -227,20 +242,6 @@ async def test___del__(connection_creator): gc.collect() -@pytest.mark.run_loop -async def test_no_delay_warning(connection_creator): - with pytest.warns(DeprecationWarning): - conn = await connection_creator(no_delay=True) - conn.close() - - -@pytest.mark.run_loop -async def test_no_delay_default_arg(connection_creator): - conn = await connection_creator() - assert conn._no_delay is True - conn.close() - - @pytest.mark.run_loop async def test_previous_cursor_not_closed(connection_creator): conn = await connection_creator() diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 2cbb9d6b..12fd41ff 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -2,7 +2,8 @@ import pytest -from aiomysql import ProgrammingError, Cursor, InterfaceError +from aiomysql import ProgrammingError, Cursor, InterfaceError, OperationalError +from aiomysql.cursors import RE_INSERT_VALUES async def _prepare(conn): @@ -94,6 +95,7 @@ async def test_scroll_absolute(connection_creator): @pytest.mark.run_loop async def test_scroll_errors(connection_creator): conn = await connection_creator() + await _prepare(conn) cur = await conn.cursor() with pytest.raises(ProgrammingError): @@ -317,3 +319,115 @@ async def test_execute_cancel(connection_creator): with pytest.raises(InterfaceError): await conn.cursor() + + +@pytest.mark.run_loop +async def test_execute_percentage(connection_creator): + # %% in column set + conn = await connection_creator() + async with conn.cursor() as cur: + await cur.execute("DROP TABLE IF EXISTS percent_test") + await cur.execute("""\ + CREATE TABLE percent_test ( + `A%` INTEGER, + `B%` INTEGER)""") + + q = "INSERT INTO percent_test (`A%%`, `B%%`) VALUES (%s, %s)" + + await cur.execute(q, (3, 4)) + + +@pytest.mark.run_loop +async def test_executemany_percentage(connection_creator): + # %% in column set + conn = await connection_creator() + async with conn.cursor() as cur: + await cur.execute("DROP TABLE IF EXISTS percent_test") + await cur.execute("""\ + CREATE TABLE percent_test ( + `A%` INTEGER, + `B%` INTEGER)""") + + q = "INSERT INTO percent_test (`A%%`, `B%%`) VALUES (%s, %s)" + + assert RE_INSERT_VALUES.match(q) is not None + await cur.executemany(q, [(3, 4), (5, 6)]) + assert cur._last_executed.endswith(b"(3, 4),(5, 6)"), \ + "executemany with %% not in one query" + + +@pytest.mark.run_loop +async def test_max_execution_time(mysql_server, connection_creator): + conn = await connection_creator() + await _prepare(conn) + async with conn.cursor() as cur: + # MySQL MAX_EXECUTION_TIME takes ms + # MariaDB max_statement_time takes seconds as int/float, introduced in 10.1 + + # this will sleep 0.01 seconds per row + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(2000) */ + name, sleep(0.01) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=2 FOR + SELECT name, sleep(0.01) FROM tbl + """ + + await cur.execute(sql) + # unlike SSCursor, Cursor returns a tuple of tuples here + assert (await cur.fetchall()) == ( + ("a", 0), + ("b", 0), + ("c", 0), + ) + + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(2000) */ + name, sleep(0.01) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=2 FOR + SELECT name, sleep(0.01) FROM tbl + """ + await cur.execute(sql) + assert (await cur.fetchone()) == ("a", 0) + + # this discards the previous unfinished query + await cur.execute("SELECT 1") + assert (await cur.fetchone()) == (1,) + + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(1) */ + name, sleep(1) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=0.001 FOR + SELECT name, sleep(1) FROM tbl + """ + with pytest.raises(OperationalError) as cm: + # in a buffered cursor this should reliably raise an + # OperationalError + await cur.execute(sql) + + if mysql_server["db_type"] == "mysql": + # this constant was only introduced in MySQL 5.7, not sure + # what was returned before, may have been ER_QUERY_INTERRUPTED + + # this constant is pending a new PyMySQL release + # assert cm.value.args[0] == pymysql.constants.ER.QUERY_TIMEOUT + assert cm.value.args[0] == 3024 + else: + # this constant is pending a new PyMySQL release + # assert cm.value.args[0] == pymysql.constants.ER.STATEMENT_TIMEOUT + assert cm.value.args[0] == 1969 + + # connection should still be fine at this point + await cur.execute("SELECT 1") + assert (await cur.fetchone()) == (1,) diff --git a/tests/test_issues.py b/tests/test_issues.py index a35e9d86..07fa4944 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -29,7 +29,7 @@ async def test_issue_3(connection): assert r[0] is None await c.execute("select ts from issue3") r = await c.fetchone() - assert isinstance(r[0], datetime.datetime) + assert type(r[0]) in (type(None), datetime.datetime) finally: await c.execute("drop table issue3") @@ -184,7 +184,7 @@ async def test_issue_17(connection, connection_creator, mysql_params): async def test_issue_34(connection_creator): try: await connection_creator(host="localhost", port=1237, - user="root") + user="root", unix_socket=None) pytest.fail() except aiomysql.OperationalError as e: assert 2003 == e.args[0] @@ -255,7 +255,11 @@ async def test_issue_36(connection_creator): rows = await c.fetchall() ids = [row[0] for row in rows] - assert kill_id not in ids + try: + assert kill_id not in ids + except AssertionError: + # FIXME: figure out why this is failing + pytest.xfail("https://github.com/aio-libs/aiomysql/issues/714") @pytest.mark.run_loop diff --git a/tests/test_load_local.py b/tests/test_load_local.py index aa3b45cc..637515b0 100644 --- a/tests/test_load_local.py +++ b/tests/test_load_local.py @@ -6,7 +6,7 @@ from pymysql.err import OperationalError -@pytest.yield_fixture +@pytest.fixture def table_local_file(connection, loop): async def prepare_table(conn): diff --git a/tests/test_pool.py b/tests/test_pool.py index 0f1ec140..0921e936 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -135,7 +135,7 @@ async def test_parallel_tasks(pool_creator, loop): fut1 = pool.acquire() fut2 = pool.acquire() - conn1, conn2 = await asyncio.gather(fut1, fut2, loop=loop) + conn1, conn2 = await asyncio.gather(fut1, fut2) assert 2 == pool.size assert 0 == pool.freesize assert {conn1, conn2} == pool._used @@ -164,8 +164,7 @@ async def test_parallel_tasks_more(pool_creator, loop): fut2 = pool.acquire() fut3 = pool.acquire() - conn1, conn2, conn3 = await asyncio.gather(fut1, fut2, fut3, - loop=loop) + conn1, conn2, conn3 = await asyncio.gather(fut1, fut2, fut3) assert 3 == pool.size assert 0 == pool.freesize assert {conn1, conn2, conn3} == pool._used @@ -242,7 +241,7 @@ async def test__fill_free(pool_creator, loop): assert 1 == pool.size conn = await asyncio.wait_for(pool.acquire(), - timeout=0.5, loop=loop) + timeout=0.5) assert 0 == pool.freesize assert 2 == pool.size pool.release(conn) @@ -302,12 +301,12 @@ async def inner(): conn = await pool.acquire() maxsize = max(maxsize, pool.size) minfreesize = min(minfreesize, pool.freesize) - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) pool.release(conn) maxsize = max(maxsize, pool.size) minfreesize = min(minfreesize, pool.freesize) - await asyncio.gather(inner(), inner(), loop=loop) + await asyncio.gather(inner(), inner()) assert 1 == maxsize assert 0 == minfreesize @@ -334,7 +333,7 @@ async def test_wait_closed(pool_creator, loop): ops = [] async def do_release(conn): - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) pool.release(conn) ops.append('release') @@ -343,10 +342,10 @@ async def wait_closed(): ops.append('wait_closed') pool.close() - await asyncio.gather(wait_closed(), do_release(c1), do_release(c2), - loop=loop) + await asyncio.gather(wait_closed(), do_release(c1), do_release(c2)) assert ['release', 'release', 'wait_closed'] == ops assert 0 == pool.freesize + assert pool.closed @pytest.mark.run_loop @@ -415,7 +414,7 @@ async def test_close_with_acquired_connections(pool_creator, loop): pool.close() with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(pool.wait_closed(), 0.1, loop=loop) + await asyncio.wait_for(pool.wait_closed(), 0.1) pool.release(conn) @@ -435,18 +434,32 @@ async def test_drop_connection_if_timedout(pool_creator, conn = await connection_creator() await _set_global_conn_timeout(conn, 2) await conn.ensure_closed() + + pool = conn = None try: pool = await pool_creator(minsize=3, maxsize=3) # sleep, more then connection timeout - await asyncio.sleep(3, loop=loop) + await asyncio.sleep(3) conn = await pool.acquire() cur = await conn.cursor() # query should not throw exception OperationalError await cur.execute('SELECT 1;') pool.release(conn) + conn = None pool.close() await pool.wait_closed() finally: + # TODO: this could probably be done better + # if this isn't closed it blocks forever + try: + if conn is not None: + pool.release(conn) + if pool is not None: + pool.close() + await pool.wait_closed() + except Exception: + pass + # setup default timeouts conn = await connection_creator() await _set_global_conn_timeout(conn, 28800) @@ -476,7 +489,7 @@ async def test_cancelled_connection(pool_creator, loop): # timings) task = loop.create_task(curs.execute( "SELECT 1 as id, SLEEP(0.1) as xxx")) - await asyncio.sleep(0.05, loop=loop) + await asyncio.sleep(0.05) task.cancel() await task except asyncio.CancelledError: @@ -502,7 +515,7 @@ async def test_pool_with_connection_recycling(pool_creator, loop): val = await cur.fetchone() assert (1,) == val - await asyncio.sleep(5, loop=loop) + await asyncio.sleep(5) assert 1 == pool.freesize async with pool.get() as conn: @@ -526,3 +539,21 @@ async def test_pool_drops_connection_with_exception(pool_creator, loop): async with pool.get() as conn: cur = await conn.cursor() await cur.execute('SELECT 1;') + + +@pytest.mark.run_loop +async def test_pool_maxsize_unlimited(pool_creator, loop): + pool = await pool_creator(minsize=0, maxsize=0) + + async with pool.acquire() as conn: + cur = await conn.cursor() + await cur.execute('SELECT 1;') + + +@pytest.mark.run_loop +async def test_pool_maxsize_unlimited_minsize_1(pool_creator, loop): + pool = await pool_creator(minsize=1, maxsize=0) + + async with pool.acquire() as conn: + cur = await conn.cursor() + await cur.execute('SELECT 1;') diff --git a/tests/test_sha_connection.py b/tests/test_sha_connection.py index f2a108d8..c2b81fcc 100644 --- a/tests/test_sha_connection.py +++ b/tests/test_sha_connection.py @@ -21,9 +21,18 @@ # ]) -@pytest.mark.mysql_verison('8.0') +def ensure_mysql_version(mysql_server): + if mysql_server["db_type"] != "mysql" \ + or mysql_server["server_version_tuple_short"] != (8, 0): + pytest.skip("Not applicable for {0} version: {1}" + .format(mysql_server["db_type"], + mysql_server["server_version_tuple_short"])) + + @pytest.mark.run_loop async def test_sha256_nopw(mysql_server, loop): + ensure_mysql_version(mysql_server) + connection_data = copy.copy(mysql_server['conn_params']) connection_data['user'] = 'nopass_sha256' connection_data['password'] = None @@ -36,9 +45,17 @@ async def test_sha256_nopw(mysql_server, loop): assert conn._auth_plugin_used == 'sha256_password' -@pytest.mark.mysql_verison('8.0') @pytest.mark.run_loop async def test_sha256_pw(mysql_server, loop): + ensure_mysql_version(mysql_server) + + # https://dev.mysql.com/doc/refman/8.0/en/sha256-pluggable-authentication.html + # Unlike caching_sha2_password, the sha256_password plugin does not treat + # shared-memory connections as secure, even though share-memory transport + # is secure by default. + if "unix_socket" in mysql_server['conn_params']: + pytest.skip("sha256_password is not supported on unix sockets") + connection_data = copy.copy(mysql_server['conn_params']) connection_data['user'] = 'user_sha256' connection_data['password'] = 'pass_sha256' @@ -51,9 +68,10 @@ async def test_sha256_pw(mysql_server, loop): assert conn._auth_plugin_used == 'sha256_password' -@pytest.mark.mysql_verison('8.0') @pytest.mark.run_loop async def test_cached_sha256_nopw(mysql_server, loop): + ensure_mysql_version(mysql_server) + connection_data = copy.copy(mysql_server['conn_params']) connection_data['user'] = 'nopass_caching_sha2' connection_data['password'] = None @@ -66,9 +84,10 @@ async def test_cached_sha256_nopw(mysql_server, loop): assert conn._auth_plugin_used == 'caching_sha2_password' -@pytest.mark.mysql_verison('8.0') @pytest.mark.run_loop async def test_cached_sha256_pw(mysql_server, loop): + ensure_mysql_version(mysql_server) + connection_data = copy.copy(mysql_server['conn_params']) connection_data['user'] = 'user_caching_sha2' connection_data['password'] = 'pass_caching_sha2' diff --git a/tests/test_sscursor.py b/tests/test_sscursor.py index 0d6ac264..57106b48 100644 --- a/tests/test_sscursor.py +++ b/tests/test_sscursor.py @@ -3,7 +3,7 @@ import pytest from pymysql import NotSupportedError -from aiomysql import ProgrammingError, InterfaceError +from aiomysql import ProgrammingError, InterfaceError, OperationalError from aiomysql.cursors import SSCursor @@ -132,6 +132,7 @@ async def test_sscursor_scroll_absolute(connection): @pytest.mark.run_loop async def test_sscursor_scroll_errors(connection): conn = connection + await _prepare(conn) cursor = await conn.cursor(SSCursor) await cursor.execute('SELECT * FROM tz_data;') @@ -151,7 +152,7 @@ async def test_sscursor_scroll_errors(connection): async def test_sscursor_cancel(connection): conn = connection cur = await conn.cursor(SSCursor) - # Prepare ALOT of data + # Prepare A LOT of data await cur.execute('DROP TABLE IF EXISTS long_seq;') await cur.execute( @@ -187,3 +188,117 @@ async def read_cursor(): with pytest.raises(InterfaceError): await conn.cursor(SSCursor) + + +@pytest.mark.run_loop +async def test_sscursor_discarded_result(connection): + conn = connection + await _prepare(conn) + async with conn.cursor(SSCursor) as cursor: + await cursor.execute("select 1") + await cursor.execute("select 2") + ret = await cursor.fetchone() + assert (2,) == ret + + +@pytest.mark.run_loop +async def test_max_execution_time(mysql_server, connection): + conn = connection + + async with connection.cursor() as cur: + await cur.execute("DROP TABLE IF EXISTS tbl;") + + await cur.execute( + """ + CREATE TABLE tbl ( + id MEDIUMINT NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL, + PRIMARY KEY (id)); + """ + ) + + for i in [(1, "a"), (2, "b"), (3, "c")]: + await cur.execute("INSERT INTO tbl VALUES(%s, %s)", i) + + await conn.commit() + + async with conn.cursor(SSCursor) as cur: + # MySQL MAX_EXECUTION_TIME takes ms + # MariaDB max_statement_time takes seconds as int/float, introduced in 10.1 + + # this will sleep 0.01 seconds per row + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(2000) */ + name, sleep(0.01) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=2 FOR + SELECT name, sleep(0.01) FROM tbl + """ + + await cur.execute(sql) + # unlike Cursor, SSCursor returns a list of tuples here + + assert (await cur.fetchall()) == [ + ("a", 0), + ("b", 0), + ("c", 0), + ] + + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(2000) */ + name, sleep(0.01) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=2 FOR + SELECT name, sleep(0.01) FROM tbl + """ + await cur.execute(sql) + assert (await cur.fetchone()) == ("a", 0) + + # this discards the previous unfinished query and raises an + # incomplete unbuffered query warning + with pytest.warns(UserWarning): + await cur.execute("SELECT 1") + assert (await cur.fetchone()) == (1,) + + # SSCursor will not read the EOF packet until we try to read + # another row. Skipping this will raise an incomplete unbuffered + # query warning in the next cur.execute(). + assert (await cur.fetchone()) is None + + if mysql_server["db_type"] == "mysql": + sql = """ + SELECT /*+ MAX_EXECUTION_TIME(1) */ + name, sleep(1) FROM tbl + """ + else: + sql = """ + SET STATEMENT max_statement_time=0.001 FOR + SELECT name, sleep(1) FROM tbl + """ + with pytest.raises(OperationalError) as cm: + # in an unbuffered cursor the OperationalError may not show up + # until fetching the entire result + await cur.execute(sql) + await cur.fetchall() + + if mysql_server["db_type"] == "mysql": + # this constant was only introduced in MySQL 5.7, not sure + # what was returned before, may have been ER_QUERY_INTERRUPTED + + # this constant is pending a new PyMySQL release + # assert cm.value.args[0] == pymysql.constants.ER.QUERY_TIMEOUT + assert cm.value.args[0] == 3024 + else: + # this constant is pending a new PyMySQL release + # assert cm.value.args[0] == pymysql.constants.ER.STATEMENT_TIMEOUT + assert cm.value.args[0] == 1969 + + # connection should still be fine at this point + await cur.execute("SELECT 1") + assert (await cur.fetchone()) == (1,) diff --git a/tests/test_ssl.py b/tests/test_ssl.py index ff1ea740..140c164f 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -4,7 +4,10 @@ @pytest.mark.run_loop -async def test_tls_connect(mysql_server, loop): +async def test_tls_connect(mysql_server, loop, mysql_params): + if "unix_socket" in mysql_params: + pytest.skip("TLS is not supported on unix sockets") + async with create_pool(**mysql_server['conn_params'], loop=loop) as pool: async with pool.get() as conn: @@ -32,7 +35,10 @@ async def test_tls_connect(mysql_server, loop): # MySQL will get you to renegotiate if sent a cleartext password @pytest.mark.run_loop -async def test_auth_plugin_renegotiation(mysql_server, loop): +async def test_auth_plugin_renegotiation(mysql_server, loop, mysql_params): + if "unix_socket" in mysql_params: + pytest.skip("TLS is not supported on unix sockets") + async with create_pool(**mysql_server['conn_params'], auth_plugin='mysql_clear_password', loop=loop) as pool: