From 11aed30bcc5db7e37fd574b0e6323e3127e3cbf3 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Mon, 15 Mar 2021 22:33:21 +0100 Subject: [PATCH 1/2] build(deps): switch to sqlalchemy 1.4 --- databases/backends/aiopg.py | 35 ++++++++++++++++++++++++++-------- databases/backends/mysql.py | 35 ++++++++++++++++++++++++++-------- databases/backends/postgres.py | 21 +++++++++++++++++++- databases/backends/sqlite.py | 35 ++++++++++++++++++++++++++-------- setup.py | 2 +- tests/test_databases.py | 8 ++++---- 6 files changed, 106 insertions(+), 30 deletions(-) diff --git a/databases/backends/aiopg.py b/databases/backends/aiopg.py index 2fecb1b5..e8ccc99d 100644 --- a/databases/backends/aiopg.py +++ b/databases/backends/aiopg.py @@ -7,11 +7,11 @@ import aiopg from aiopg.sa.engine import APGCompiler_psycopg2 from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 +from sqlalchemy.engine.cursor import CursorResultMetaData from sqlalchemy.engine.interfaces import Dialect, ExecutionContext -from sqlalchemy.engine.result import ResultMetaData, RowProxy +from sqlalchemy.engine.result import Row from sqlalchemy.sql import ClauseElement from sqlalchemy.sql.ddl import DDLElement -from sqlalchemy.types import TypeEngine from databases.core import DatabaseURL from databases.interfaces import ConnectionBackend, DatabaseBackend, TransactionBackend @@ -119,9 +119,15 @@ async def fetch_all(self, query: ClauseElement) -> typing.List[typing.Mapping]: try: await cursor.execute(query, args) rows = await cursor.fetchall() - metadata = ResultMetaData(context, cursor.description) + metadata = CursorResultMetaData(context, cursor.description) return [ - RowProxy(metadata, row, metadata._processors, metadata._keymap) + Row( + metadata, + metadata._processors, + metadata._keymap, + Row._default_key_style, + row, + ) for row in rows ] finally: @@ -136,8 +142,14 @@ async def fetch_one(self, query: ClauseElement) -> typing.Optional[typing.Mappin row = await cursor.fetchone() if row is None: return None - metadata = ResultMetaData(context, cursor.description) - return RowProxy(metadata, row, metadata._processors, metadata._keymap) + metadata = CursorResultMetaData(context, cursor.description) + return Row( + metadata, + metadata._processors, + metadata._keymap, + Row._default_key_style, + row, + ) finally: cursor.close() @@ -169,9 +181,15 @@ async def iterate( cursor = await self._connection.cursor() try: await cursor.execute(query, args) - metadata = ResultMetaData(context, cursor.description) + metadata = CursorResultMetaData(context, cursor.description) async for row in cursor: - yield RowProxy(metadata, row, metadata._processors, metadata._keymap) + yield Row( + metadata, + metadata._processors, + metadata._keymap, + Row._default_key_style, + row, + ) finally: cursor.close() @@ -196,6 +214,7 @@ def _compile( compiled._result_columns, compiled._ordered_columns, compiled._textual_ordered_columns, + compiled._loose_column_name_matching, ) else: args = {} diff --git a/databases/backends/mysql.py b/databases/backends/mysql.py index b6476add..b615488d 100644 --- a/databases/backends/mysql.py +++ b/databases/backends/mysql.py @@ -5,11 +5,11 @@ import aiomysql from sqlalchemy.dialects.mysql import pymysql +from sqlalchemy.engine.cursor import CursorResultMetaData from sqlalchemy.engine.interfaces import Dialect, ExecutionContext -from sqlalchemy.engine.result import ResultMetaData, RowProxy +from sqlalchemy.engine.result import Row from sqlalchemy.sql import ClauseElement from sqlalchemy.sql.ddl import DDLElement -from sqlalchemy.types import TypeEngine from databases.core import LOG_EXTRA, DatabaseURL from databases.interfaces import ConnectionBackend, DatabaseBackend, TransactionBackend @@ -107,9 +107,15 @@ async def fetch_all(self, query: ClauseElement) -> typing.List[typing.Mapping]: try: await cursor.execute(query, args) rows = await cursor.fetchall() - metadata = ResultMetaData(context, cursor.description) + metadata = CursorResultMetaData(context, cursor.description) return [ - RowProxy(metadata, row, metadata._processors, metadata._keymap) + Row( + metadata, + metadata._processors, + metadata._keymap, + Row._default_key_style, + row, + ) for row in rows ] finally: @@ -124,8 +130,14 @@ async def fetch_one(self, query: ClauseElement) -> typing.Optional[typing.Mappin row = await cursor.fetchone() if row is None: return None - metadata = ResultMetaData(context, cursor.description) - return RowProxy(metadata, row, metadata._processors, metadata._keymap) + metadata = CursorResultMetaData(context, cursor.description) + return Row( + metadata, + metadata._processors, + metadata._keymap, + Row._default_key_style, + row, + ) finally: await cursor.close() @@ -159,9 +171,15 @@ async def iterate( cursor = await self._connection.cursor() try: await cursor.execute(query, args) - metadata = ResultMetaData(context, cursor.description) + metadata = CursorResultMetaData(context, cursor.description) async for row in cursor: - yield RowProxy(metadata, row, metadata._processors, metadata._keymap) + yield Row( + metadata, + metadata._processors, + metadata._keymap, + Row._default_key_style, + row, + ) finally: await cursor.close() @@ -186,6 +204,7 @@ def _compile( compiled._result_columns, compiled._ordered_columns, compiled._textual_ordered_columns, + compiled._loose_column_name_matching, ) else: args = {} diff --git a/databases/backends/postgres.py b/databases/backends/postgres.py index 8c1d75b1..7d0f8e92 100644 --- a/databases/backends/postgres.py +++ b/databases/backends/postgres.py @@ -104,8 +104,27 @@ def __init__( self._dialect = dialect self._column_map, self._column_map_int, self._column_map_full = column_maps + @property + def _mapping(self) -> asyncpg.Record: + return self._row + + def keys(self) -> typing.KeysView: + import warnings + + warnings.warn( + "The `Row.keys()` method is deprecated to mimic SQLAlchemy behaviour, " + "use `Row._mapping.keys()` instead." + ) + return self._mapping.keys() + def values(self) -> typing.ValuesView: - return self._row.values() + import warnings + + warnings.warn( + "The `Row.values()` method is deprecated to mimic SQLAlchemy behaviour, " + "use `Row._mapping.values()` instead." + ) + return self._mapping.values() def __getitem__(self, key: typing.Any) -> typing.Any: if len(self._column_map) == 0: # raw query diff --git a/databases/backends/sqlite.py b/databases/backends/sqlite.py index 28ceb6fb..a7e30cc4 100644 --- a/databases/backends/sqlite.py +++ b/databases/backends/sqlite.py @@ -4,11 +4,11 @@ import aiosqlite from sqlalchemy.dialects.sqlite import pysqlite +from sqlalchemy.engine.cursor import CursorResultMetaData from sqlalchemy.engine.interfaces import Dialect, ExecutionContext -from sqlalchemy.engine.result import ResultMetaData, RowProxy +from sqlalchemy.engine.result import Row from sqlalchemy.sql import ClauseElement from sqlalchemy.sql.ddl import DDLElement -from sqlalchemy.types import TypeEngine from databases.core import LOG_EXTRA, DatabaseURL from databases.interfaces import ConnectionBackend, DatabaseBackend, TransactionBackend @@ -92,9 +92,15 @@ async def fetch_all(self, query: ClauseElement) -> typing.List[typing.Mapping]: async with self._connection.execute(query, args) as cursor: rows = await cursor.fetchall() - metadata = ResultMetaData(context, cursor.description) + metadata = CursorResultMetaData(context, cursor.description) return [ - RowProxy(metadata, row, metadata._processors, metadata._keymap) + Row( + metadata, + metadata._processors, + metadata._keymap, + Row._default_key_style, + row, + ) for row in rows ] @@ -106,8 +112,14 @@ async def fetch_one(self, query: ClauseElement) -> typing.Optional[typing.Mappin row = await cursor.fetchone() if row is None: return None - metadata = ResultMetaData(context, cursor.description) - return RowProxy(metadata, row, metadata._processors, metadata._keymap) + metadata = CursorResultMetaData(context, cursor.description) + return Row( + metadata, + metadata._processors, + metadata._keymap, + Row._default_key_style, + row, + ) async def execute(self, query: ClauseElement) -> typing.Any: assert self._connection is not None, "Connection is not acquired" @@ -129,9 +141,15 @@ async def iterate( assert self._connection is not None, "Connection is not acquired" query, args, context = self._compile(query) async with self._connection.execute(query, args) as cursor: - metadata = ResultMetaData(context, cursor.description) + metadata = CursorResultMetaData(context, cursor.description) async for row in cursor: - yield RowProxy(metadata, row, metadata._processors, metadata._keymap) + yield Row( + metadata, + metadata._processors, + metadata._keymap, + Row._default_key_style, + row, + ) def transaction(self) -> TransactionBackend: return SQLiteTransaction(self) @@ -158,6 +176,7 @@ def _compile( compiled._result_columns, compiled._ordered_columns, compiled._textual_ordered_columns, + compiled._loose_column_name_matching, ) query_message = compiled.string.replace(" \n", " ").replace("\n", " ") diff --git a/setup.py b/setup.py index 4cdd0fff..b4133b16 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def get_packages(package): packages=get_packages("databases"), package_data={"databases": ["py.typed"]}, data_files=[("", ["LICENSE.md"])], - install_requires=['sqlalchemy<1.4', 'aiocontextvars;python_version<"3.7"'], + install_requires=['sqlalchemy>=1.4,<1.5', 'aiocontextvars;python_version<"3.7"'], extras_require={ "postgresql": ["asyncpg"], "mysql": ["aiomysql"], diff --git a/tests/test_databases.py b/tests/test_databases.py index 5aae152a..2477960e 100644 --- a/tests/test_databases.py +++ b/tests/test_databases.py @@ -336,8 +336,8 @@ async def test_result_values_allow_duplicate_names(database_url): query = "SELECT 1 AS id, 2 AS id" row = await database.fetch_one(query=query) - assert list(row.keys()) == ["id", "id"] - assert list(row.values()) == [1, 2] + assert list(row._mapping.keys()) == ["id", "id"] + assert list(row._mapping.values()) == [1, 2] @pytest.mark.parametrize("database_url", DATABASE_URLS) @@ -981,7 +981,7 @@ async def test_iterate_outside_transaction_with_temp_table(database_url): @async_adapter async def test_column_names(database_url, select_query): """ - Test that column names are exposed correctly through `.keys()` on each row. + Test that column names are exposed correctly through `._mapping.keys()` on each row. """ async with Database(database_url) as database: async with database.transaction(force_rollback=True): @@ -993,7 +993,7 @@ async def test_column_names(database_url, select_query): results = await database.fetch_all(query=select_query) assert len(results) == 1 - assert sorted(results[0].keys()) == ["completed", "id", "text"] + assert sorted(results[0]._mapping.keys()) == ["completed", "id", "text"] assert results[0]["text"] == "example1" assert results[0]["completed"] == True From bce05db11ed3b7f251dc53b9166b87dc1ad998c2 Mon Sep 17 00:00:00 2001 From: PrettyWood Date: Thu, 13 May 2021 16:27:33 +0200 Subject: [PATCH 2/2] fix deprecation warning and add tests --- databases/backends/postgres.py | 6 +++-- tests/test_databases.py | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/databases/backends/postgres.py b/databases/backends/postgres.py index 7d0f8e92..73f998a2 100644 --- a/databases/backends/postgres.py +++ b/databases/backends/postgres.py @@ -113,7 +113,8 @@ def keys(self) -> typing.KeysView: warnings.warn( "The `Row.keys()` method is deprecated to mimic SQLAlchemy behaviour, " - "use `Row._mapping.keys()` instead." + "use `Row._mapping.keys()` instead.", + DeprecationWarning, ) return self._mapping.keys() @@ -122,7 +123,8 @@ def values(self) -> typing.ValuesView: warnings.warn( "The `Row.values()` method is deprecated to mimic SQLAlchemy behaviour, " - "use `Row._mapping.values()` instead." + "use `Row._mapping.values()` instead.", + DeprecationWarning, ) return self._mapping.values() diff --git a/tests/test_databases.py b/tests/test_databases.py index 2477960e..61568a7c 100644 --- a/tests/test_databases.py +++ b/tests/test_databases.py @@ -3,6 +3,7 @@ import decimal import functools import os +import re import pytest import sqlalchemy @@ -1014,3 +1015,49 @@ async def test_task(db): tasks = [test_task(database) for i in range(4)] await asyncio.gather(*tasks) + + +@pytest.mark.parametrize("database_url", DATABASE_URLS) +@async_adapter +async def test_posgres_interface(database_url): + """ + Since SQLAlchemy 1.4, `Row.values()` is removed and `Row.keys()` is deprecated. + Custom postgres interface mimics more or less this behaviour by deprecating those + two methods + """ + database_url = DatabaseURL(database_url) + + if database_url.scheme != "postgresql": + pytest.skip("Test is only for postgresql") + + async with Database(database_url) as database: + async with database.transaction(force_rollback=True): + query = notes.insert() + values = {"text": "example1", "completed": True} + await database.execute(query, values) + + query = notes.select() + result = await database.fetch_one(query=query) + + with pytest.warns( + DeprecationWarning, + match=re.escape( + "The `Row.keys()` method is deprecated to mimic SQLAlchemy behaviour, " + "use `Row._mapping.keys()` instead." + ), + ): + assert ( + list(result.keys()) + == [k for k in result] + == ["id", "text", "completed"] + ) + + with pytest.warns( + DeprecationWarning, + match=re.escape( + "The `Row.values()` method is deprecated to mimic SQLAlchemy behaviour, " + "use `Row._mapping.values()` instead." + ), + ): + # avoid checking `id` at index 0 since it may change depending on the launched tests + assert list(result.values())[1:] == ["example1", True]