From 3f94791f7bfa979eb1d6dd66c3224205e60d92b1 Mon Sep 17 00:00:00 2001 From: dosisod <39638017+dosisod@users.noreply.github.com> Date: Sat, 30 Oct 2021 19:47:47 -0700 Subject: [PATCH 1/8] Allow for passing keyword arguments to from_orm: You can already use Field(alias='whatever') on the model to allow for converting from ORM models that might be named differently, but this means that the model knows about the existence of a database (or at least it's columns), which seems like a leaky abstraction. With this modification though, the database will be the one to handle the naming conflict, which keeps the model more succinct. --- pydantic/main.py | 4 +++- tests/test_orm_mode.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pydantic/main.py b/pydantic/main.py index a25e96efea..75ad1cf22d 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -545,11 +545,13 @@ def parse_file( return cls.parse_obj(obj) @classmethod - def from_orm(cls: Type['Model'], obj: Any) -> 'Model': + def from_orm(cls: Type['Model'], obj: Any, **kwargs: Any) -> 'Model': if not cls.__config__.orm_mode: raise ConfigError('You must have the config attribute orm_mode=True to use from_orm') obj = {ROOT_KEY: obj} if cls.__custom_root_type__ else cls._decompose_class(obj) m = cls.__new__(cls) + if kwargs: + obj = {**obj, **kwargs} values, fields_set, validation_error = validate_model(cls, obj) if validation_error: raise validation_error diff --git a/tests/test_orm_mode.py b/tests/test_orm_mode.py index 67f44e9bd4..7bd4adba4e 100644 --- a/tests/test_orm_mode.py +++ b/tests/test_orm_mode.py @@ -337,3 +337,27 @@ class ModelB(Model): # test recursive parsing with dict keys obj = dict(bb=dict(aa=1)) assert ModelB.from_orm(obj) == ModelB(b=ModelA(a=1)) + + +def test_orm_mode_with_kwargs(): + class User(BaseModel): + username: str + name: str + + class Config: + orm_mode = True + + class UserCls: + username: str + full_name: str + + def __init__(self, username: str, full_name: str) -> None: + self.username = username + self.full_name = full_name + + user = UserCls('admin', 'john smith') + + converted = User.from_orm(user, name=user.full_name) + + assert user.username == converted.username + assert user.full_name == converted.name From c59f6c47f8b66744dca76418dcb129c6fe7819a2 Mon Sep 17 00:00:00 2001 From: dosisod <39638017+dosisod@users.noreply.github.com> Date: Sat, 30 Oct 2021 20:17:00 -0700 Subject: [PATCH 2/8] Add file to changes/ --- changes/3375-dosisod.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/3375-dosisod.md diff --git a/changes/3375-dosisod.md b/changes/3375-dosisod.md new file mode 100644 index 0000000000..a7eca2040e --- /dev/null +++ b/changes/3375-dosisod.md @@ -0,0 +1 @@ +Allow for passing keyword arguments to `from_orm` From 651fcceccaa4c23caec308828f0e20e50268eb58 Mon Sep 17 00:00:00 2001 From: dosisod <39638017+dosisod@users.noreply.github.com> Date: Sun, 12 Dec 2021 10:41:17 -0800 Subject: [PATCH 3/8] Add updated documentation with example --- docs/examples/models_orm_mode_kwargs.py | 31 +++++++++++++++++++++++++ docs/usage/models.md | 8 +++++++ 2 files changed, 39 insertions(+) create mode 100644 docs/examples/models_orm_mode_kwargs.py diff --git a/docs/examples/models_orm_mode_kwargs.py b/docs/examples/models_orm_mode_kwargs.py new file mode 100644 index 0000000000..b844f696f6 --- /dev/null +++ b/docs/examples/models_orm_mode_kwargs.py @@ -0,0 +1,31 @@ +import typing + +from pydantic import BaseModel +import sqlalchemy as sa +from sqlalchemy.ext.declarative import declarative_base + + +class MyModel(BaseModel): + metadata: typing.Dict[str, str] + + class Config: + orm_mode = True + + +BaseModel = declarative_base() + + +class SQLModel(BaseModel): + __tablename__ = 'my_table' + id = sa.Column('id', sa.Integer, primary_key=True) + # 'metadata' is reserved by SQLAlchemy, hence the '_' + metadata_ = sa.Column('metadata', sa.JSON) + + +sql_model = SQLModel(metadata_={'key': 'val'}, id=1) + +# notice that we are explicitly setting the value of 'metadata' +pydantic_model = MyModel.from_orm(sql_model, metadata=sql_model.metadata_) + +print(pydantic_model.dict()) +print(pydantic_model.dict(by_alias=True)) diff --git a/docs/usage/models.md b/docs/usage/models.md index 072038d415..440e1836d7 100644 --- a/docs/usage/models.md +++ b/docs/usage/models.md @@ -148,6 +148,14 @@ _(This script is complete, it should run "as is")_ The example above works because aliases have priority over field names for field population. Accessing `SQLModel`'s `metadata` attribute would lead to a `ValidationError`. +You can also achieve the same thing by passing keyword arguments to `from_orm`, instead of +using Field aliases: + +```py +{!.tmp_examples/models_orm_mode_kwargs.py!} +``` +_(This script is complete, it should run "as is")_ + ### Recursive ORM models ORM instances will be parsed with `from_orm` recursively as well as at the top level. From e4623f6d45edc7cdeda4c4fb19544af61587d690 Mon Sep 17 00:00:00 2001 From: dosisod <39638017+dosisod@users.noreply.github.com> Date: Fri, 5 Aug 2022 12:55:03 -0700 Subject: [PATCH 4/8] Fix properties being accessed when using kwargs --- pydantic/main.py | 8 +++----- pydantic/utils.py | 9 +++++---- tests/test_orm_mode.py | 20 ++++++++++++++++++++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pydantic/main.py b/pydantic/main.py index db2ff35475..09bb0fa230 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -571,10 +571,8 @@ def parse_file( def from_orm(cls: Type['Model'], obj: Any, **kwargs: Any) -> 'Model': if not cls.__config__.orm_mode: raise ConfigError('You must have the config attribute orm_mode=True to use from_orm') - obj = {ROOT_KEY: obj} if cls.__custom_root_type__ else cls._decompose_class(obj) + obj = {ROOT_KEY: obj} if cls.__custom_root_type__ else cls._decompose_class(obj, **kwargs) m = cls.__new__(cls) - if kwargs: - obj = {**obj, **kwargs} values, fields_set, validation_error = validate_model(cls, obj) if validation_error: raise validation_error @@ -700,10 +698,10 @@ def validate(cls: Type['Model'], value: Any) -> 'Model': return cls(**value_as_dict) @classmethod - def _decompose_class(cls: Type['Model'], obj: Any) -> GetterDict: + def _decompose_class(cls: Type['Model'], obj: Any, **kwargs: Any) -> GetterDict: if isinstance(obj, GetterDict): return obj - return cls.__config__.getter_dict(obj) + return cls.__config__.getter_dict(obj, **kwargs) @classmethod @no_type_check diff --git a/pydantic/utils.py b/pydantic/utils.py index 972f2e20ca..29a8d96bf5 100644 --- a/pydantic/utils.py +++ b/pydantic/utils.py @@ -413,19 +413,20 @@ class GetterDict(Representation): We can't inherit from Mapping[str, Any] because it upsets cython so we have to implement all methods ourselves. """ - __slots__ = ('_obj',) + __slots__ = ('_obj', '_kwargs') - def __init__(self, obj: Any): + def __init__(self, obj: Any, **kwargs: Any): self._obj = obj + self._kwargs = kwargs def __getitem__(self, key: str) -> Any: try: - return getattr(self._obj, key) + return self._kwargs.get(key, getattr(self._obj, key)) except AttributeError as e: raise KeyError(key) from e def get(self, key: Any, default: Any = None) -> Any: - return getattr(self._obj, key, default) + return self._kwargs.get(key, getattr(self._obj, key, default)) def extra_keys(self) -> Set[Any]: """ diff --git a/tests/test_orm_mode.py b/tests/test_orm_mode.py index 6cf0f91f54..ec80e3d659 100644 --- a/tests/test_orm_mode.py +++ b/tests/test_orm_mode.py @@ -381,3 +381,23 @@ def __init__(self, username: str, full_name: str) -> None: assert user.username == converted.username assert user.full_name == converted.name + + +def test_orm_mode_with_kwargs_property_wont_be_acessed(): + class Model(BaseModel): + a: int + + class Config: + orm_mode = True + + class Test: + a = 1 + + @property + def test(self): + self.was_called = True + + test = Test() + Model.from_orm(test, extra=42) + + assert not hasattr(test, 'was_called') From 0f6f698b20fe1182688058ed9cd26c555389d0ea Mon Sep 17 00:00:00 2001 From: dosisod <39638017+dosisod@users.noreply.github.com> Date: Mon, 8 Aug 2022 20:50:17 -0700 Subject: [PATCH 5/8] Add requested changes --- docs/examples/models_orm_mode_kwargs.py | 16 +++++++--------- pydantic/main.py | 6 +++--- pydantic/utils.py | 18 ++++++++++++++---- tests/test_orm_mode.py | 2 +- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/docs/examples/models_orm_mode_kwargs.py b/docs/examples/models_orm_mode_kwargs.py index b844f696f6..00b79c9d51 100644 --- a/docs/examples/models_orm_mode_kwargs.py +++ b/docs/examples/models_orm_mode_kwargs.py @@ -1,12 +1,12 @@ -import typing - from pydantic import BaseModel import sqlalchemy as sa from sqlalchemy.ext.declarative import declarative_base class MyModel(BaseModel): - metadata: typing.Dict[str, str] + foo: str + bar: int + spam: bytes class Config: orm_mode = True @@ -18,14 +18,12 @@ class Config: class SQLModel(BaseModel): __tablename__ = 'my_table' id = sa.Column('id', sa.Integer, primary_key=True) - # 'metadata' is reserved by SQLAlchemy, hence the '_' - metadata_ = sa.Column('metadata', sa.JSON) - + foo = sa.Column('metadata', sa.String(32)) + bar = sa.Column('metadata', sa.Integer) -sql_model = SQLModel(metadata_={'key': 'val'}, id=1) +sql_model = SQLModel(id=1, foo="hello world", bar=123) -# notice that we are explicitly setting the value of 'metadata' -pydantic_model = MyModel.from_orm(sql_model, metadata=sql_model.metadata_) +pydantic_model = MyModel.from_orm(sql_model, bar=456, spam=b"placeholder") print(pydantic_model.dict()) print(pydantic_model.dict(by_alias=True)) diff --git a/pydantic/main.py b/pydantic/main.py index 09bb0fa230..48c22dd421 100644 --- a/pydantic/main.py +++ b/pydantic/main.py @@ -571,7 +571,7 @@ def parse_file( def from_orm(cls: Type['Model'], obj: Any, **kwargs: Any) -> 'Model': if not cls.__config__.orm_mode: raise ConfigError('You must have the config attribute orm_mode=True to use from_orm') - obj = {ROOT_KEY: obj} if cls.__custom_root_type__ else cls._decompose_class(obj, **kwargs) + obj = {ROOT_KEY: obj} if cls.__custom_root_type__ else cls._decompose_class(obj, kwargs) m = cls.__new__(cls) values, fields_set, validation_error = validate_model(cls, obj) if validation_error: @@ -698,10 +698,10 @@ def validate(cls: Type['Model'], value: Any) -> 'Model': return cls(**value_as_dict) @classmethod - def _decompose_class(cls: Type['Model'], obj: Any, **kwargs: Any) -> GetterDict: + def _decompose_class(cls: Type['Model'], obj: Any, kwargs: Dict[str, Any]) -> GetterDict: if isinstance(obj, GetterDict): return obj - return cls.__config__.getter_dict(obj, **kwargs) + return cls.__config__.getter_dict(obj, kwargs) @classmethod @no_type_check diff --git a/pydantic/utils.py b/pydantic/utils.py index 29a8d96bf5..dc9b9441a7 100644 --- a/pydantic/utils.py +++ b/pydantic/utils.py @@ -415,18 +415,28 @@ class GetterDict(Representation): __slots__ = ('_obj', '_kwargs') - def __init__(self, obj: Any, **kwargs: Any): + def __init__(self, obj: Any, kwargs: Optional[Dict[str, Any]] = None): self._obj = obj - self._kwargs = kwargs + self._kwargs = kwargs or None def __getitem__(self, key: str) -> Any: try: - return self._kwargs.get(key, getattr(self._obj, key)) + if self._kwargs: + try: + return self._kwargs[key] + except KeyError: + pass + return getattr(self._obj, key) except AttributeError as e: raise KeyError(key) from e def get(self, key: Any, default: Any = None) -> Any: - return self._kwargs.get(key, getattr(self._obj, key, default)) + if self._kwargs: + try: + return self._kwargs[key] + except KeyError: + pass + return getattr(self._obj, key, default) def extra_keys(self) -> Set[Any]: """ diff --git a/tests/test_orm_mode.py b/tests/test_orm_mode.py index ec80e3d659..62531bdaf6 100644 --- a/tests/test_orm_mode.py +++ b/tests/test_orm_mode.py @@ -253,7 +253,7 @@ class TestCls: x = 1 y = 2 - def custom_getter_dict(obj): + def custom_getter_dict(obj, _): assert isinstance(obj, TestCls) return {'x': 42, 'y': 24} From 2ac817e71c75e3c0ba06e83cec9068df75c76265 Mon Sep 17 00:00:00 2001 From: dosisod <39638017+dosisod@users.noreply.github.com> Date: Mon, 8 Aug 2022 21:03:15 -0700 Subject: [PATCH 6/8] Fix docs --- docs/examples/models_orm_mode_kwargs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/examples/models_orm_mode_kwargs.py b/docs/examples/models_orm_mode_kwargs.py index 00b79c9d51..2610c08f09 100644 --- a/docs/examples/models_orm_mode_kwargs.py +++ b/docs/examples/models_orm_mode_kwargs.py @@ -18,12 +18,13 @@ class Config: class SQLModel(BaseModel): __tablename__ = 'my_table' id = sa.Column('id', sa.Integer, primary_key=True) - foo = sa.Column('metadata', sa.String(32)) - bar = sa.Column('metadata', sa.Integer) + foo = sa.Column('foo', sa.String(32)) + bar = sa.Column('bar', sa.Integer) -sql_model = SQLModel(id=1, foo="hello world", bar=123) -pydantic_model = MyModel.from_orm(sql_model, bar=456, spam=b"placeholder") +sql_model = SQLModel(id=1, foo='hello world', bar=123) + +pydantic_model = MyModel.from_orm(sql_model, bar=456, spam=b'placeholder') print(pydantic_model.dict()) print(pydantic_model.dict(by_alias=True)) From bcad29cf1ea37b7a24023155ab1f4d73aa4eacc2 Mon Sep 17 00:00:00 2001 From: dosisod <39638017+dosisod@users.noreply.github.com> Date: Mon, 15 Aug 2022 17:24:41 -0700 Subject: [PATCH 7/8] Add requested changes --- pydantic/utils.py | 7 ++----- tests/test_orm_mode.py | 38 ++++++++++++++++++++------------------ 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/pydantic/utils.py b/pydantic/utils.py index dc9b9441a7..d407fd7798 100644 --- a/pydantic/utils.py +++ b/pydantic/utils.py @@ -421,11 +421,8 @@ def __init__(self, obj: Any, kwargs: Optional[Dict[str, Any]] = None): def __getitem__(self, key: str) -> Any: try: - if self._kwargs: - try: - return self._kwargs[key] - except KeyError: - pass + if self._kwargs and key in self._kwargs: + return self._kwargs[key] return getattr(self._obj, key) except AttributeError as e: raise KeyError(key) from e diff --git a/tests/test_orm_mode.py b/tests/test_orm_mode.py index 62531bdaf6..ae837118c1 100644 --- a/tests/test_orm_mode.py +++ b/tests/test_orm_mode.py @@ -26,7 +26,7 @@ def __getattr__(self, key): raise AttributeError() t = TestCls() - gd = GetterDict(t) + gd = GetterDict(t, {'extra': 42}) assert gd.keys() == ['a', 'c', 'd'] assert gd.get('a') == 1 assert gd['a'] == 1 @@ -46,6 +46,7 @@ def __getattr__(self, key): assert len(gd) == 3 assert str(gd) == "{'a': 1, 'c': 3, 'd': 4}" assert repr(gd) == "GetterDict[TestCls]({'a': 1, 'c': 3, 'd': 4})" + assert gd['extra'] == 42 def test_orm_mode_root(): @@ -360,27 +361,25 @@ class Config: def test_orm_mode_with_kwargs(): - class User(BaseModel): - username: str - name: str + class MyModel(BaseModel): + foo: str + bar: int + spam: bytes class Config: orm_mode = True - class UserCls: - username: str - full_name: str - - def __init__(self, username: str, full_name: str) -> None: - self.username = username - self.full_name = full_name - - user = UserCls('admin', 'john smith') + class SQLModel(BaseModel): + id: int + foo: str + bar: int - converted = User.from_orm(user, name=user.full_name) + sql_model = SQLModel(id=1, foo='hello world', bar=123) + pydantic_model = MyModel.from_orm(sql_model, bar=456, spam=b'placeholder') - assert user.username == converted.username - assert user.full_name == converted.name + assert pydantic_model.foo == 'hello world' + assert pydantic_model.bar == 456 + assert pydantic_model.spam == b'placeholder' def test_orm_mode_with_kwargs_property_wont_be_acessed(): @@ -390,14 +389,17 @@ class Model(BaseModel): class Config: orm_mode = True + property_was_accessed = False + class Test: a = 1 @property def test(self): - self.was_called = True + nonlocal property_was_accessed + property_was_accessed = True test = Test() Model.from_orm(test, extra=42) - assert not hasattr(test, 'was_called') + assert property_was_accessed is False From 591d591810fe7af088a757d06dbff81674f25172 Mon Sep 17 00:00:00 2001 From: dosisod <39638017+dosisod@users.noreply.github.com> Date: Mon, 15 Aug 2022 17:38:15 -0700 Subject: [PATCH 8/8] Trigger CI