From 5e7ea38f0deb7af1d23e8b153f48a729de757f5c Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Mon, 4 Apr 2022 16:37:53 -0700 Subject: [PATCH 01/13] Fix #1458 - Allow for custom parsing of environment variables via env_parse --- pydantic/env_settings.py | 19 +++++++--- tests/test_settings.py | 75 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/pydantic/env_settings.py b/pydantic/env_settings.py index 6be4b8e710..5cc489f7ee 100644 --- a/pydantic/env_settings.py +++ b/pydantic/env_settings.py @@ -188,12 +188,18 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: # noqa C901 if env_val_built: d[field.alias] = env_val_built else: - # field is complex and there's a value, decode that as JSON, then add explode_env_vars + # field is complex and there's a value, decode using the + # parsing function passed via env_parse or json_loads, then + # add explode_env_vars + parse_func: Callable[[str], Any] = field.field_info.extra.get( + 'env_parse', settings.__config__.json_loads + ) try: - env_val = settings.__config__.json_loads(env_val) + print(parse_func) + env_val = parse_func(env_val) except ValueError as e: if not allow_json_failure: - raise SettingsError(f'error parsing JSON for "{env_name}"') from e + raise SettingsError(f'error parsing envvar "{env_name}"') from e if isinstance(env_val, dict): d[field.alias] = deep_update(env_val, self.explode_env_vars(field, env_vars)) @@ -298,10 +304,13 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: if path.is_file(): secret_value = path.read_text().strip() if field.is_complex(): + parse_func: Callable[[str], Any] = field.field_info.extra.get( + 'env_parse', settings.__config__.json_loads + ) try: - secret_value = settings.__config__.json_loads(secret_value) + secret_value = parse_func(secret_value) except ValueError as e: - raise SettingsError(f'error parsing JSON for "{env_name}"') from e + raise SettingsError(f'error parsing envvar "{env_name}"') from e secrets[field.alias] = secret_value else: diff --git a/tests/test_settings.py b/tests/test_settings.py index e08fd08b6a..c309b7b323 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -3,11 +3,21 @@ import uuid from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union import pytest -from pydantic import BaseModel, BaseSettings, Field, HttpUrl, NoneStr, SecretStr, ValidationError, dataclasses +from pydantic import ( + BaseModel, + BaseSettings, + Field, + HttpUrl, + NoneStr, + SecretStr, + ValidationError, + dataclasses, + validator, +) from pydantic.env_settings import ( EnvSettingsSource, InitSettingsSource, @@ -221,7 +231,7 @@ def test_set_dict_model(env): def test_invalid_json(env): env.set('apples', '["russet", "granny smith",]') - with pytest.raises(SettingsError, match='error parsing JSON for "apples"'): + with pytest.raises(SettingsError, match='error parsing envvar "apples"'): ComplexSettings() @@ -1054,7 +1064,7 @@ class Settings(BaseSettings): class Config: secrets_dir = tmp_path - with pytest.raises(SettingsError, match='error parsing JSON for "foo"'): + with pytest.raises(SettingsError, match='error parsing envvar "foo"'): Settings() @@ -1215,3 +1225,60 @@ def test_builtins_settings_source_repr(): == "EnvSettingsSource(env_file='.env', env_file_encoding='utf-8', env_nested_delimiter=None)" ) assert repr(SecretsSettingsSource(secrets_dir='/secrets')) == "SecretsSettingsSource(secrets_dir='/secrets')" + + +def _parse_custom_dict(value: str) -> Callable[[str], Dict[int, str]]: + """A custom parsing function passed into env parsing test.""" + res = {} + for part in value.split(','): + k, v = part.split('=') + res[int(k)] = v + return res + + +def test_env_setting_source_custom_env_parse(env): + class Settings(BaseSettings): + top: Dict[int, str] = Field(env_parse=_parse_custom_dict) + + with pytest.raises(ValidationError): + Settings() + env.set('top', '1=apple,2=banana') + s = Settings() + assert s.top == {1: 'apple', 2: 'banana'} + + +def test_env_settings_source_custom_env_parse_is_bad(env): + class Settings(BaseSettings): + top: Dict[int, str] = Field(env_parse=int) + + env.set('top', '1=apple,2=banana') + with pytest.raises(SettingsError, match='error parsing envvar "top"'): + Settings() + + +def test_env_settings_source_custom_env_parse_validator(env): + class Settings(BaseSettings): + top: Dict[int, str] = Field(env_parse=str) + + @validator('top', pre=True) + def val_top(cls, v): + assert isinstance(v, str) + return _parse_custom_dict(v) + + env.set('top', '1=apple,2=banana') + s = Settings() + assert s.top == {1: 'apple', 2: 'banana'} + + +def test_secret_settings_source_custom_env_parse(tmp_path): + p = tmp_path / 'top' + p.write_text('1=apple,2=banana') + + class Settings(BaseSettings): + top: Dict[int, str] = Field(env_parse=_parse_custom_dict) + + class Config: + secrets_dir = tmp_path + + s = Settings() + assert s.top == {1: 'apple', 2: 'banana'} From 32b7c5bc59b0f8d3af2410559b991932abd275a7 Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Mon, 4 Apr 2022 17:16:54 -0700 Subject: [PATCH 02/13] Add docs for env_parse usage --- docs/examples/settings_with_custom_parsing.py | 17 +++++++++++++++ .../settings_with_custom_parsing_validator.py | 17 +++++++++++++++ docs/usage/settings.md | 21 ++++++++++++++++--- 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 docs/examples/settings_with_custom_parsing.py create mode 100644 docs/examples/settings_with_custom_parsing_validator.py diff --git a/docs/examples/settings_with_custom_parsing.py b/docs/examples/settings_with_custom_parsing.py new file mode 100644 index 0000000000..219c87aa48 --- /dev/null +++ b/docs/examples/settings_with_custom_parsing.py @@ -0,0 +1,17 @@ +# output-json +import os +from typing import List + +from pydantic import BaseSettings, Field + + +def parse_list(s: str) -> List[int]: + return [int(x.strip()) for x in s.split(',')] + + +class Settings(BaseSettings): + numbers: List[int] = Field(env_parse=parse_list) + + +os.environ['numbers'] = '1,2,3' +print(Settings().dict()) diff --git a/docs/examples/settings_with_custom_parsing_validator.py b/docs/examples/settings_with_custom_parsing_validator.py new file mode 100644 index 0000000000..a0e4dcf148 --- /dev/null +++ b/docs/examples/settings_with_custom_parsing_validator.py @@ -0,0 +1,17 @@ +# output-json +import os +from typing import List + +from pydantic import BaseSettings, Field, validator + + +class Settings(BaseSettings): + numbers: List[int] = Field(env_parse=str) + + @validator('numbers', pre=True) + def validate_numbers(cls, s): + return [int(x.strip()) for x in s.split(',')] + + +os.environ['numbers'] = '1,2,3' +print(Settings().dict()) diff --git a/docs/usage/settings.md b/docs/usage/settings.md index 1ba926f6eb..6de06748a9 100644 --- a/docs/usage/settings.md +++ b/docs/usage/settings.md @@ -87,7 +87,7 @@ export SUB_MODEL__DEEP__V4=v4 You could load a settings module thus: {!.tmp_examples/settings_nested_env.md!} -`env_nested_delimiter` can be configured via the `Config` class as shown above, or via the +`env_nested_delimiter` can be configured via the `Config` class as shown above, or via the `_env_nested_delimiter` keyword argument on instantiation. JSON is only parsed in top-level fields, if you need to parse JSON in sub-models, you will need to implement @@ -96,6 +96,21 @@ validators on those models. Nested environment variables take precedence over the top-level environment variable JSON (e.g. in the example above, `SUB_MODEL__V2` trumps `SUB_MODEL`). +You may also populate a complex type by providing your own parsing function to +`env_parse` in the field extras. + +```py +{!.tmp_examples/settings_with_custom_parsing.py!} +``` + +You might choose to pass the environment string value through to pydantic and +transform in a validator instead. + +```py +{!.tmp_examples/settings_with_custom_parsing_validator.py!} +``` + + ## Dotenv (.env) support !!! note @@ -178,7 +193,7 @@ see [python-dotenv's documentation](https://saurabh-kumar.com/python-dotenv/#usa Placing secret values in files is a common pattern to provide sensitive configuration to an application. -A secret file follows the same principal as a dotenv file except it only contains a single value and the file name +A secret file follows the same principal as a dotenv file except it only contains a single value and the file name is used as the key. A secret file will look like the following: `/var/run/database_password`: @@ -231,7 +246,7 @@ class Settings(BaseSettings): secrets_dir = '/run/secrets' ``` !!! note - By default Docker uses `/run/secrets` as the target mount point. If you want to use a different location, change + By default Docker uses `/run/secrets` as the target mount point. If you want to use a different location, change `Config.secrets_dir` accordingly. Then, create your secret via the Docker CLI From 3ef50b90126df8978b68db91f50b42ef5f9f0a73 Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Mon, 4 Apr 2022 17:29:57 -0700 Subject: [PATCH 03/13] Add changes file for #3977 --- changes/3977-acmiyaguchi.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/3977-acmiyaguchi.md diff --git a/changes/3977-acmiyaguchi.md b/changes/3977-acmiyaguchi.md new file mode 100644 index 0000000000..da15e7259f --- /dev/null +++ b/changes/3977-acmiyaguchi.md @@ -0,0 +1 @@ +Allow for custom parsing of environment variables via `env_parse` in `Field`. From 85cfd48b65515056a60027358b9e2a2da3160398 Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Tue, 5 Apr 2022 09:25:00 -0700 Subject: [PATCH 04/13] fixup: remove stray print statement --- pydantic/env_settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pydantic/env_settings.py b/pydantic/env_settings.py index 5cc489f7ee..f0f9e844c2 100644 --- a/pydantic/env_settings.py +++ b/pydantic/env_settings.py @@ -195,7 +195,6 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: # noqa C901 'env_parse', settings.__config__.json_loads ) try: - print(parse_func) env_val = parse_func(env_val) except ValueError as e: if not allow_json_failure: From 6d19f7ea930ffe7f5dc0f8fa32fa8ed258aa30d4 Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Fri, 19 Aug 2022 14:08:36 -0700 Subject: [PATCH 05/13] Revert env_parse property on field --- pydantic/env_settings.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pydantic/env_settings.py b/pydantic/env_settings.py index f0f9e844c2..c464bebe9f 100644 --- a/pydantic/env_settings.py +++ b/pydantic/env_settings.py @@ -188,14 +188,9 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: # noqa C901 if env_val_built: d[field.alias] = env_val_built else: - # field is complex and there's a value, decode using the - # parsing function passed via env_parse or json_loads, then - # add explode_env_vars - parse_func: Callable[[str], Any] = field.field_info.extra.get( - 'env_parse', settings.__config__.json_loads - ) + # field is complex and there's a value, decode that as JSON, then add explode_env_vars try: - env_val = parse_func(env_val) + env_val = settings.__config__.json_loads(env_val) except ValueError as e: if not allow_json_failure: raise SettingsError(f'error parsing envvar "{env_name}"') from e @@ -303,11 +298,8 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: if path.is_file(): secret_value = path.read_text().strip() if field.is_complex(): - parse_func: Callable[[str], Any] = field.field_info.extra.get( - 'env_parse', settings.__config__.json_loads - ) try: - secret_value = parse_func(secret_value) + secret_value = settings.__config__.json_loads(secret_value) except ValueError as e: raise SettingsError(f'error parsing envvar "{env_name}"') from e From 7e1579c21ff54f044e5c6eab93fdb7e4f60419cc Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Fri, 19 Aug 2022 14:31:38 -0700 Subject: [PATCH 06/13] Add parse_env_var classmethod in nested Config --- pydantic/env_settings.py | 8 +++++-- tests/test_settings.py | 50 ++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/pydantic/env_settings.py b/pydantic/env_settings.py index c464bebe9f..c4cdedb65d 100644 --- a/pydantic/env_settings.py +++ b/pydantic/env_settings.py @@ -126,6 +126,10 @@ def customise_sources( ) -> Tuple[SettingsSourceCallable, ...]: return init_settings, env_settings, file_secret_settings + @classmethod + def parse_env_var(cls, field_name: str, raw_val: str) -> Any: + return cls.json_loads(raw_val) + # populated by the metaclass using the Config class defined above, annotated here to help IDEs only __config__: ClassVar[Type[Config]] @@ -190,7 +194,7 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: # noqa C901 else: # field is complex and there's a value, decode that as JSON, then add explode_env_vars try: - env_val = settings.__config__.json_loads(env_val) + env_val = settings.__config__.parse_env_var(field.name, env_val) except ValueError as e: if not allow_json_failure: raise SettingsError(f'error parsing envvar "{env_name}"') from e @@ -299,7 +303,7 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: secret_value = path.read_text().strip() if field.is_complex(): try: - secret_value = settings.__config__.json_loads(secret_value) + secret_value = settings.__config__.parse_env_var(field.name, secret_value) except ValueError as e: raise SettingsError(f'error parsing envvar "{env_name}"') from e diff --git a/tests/test_settings.py b/tests/test_settings.py index c309b7b323..933130c224 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -7,17 +7,7 @@ import pytest -from pydantic import ( - BaseModel, - BaseSettings, - Field, - HttpUrl, - NoneStr, - SecretStr, - ValidationError, - dataclasses, - validator, -) +from pydantic import BaseModel, BaseSettings, Field, HttpUrl, NoneStr, SecretStr, ValidationError, dataclasses from pydantic.env_settings import ( EnvSettingsSource, InitSettingsSource, @@ -1238,7 +1228,14 @@ def _parse_custom_dict(value: str) -> Callable[[str], Dict[int, str]]: def test_env_setting_source_custom_env_parse(env): class Settings(BaseSettings): - top: Dict[int, str] = Field(env_parse=_parse_custom_dict) + top: Dict[int, str] + + class Config: + @classmethod + def parse_env_var(cls, field_name: str, raw_val: str): + if field_name == 'top': + return _parse_custom_dict(raw_val) + return cls.json_loads(raw_val) with pytest.raises(ValidationError): Settings() @@ -1249,27 +1246,20 @@ class Settings(BaseSettings): def test_env_settings_source_custom_env_parse_is_bad(env): class Settings(BaseSettings): - top: Dict[int, str] = Field(env_parse=int) + top: Dict[int, str] + + class Config: + @classmethod + def parse_env_var(cls, field_name: str, raw_val: str): + if field_name == 'top': + return int(raw_val) + return cls.json_loads(raw_val) env.set('top', '1=apple,2=banana') with pytest.raises(SettingsError, match='error parsing envvar "top"'): Settings() -def test_env_settings_source_custom_env_parse_validator(env): - class Settings(BaseSettings): - top: Dict[int, str] = Field(env_parse=str) - - @validator('top', pre=True) - def val_top(cls, v): - assert isinstance(v, str) - return _parse_custom_dict(v) - - env.set('top', '1=apple,2=banana') - s = Settings() - assert s.top == {1: 'apple', 2: 'banana'} - - def test_secret_settings_source_custom_env_parse(tmp_path): p = tmp_path / 'top' p.write_text('1=apple,2=banana') @@ -1280,5 +1270,11 @@ class Settings(BaseSettings): class Config: secrets_dir = tmp_path + @classmethod + def parse_env_var(cls, field_name: str, raw_val: str): + if field_name == 'top': + return _parse_custom_dict(raw_val) + return cls.json_loads(raw_val) + s = Settings() assert s.top == {1: 'apple', 2: 'banana'} From e61a30f2dd6b2cff00354db92fc25fb26ab5e1bc Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Fri, 19 Aug 2022 14:36:44 -0700 Subject: [PATCH 07/13] Update documentation for parse_env_var --- docs/examples/settings_with_custom_parsing.py | 14 ++++++++++---- .../settings_with_custom_parsing_validator.py | 17 ----------------- docs/usage/settings.md | 10 +--------- 3 files changed, 11 insertions(+), 30 deletions(-) delete mode 100644 docs/examples/settings_with_custom_parsing_validator.py diff --git a/docs/examples/settings_with_custom_parsing.py b/docs/examples/settings_with_custom_parsing.py index 219c87aa48..6b06e504bb 100644 --- a/docs/examples/settings_with_custom_parsing.py +++ b/docs/examples/settings_with_custom_parsing.py @@ -1,8 +1,7 @@ # output-json -import os -from typing import List +from typing import Any, List -from pydantic import BaseSettings, Field +from pydantic import BaseSettings def parse_list(s: str) -> List[int]: @@ -10,7 +9,14 @@ def parse_list(s: str) -> List[int]: class Settings(BaseSettings): - numbers: List[int] = Field(env_parse=parse_list) + numbers: List[int] + + class Config: + @classmethod + def parse_env_var(cls, field_name: str, raw_val: str) -> Any: + if field_name == "numbers": + return parse_list(raw_val) + return cls.json_loads(raw_val) os.environ['numbers'] = '1,2,3' diff --git a/docs/examples/settings_with_custom_parsing_validator.py b/docs/examples/settings_with_custom_parsing_validator.py deleted file mode 100644 index a0e4dcf148..0000000000 --- a/docs/examples/settings_with_custom_parsing_validator.py +++ /dev/null @@ -1,17 +0,0 @@ -# output-json -import os -from typing import List - -from pydantic import BaseSettings, Field, validator - - -class Settings(BaseSettings): - numbers: List[int] = Field(env_parse=str) - - @validator('numbers', pre=True) - def validate_numbers(cls, s): - return [int(x.strip()) for x in s.split(',')] - - -os.environ['numbers'] = '1,2,3' -print(Settings().dict()) diff --git a/docs/usage/settings.md b/docs/usage/settings.md index 6de06748a9..fbee80c9ba 100644 --- a/docs/usage/settings.md +++ b/docs/usage/settings.md @@ -97,20 +97,12 @@ Nested environment variables take precedence over the top-level environment vari (e.g. in the example above, `SUB_MODEL__V2` trumps `SUB_MODEL`). You may also populate a complex type by providing your own parsing function to -`env_parse` in the field extras. +the `parse_env_var` classmethod in the Config object. ```py {!.tmp_examples/settings_with_custom_parsing.py!} ``` -You might choose to pass the environment string value through to pydantic and -transform in a validator instead. - -```py -{!.tmp_examples/settings_with_custom_parsing_validator.py!} -``` - - ## Dotenv (.env) support !!! note From 955625a836847b167bb41107478d5a034f4849dd Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Fri, 19 Aug 2022 14:40:46 -0700 Subject: [PATCH 08/13] Update changes file. --- changes/3977-acmiyaguchi.md | 1 - changes/4406-acmiyaguchi.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changes/3977-acmiyaguchi.md create mode 100644 changes/4406-acmiyaguchi.md diff --git a/changes/3977-acmiyaguchi.md b/changes/3977-acmiyaguchi.md deleted file mode 100644 index da15e7259f..0000000000 --- a/changes/3977-acmiyaguchi.md +++ /dev/null @@ -1 +0,0 @@ -Allow for custom parsing of environment variables via `env_parse` in `Field`. diff --git a/changes/4406-acmiyaguchi.md b/changes/4406-acmiyaguchi.md new file mode 100644 index 0000000000..c7028c4d15 --- /dev/null +++ b/changes/4406-acmiyaguchi.md @@ -0,0 +1 @@ +Allow for custom parsing of environment variables via `parse_env_var` in `Config`. From 0a9e91793357af6048d0c32f738034d3c927b087 Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Fri, 19 Aug 2022 14:42:09 -0700 Subject: [PATCH 09/13] fixup: linting in example --- docs/examples/settings_with_custom_parsing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/examples/settings_with_custom_parsing.py b/docs/examples/settings_with_custom_parsing.py index 6b06e504bb..245f1747e6 100644 --- a/docs/examples/settings_with_custom_parsing.py +++ b/docs/examples/settings_with_custom_parsing.py @@ -1,4 +1,5 @@ # output-json +import os from typing import Any, List from pydantic import BaseSettings @@ -14,7 +15,7 @@ class Settings(BaseSettings): class Config: @classmethod def parse_env_var(cls, field_name: str, raw_val: str) -> Any: - if field_name == "numbers": + if field_name == 'numbers': return parse_list(raw_val) return cls.json_loads(raw_val) From 4b2c372bb38008093ee2f949695e4e31f8a39936 Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Fri, 19 Aug 2022 14:53:00 -0700 Subject: [PATCH 10/13] Rebase and remove quotes around imported example --- docs/usage/settings.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/usage/settings.md b/docs/usage/settings.md index fbee80c9ba..be9a3b7f5e 100644 --- a/docs/usage/settings.md +++ b/docs/usage/settings.md @@ -99,9 +99,7 @@ Nested environment variables take precedence over the top-level environment vari You may also populate a complex type by providing your own parsing function to the `parse_env_var` classmethod in the Config object. -```py {!.tmp_examples/settings_with_custom_parsing.py!} -``` ## Dotenv (.env) support From 1d6e6c00f6f22eb52b58ffd094b5ec6646cdde69 Mon Sep 17 00:00:00 2001 From: Anthony Miyaguchi Date: Fri, 19 Aug 2022 16:20:32 -0700 Subject: [PATCH 11/13] fix example --- docs/usage/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/settings.md b/docs/usage/settings.md index be9a3b7f5e..7268e9f0fa 100644 --- a/docs/usage/settings.md +++ b/docs/usage/settings.md @@ -99,7 +99,7 @@ Nested environment variables take precedence over the top-level environment vari You may also populate a complex type by providing your own parsing function to the `parse_env_var` classmethod in the Config object. -{!.tmp_examples/settings_with_custom_parsing.py!} +{!.tmp_examples/settings_with_custom_parsing.md!} ## Dotenv (.env) support From 8cff96911bb3fbf7fac72d85213f6f791f0cc3f6 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Mon, 22 Aug 2022 15:40:03 +0100 Subject: [PATCH 12/13] my suggestions --- docs/examples/settings_with_custom_parsing.py | 7 +------ pydantic/env_settings.py | 14 +++++++------- tests/test_settings.py | 6 +++--- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/docs/examples/settings_with_custom_parsing.py b/docs/examples/settings_with_custom_parsing.py index 245f1747e6..9de25643e1 100644 --- a/docs/examples/settings_with_custom_parsing.py +++ b/docs/examples/settings_with_custom_parsing.py @@ -1,14 +1,9 @@ -# output-json import os from typing import Any, List from pydantic import BaseSettings -def parse_list(s: str) -> List[int]: - return [int(x.strip()) for x in s.split(',')] - - class Settings(BaseSettings): numbers: List[int] @@ -16,7 +11,7 @@ class Config: @classmethod def parse_env_var(cls, field_name: str, raw_val: str) -> Any: if field_name == 'numbers': - return parse_list(raw_val) + return [int(x) for x in raw_val.split(',')] return cls.json_loads(raw_val) diff --git a/pydantic/env_settings.py b/pydantic/env_settings.py index c4cdedb65d..7587a1f50c 100644 --- a/pydantic/env_settings.py +++ b/pydantic/env_settings.py @@ -184,7 +184,7 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: # noqa C901 if env_val is not None: break - is_complex, allow_json_failure = self.field_is_complex(field) + is_complex, allow_parse_failure = self.field_is_complex(field) if is_complex: if env_val is None: # field is complex but no value found so far, try explode_env_vars @@ -196,8 +196,8 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: # noqa C901 try: env_val = settings.__config__.parse_env_var(field.name, env_val) except ValueError as e: - if not allow_json_failure: - raise SettingsError(f'error parsing envvar "{env_name}"') from e + if not allow_parse_failure: + raise SettingsError(f'error parsing env var "{env_name}"') from e if isinstance(env_val, dict): d[field.alias] = deep_update(env_val, self.explode_env_vars(field, env_vars)) @@ -232,13 +232,13 @@ def field_is_complex(self, field: ModelField) -> Tuple[bool, bool]: Find out if a field is complex, and if so whether JSON errors should be ignored """ if field.is_complex(): - allow_json_failure = False + allow_parse_failure = False elif is_union(get_origin(field.type_)) and field.sub_fields and any(f.is_complex() for f in field.sub_fields): - allow_json_failure = True + allow_parse_failure = True else: return False, False - return True, allow_json_failure + return True, allow_parse_failure def explode_env_vars(self, field: ModelField, env_vars: Mapping[str, Optional[str]]) -> Dict[str, Any]: """ @@ -305,7 +305,7 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]: try: secret_value = settings.__config__.parse_env_var(field.name, secret_value) except ValueError as e: - raise SettingsError(f'error parsing envvar "{env_name}"') from e + raise SettingsError(f'error parsing env var "{env_name}"') from e secrets[field.alias] = secret_value else: diff --git a/tests/test_settings.py b/tests/test_settings.py index 933130c224..54f28b3016 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -221,7 +221,7 @@ def test_set_dict_model(env): def test_invalid_json(env): env.set('apples', '["russet", "granny smith",]') - with pytest.raises(SettingsError, match='error parsing envvar "apples"'): + with pytest.raises(SettingsError, match='error parsing env var "apples"'): ComplexSettings() @@ -1054,7 +1054,7 @@ class Settings(BaseSettings): class Config: secrets_dir = tmp_path - with pytest.raises(SettingsError, match='error parsing envvar "foo"'): + with pytest.raises(SettingsError, match='error parsing env var "foo"'): Settings() @@ -1256,7 +1256,7 @@ def parse_env_var(cls, field_name: str, raw_val: str): return cls.json_loads(raw_val) env.set('top', '1=apple,2=banana') - with pytest.raises(SettingsError, match='error parsing envvar "top"'): + with pytest.raises(SettingsError, match='error parsing env var "top"'): Settings() From 482cf3a6a9b62ff96bb21c7d4309235fb9adc235 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Mon, 22 Aug 2022 16:52:32 +0100 Subject: [PATCH 13/13] remove unnecessary Field(env_parse=_parse_custom_dict) --- tests/test_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 54f28b3016..d61e6209bc 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1265,7 +1265,7 @@ def test_secret_settings_source_custom_env_parse(tmp_path): p.write_text('1=apple,2=banana') class Settings(BaseSettings): - top: Dict[int, str] = Field(env_parse=_parse_custom_dict) + top: Dict[int, str] class Config: secrets_dir = tmp_path