From 24e3af2b648f0758481d0cfa78678a8da448f3ac Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 25 Mar 2022 11:00:32 -0700 Subject: [PATCH] more from_prefixed_env features * support nested dict access with "__" separator * don't specify separator in prefix * catch exceptions for any loads function --- CHANGES.rst | 6 ++-- docs/config.rst | 50 +++++++++++++++++--------- src/flask/config.py | 84 +++++++++++++++++++++++++++++--------------- tests/test_config.py | 60 ++++++++++++++++++++++--------- 4 files changed, 136 insertions(+), 64 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 633e6ea7c4..d8f237211b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -52,8 +52,10 @@ Unreleased :issue:`4095, 4295, 4297` - Fix typing for ``__exit__`` methods for better compatibility with ``ExitStack``. :issue:`4474` -- Allow loading of prefixed environment variables into the Config. - :pr:`4479` +- Add ``Config.from_prefixed_env()`` to load config values from + environment variables that start with ``FLASK_`` or another prefix. + This parses values as JSON by default, and allows setting keys in + nested dicts. :pr:`4479` Version 2.0.3 diff --git a/docs/config.rst b/docs/config.rst index 9e66e01e50..c25bf83fa3 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -521,7 +521,8 @@ configuration values directly from the environment. Flask can be instructed to load all environment variables starting with a specific prefix into the config using :meth:`~flask.Config.from_prefixed_env`. -Environment variables can be set in the shell before starting the server: +Environment variables can be set in the shell before starting the +server: .. tabs:: @@ -561,30 +562,43 @@ Environment variables can be set in the shell before starting the server: > flask run * Running on http://127.0.0.1:5000/ -The variables can then be loaded and accessed via the config with a -key equal to the environment variable name without the prefix i.e. +The variables can then be loaded and accessed via the config with a key +equal to the environment variable name without the prefix i.e. .. code-block:: python app.config.from_prefixed_env() app.config["SECRET_KEY"] # Is "5f352379324c22463451387a0aec5d2f" -The prefix is ``FLASK_`` by default, however it is an configurable via -the ``prefix`` argument of :meth:`~flask.Config.from_prefixed_env`. +The prefix is ``FLASK_`` by default. This is configurable via the +``prefix`` argument of :meth:`~flask.Config.from_prefixed_env`. -Whilst the value of any environment variable is a string, it will be -parsed before being placed into the flask config. By default the -parsing is done by json.loads, however this is configurable via the -``loads`` argument of :meth:`~flask.Config.from_prefixed_env`. +Values will be parsed to attempt to convert them to a more specific type +than strings. By default :func:`json.loads` is used, so any valid JSON +value is possible, including lists and dicts. This is configurable via +the ``loads`` argument of :meth:`~flask.Config.from_prefixed_env`. -Notice that any value besides an empty string will be interpreted as a boolean -``True`` value in Python, which requires care if an environment explicitly sets -values intended to be ``False``. +When adding a boolean value with the default JSON parsing, only "true" +and "false", lowercase, are valid values. Keep in mind that any +non-empty string is considered ``True`` by Python. -Make sure to load the configuration very early on, so that extensions have the -ability to access the configuration when starting up. There are other methods -on the config object as well to load from individual files. For a complete -reference, read the :class:`~flask.Config` class documentation. +It is possible to set keys in nested dictionaries by separating the +keys with double underscore (``__``). Any intermediate keys that don't +exist on the parent dict will be initialized to an empty dict. + +.. code-block:: text + + $ export FLASK_MYAPI__credentials__username=user123 + +.. code-block:: python + + app.config["MYAPI"]["credentials"]["username"] # Is "user123" + +For even more config loading features, including merging, try a +dedicated library such as Dynaconf_, which includes integration with +Flask. + +.. _Dynaconf: https://www.dynaconf.com/ Configuration Best Practices @@ -604,6 +618,10 @@ that experience: limit yourself to request-only accesses to the configuration you can reconfigure the object later on as needed. +3. Make sure to load the configuration very early on, so that + extensions can access the configuration when calling ``init_app``. + + .. _config-dev-prod: Development / Production diff --git a/src/flask/config.py b/src/flask/config.py index c02db27218..a266ea1d0a 100644 --- a/src/flask/config.py +++ b/src/flask/config.py @@ -78,7 +78,7 @@ class Config(dict): """ def __init__(self, root_path: str, defaults: t.Optional[dict] = None) -> None: - dict.__init__(self, defaults or {}) + super().__init__(defaults or {}) self.root_path = root_path def from_envvar(self, variable_name: str, silent: bool = False) -> bool: @@ -106,42 +106,68 @@ def from_envvar(self, variable_name: str, silent: bool = False) -> bool: return self.from_pyfile(rv, silent=silent) def from_prefixed_env( - self, - prefix: str = "FLASK_", - *, - loads: t.Callable[[t.Union[str, bytes]], t.Any] = _json_loads, + self, prefix: str = "FLASK", *, loads: t.Callable[[str], t.Any] = json.loads ) -> bool: - """Updates the config from environment variables with the prefix. - - Calling this method will result in every environment variable - starting with **prefix** being placed into the configuration - without the **prefix**. The prefix is configurable as an - argument. Note that this method updates the existing config. + """Load any environment variables that start with ``FLASK_``, + dropping the prefix from the env key for the config key. Values + are passed through a loading function to attempt to convert them + to more specific types than strings. - For example if there is an environment variable - ``FLASK_SECRET_KEY`` with value ``secretly`` and the prefix is - ``FLASK_`` the config will contain the key ``SECRET_KEY`` with - the value ``secretly`` after calling this method. + Keys are loaded in :func:`sorted` order. - The value of the environment variable will be passed to the - **loads** parameter before being placed into the config. By - default **loads** utilises the stdlib json.loads to parse the - value, falling back to the value itself on parsing error. + The default loading function attempts to parse values as any + valid JSON type, including dicts and lists. - :param loads: A callable that takes a str (or bytes) returns - the parsed value. - :return: Always returns ``True``. + Specific items in nested dicts can be set by separating the + keys with double underscores (``__``). If an intermediate key + doesn't exist, it will be initialized to an empty dict. - .. versionadded:: 2.1.0 + :param prefix: Load env vars that start with this prefix, + separated with an underscore (``_``). + :param loads: Pass each string value to this function and use + the returned value as the config value. If any error is + raised it is ignored and the value remains a string. The + default is :func:`json.loads`. + .. versionadded:: 2.1 """ - mapping = {} - for raw_key, value in os.environ.items(): - if raw_key.startswith(prefix): - key = raw_key[len(prefix) :] # Use removeprefix with Python 3.9 - mapping[key] = loads(value) + prefix = f"{prefix}_" + len_prefix = len(prefix) + + for key in sorted(os.environ): + if not key.startswith(prefix): + continue + + value = os.environ[key] + + try: + value = loads(value) + except Exception: + # Keep the value as a string if loading failed. + pass + + # Change to key.removeprefix(prefix) on Python >= 3.9. + key = key[len_prefix:] - return self.from_mapping(mapping) + if "__" not in key: + # A non-nested key, set directly. + self[key] = value + continue + + # Traverse nested dictionaries with keys separated by "__". + current = self + *parts, tail = key.split("__") + + for part in parts: + # If an intermediate dict does not exist, create it. + if part not in current: + current[part] = {} + + current = current[part] + + current[tail] = value + + return True def from_pyfile(self, filename: str, silent: bool = False) -> bool: """Updates the values in the config from a Python file. This function diff --git a/tests/test_config.py b/tests/test_config.py index e2d5cd632a..22aacd3600 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -38,28 +38,54 @@ def test_config_from_file(): common_object_test(app) -def test_config_from_prefixed_env(monkeypatch): +def test_from_prefixed_env(monkeypatch): + monkeypatch.setenv("FLASK_STRING", "value") + monkeypatch.setenv("FLASK_BOOL", "true") + monkeypatch.setenv("FLASK_INT", "1") + monkeypatch.setenv("FLASK_FLOAT", "1.2") + monkeypatch.setenv("FLASK_LIST", "[1, 2]") + monkeypatch.setenv("FLASK_DICT", '{"k": "v"}') + monkeypatch.setenv("NOT_FLASK_OTHER", "other") + app = flask.Flask(__name__) - monkeypatch.setenv("FLASK_A", "A value") - monkeypatch.setenv("FLASK_B", "true") - monkeypatch.setenv("FLASK_C", "1") - monkeypatch.setenv("FLASK_D", "1.2") - monkeypatch.setenv("NOT_FLASK_A", "Another value") app.config.from_prefixed_env() - assert app.config["A"] == "A value" - assert app.config["B"] is True - assert app.config["C"] == 1 - assert app.config["D"] == 1.2 - assert "Another value" not in app.config.items() + assert app.config["STRING"] == "value" + assert app.config["BOOL"] is True + assert app.config["INT"] == 1 + assert app.config["FLOAT"] == 1.2 + assert app.config["LIST"] == [1, 2] + assert app.config["DICT"] == {"k": "v"} + assert "OTHER" not in app.config + + +def test_from_prefixed_env_custom_prefix(monkeypatch): + monkeypatch.setenv("FLASK_A", "a") + monkeypatch.setenv("NOT_FLASK_A", "b") + + app = flask.Flask(__name__) + app.config.from_prefixed_env("NOT_FLASK") + + assert app.config["A"] == "b" + + +def test_from_prefixed_env_nested(monkeypatch): + monkeypatch.setenv("FLASK_EXIST__ok", "other") + monkeypatch.setenv("FLASK_EXIST__inner__ik", "2") + monkeypatch.setenv("FLASK_EXIST__new__more", '{"k": false}') + monkeypatch.setenv("FLASK_NEW__K", "v") -def test_config_from_custom_prefixed_env(monkeypatch): app = flask.Flask(__name__) - monkeypatch.setenv("FLASK_A", "A value") - monkeypatch.setenv("NOT_FLASK_A", "Another value") - app.config.from_prefixed_env("NOT_FLASK_") - assert app.config["A"] == "Another value" - assert "A value" not in app.config.items() + app.config["EXIST"] = {"ok": "value", "flag": True, "inner": {"ik": 1}} + app.config.from_prefixed_env() + + assert app.config["EXIST"] == { + "ok": "other", + "flag": True, + "inner": {"ik": 2}, + "new": {"more": {"k": False}}, + } + assert app.config["NEW"] == {"K": "v"} def test_config_from_mapping():