From ee61cc0eb2f6d4939cafea70188c76935e258db5 Mon Sep 17 00:00:00 2001 From: Bertrand Bonnefoy-Claudet Date: Sun, 15 Nov 2020 15:26:26 +0100 Subject: [PATCH 1/3] Fix pypy environment in Travis CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8f51de38..8ccd2405 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,7 @@ script: - tox before_install: + - pip install --upgrade pip - pip install coveralls after_success: From ab50c82d71c2e0bc87204102b6fe4d25854ae3dc Mon Sep 17 00:00:00 2001 From: Bertrand Bonnefoy-Claudet Date: Sun, 15 Nov 2020 13:06:25 +0100 Subject: [PATCH 2/3] Decouple variable parsing and expansion This is now done in two steps: - Parse the value into a sequence of atoms (literal of variable). - Resolve that sequence into a string. --- src/dotenv/main.py | 59 ++++++++-------------- src/dotenv/variables.py | 106 ++++++++++++++++++++++++++++++++++++++++ tests/test_variables.py | 35 +++++++++++++ 3 files changed, 162 insertions(+), 38 deletions(-) create mode 100644 src/dotenv/variables.py create mode 100644 tests/test_variables.py diff --git a/src/dotenv/main.py b/src/dotenv/main.py index 58a23f3d..ea523d48 100644 --- a/src/dotenv/main.py +++ b/src/dotenv/main.py @@ -4,7 +4,6 @@ import io import logging import os -import re import shutil import sys import tempfile @@ -13,13 +12,13 @@ from .compat import IS_TYPE_CHECKING, PY2, StringIO, to_env from .parser import Binding, parse_stream +from .variables import parse_variables logger = logging.getLogger(__name__) if IS_TYPE_CHECKING: - from typing import ( - Dict, Iterable, Iterator, Match, Optional, Pattern, Union, Text, IO, Tuple - ) + from typing import (IO, Dict, Iterable, Iterator, Mapping, Optional, Text, + Tuple, Union) if sys.version_info >= (3, 6): _PathLike = os.PathLike else: @@ -30,18 +29,6 @@ else: _StringIO = StringIO[Text] -__posix_variable = re.compile( - r""" - \$\{ - (?P[^\}:]*) - (?::- - (?P[^\}]*) - )? - \} - """, - re.VERBOSE, -) # type: Pattern[Text] - def with_warn_for_invalid_lines(mappings): # type: (Iterator[Binding]) -> Iterator[Binding] @@ -83,13 +70,14 @@ def dict(self): if self._dict: return self._dict + raw_values = self.parse() + if self.interpolate: - values = resolve_nested_variables(self.parse()) + self._dict = OrderedDict(resolve_variables(raw_values)) else: - values = OrderedDict(self.parse()) + self._dict = OrderedDict(raw_values) - self._dict = values - return values + return self._dict def parse(self): # type: () -> Iterator[Tuple[Text, Optional[Text]]] @@ -217,27 +205,22 @@ def unset_key(dotenv_path, key_to_unset, quote_mode="always"): return removed, key_to_unset -def resolve_nested_variables(values): - # type: (Iterable[Tuple[Text, Optional[Text]]]) -> Dict[Text, Optional[Text]] - def _replacement(name, default): - # type: (Text, Optional[Text]) -> Text - default = default if default is not None else "" - ret = new_values.get(name, os.getenv(name, default)) - return ret # type: ignore +def resolve_variables(values): + # type: (Iterable[Tuple[Text, Optional[Text]]]) -> Mapping[Text, Optional[Text]] - def _re_sub_callback(match): - # type: (Match[Text]) -> Text - """ - From a match object gets the variable name and returns - the correct replacement - """ - matches = match.groupdict() - return _replacement(name=matches["name"], default=matches["default"]) # type: ignore + new_values = {} # type: Dict[Text, Optional[Text]] - new_values = {} + for (name, value) in values: + if value is None: + result = None + else: + atoms = parse_variables(value) + env = {} # type: Dict[Text, Optional[Text]] + env.update(os.environ) # type: ignore + env.update(new_values) + result = "".join(atom.resolve(env) for atom in atoms) - for (k, v) in values: - new_values[k] = __posix_variable.sub(_re_sub_callback, v) if v is not None else None + new_values[name] = result return new_values diff --git a/src/dotenv/variables.py b/src/dotenv/variables.py new file mode 100644 index 00000000..4828dfc2 --- /dev/null +++ b/src/dotenv/variables.py @@ -0,0 +1,106 @@ +import re +from abc import ABCMeta + +from .compat import IS_TYPE_CHECKING + +if IS_TYPE_CHECKING: + from typing import Iterator, Mapping, Optional, Pattern, Text + + +_posix_variable = re.compile( + r""" + \$\{ + (?P[^\}:]*) + (?::- + (?P[^\}]*) + )? + \} + """, + re.VERBOSE, +) # type: Pattern[Text] + + +class Atom(): + __metaclass__ = ABCMeta + + def __ne__(self, other): + # type: (object) -> bool + result = self.__eq__(other) + if result is NotImplemented: + return NotImplemented + return not result + + def resolve(self, env): + # type: (Mapping[Text, Optional[Text]]) -> Text + raise NotImplementedError + + +class Literal(Atom): + def __init__(self, value): + # type: (Text) -> None + self.value = value + + def __repr__(self): + # type: () -> str + return "Literal(value={})".format(self.value) + + def __eq__(self, other): + # type: (object) -> bool + if not isinstance(other, self.__class__): + return NotImplemented + return self.value == other.value + + def __hash__(self): + # type: () -> int + return hash((self.__class__, self.value)) + + def resolve(self, env): + # type: (Mapping[Text, Optional[Text]]) -> Text + return self.value + + +class Variable(Atom): + def __init__(self, name, default): + # type: (Text, Optional[Text]) -> None + self.name = name + self.default = default + + def __repr__(self): + # type: () -> str + return "Variable(name={}, default={})".format(self.name, self.default) + + def __eq__(self, other): + # type: (object) -> bool + if not isinstance(other, self.__class__): + return NotImplemented + return (self.name, self.default) == (other.name, other.default) + + def __hash__(self): + # type: () -> int + return hash((self.__class__, self.name, self.default)) + + def resolve(self, env): + # type: (Mapping[Text, Optional[Text]]) -> Text + default = self.default if self.default is not None else "" + result = env.get(self.name, default) + return result if result is not None else "" + + +def parse_variables(value): + # type: (Text) -> Iterator[Atom] + cursor = 0 + + for match in _posix_variable.finditer(value): + (start, end) = match.span() + name = match.groupdict()["name"] + default = match.groupdict()["default"] + + if start > cursor: + yield Literal(value=value[cursor:start]) + + yield Variable(name=name, default=default) + cursor = end + + length = len(value) + if cursor < length: + yield Literal(value=value[cursor:length]) diff --git a/tests/test_variables.py b/tests/test_variables.py new file mode 100644 index 00000000..86b06466 --- /dev/null +++ b/tests/test_variables.py @@ -0,0 +1,35 @@ +import pytest + +from dotenv.variables import Literal, Variable, parse_variables + + +@pytest.mark.parametrize( + "value,expected", + [ + ("", []), + ("a", [Literal(value="a")]), + ("${a}", [Variable(name="a", default=None)]), + ("${a:-b}", [Variable(name="a", default="b")]), + ( + "${a}${b}", + [ + Variable(name="a", default=None), + Variable(name="b", default=None), + ], + ), + ( + "a${b}c${d}e", + [ + Literal(value="a"), + Variable(name="b", default=None), + Literal(value="c"), + Variable(name="d", default=None), + Literal(value="e"), + ], + ), + ] +) +def test_parse_variables(value, expected): + result = parse_variables(value) + + assert list(result) == expected From 5a797e098965257e2631fc59bd746621178de6ea Mon Sep 17 00:00:00 2001 From: Bertrand Bonnefoy-Claudet Date: Sun, 15 Nov 2020 12:25:27 +0100 Subject: [PATCH 3/3] Fix variable expansion order without override This fixes an issue when a variable is resolved differently in two bindings. For instance, take the following env file: ``` PORT=8000 URL=http://localhost:${PORT} ``` With `PORT` set to `1234` in the environment, the environment resulting from `dotenv_load(override=False)` would be: ``` PORT=1234 URL=http://localhost:8000 ``` This was inconsistent and is fixed by this commit. The environment would now be: ``` PORT=1234 URL=http://localhost:1234 ``` with override, and ``` PORT=8000 URL=http://localhost:8000 ``` without override. The behavior of `load_dotenv` is unchanged and always assumes `override=True`. --- CHANGELOG.md | 2 +- README.md | 11 ++++++++++- src/dotenv/main.py | 30 ++++++++++++++++++------------ tests/test_main.py | 22 ++++++++++++++++++++++ 4 files changed, 51 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56a7a94c..effa2510 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -_There are no unreleased changes at this time._ +- Fix resolution order in variable expansion with `override=False` (#? by [@bbc2]). ## [0.15.0] - 2020-10-28 diff --git a/README.md b/README.md index 5c9aeaf9..36f3b2b0 100644 --- a/README.md +++ b/README.md @@ -41,13 +41,22 @@ export SECRET_KEY=YOURSECRETKEYGOESHERE Python-dotenv can interpolate variables using POSIX variable expansion. -The value of a variable is the first of the values defined in the following list: +With `load_dotenv(override=True)` or `dotenv_values()`, the value of a variable is the +first of the values defined in the following list: - Value of that variable in the `.env` file. - Value of that variable in the environment. - Default value, if provided. - Empty string. +With `load_dotenv(override=False)`, the value of a variable is the first of the values +defined in the following list: + +- Value of that variable in the environment. +- Value of that variable in the `.env` file. +- Default value, if provided. +- Empty string. + Ensure that variables are surrounded with `{}` like `${HOME}` as bare variables such as `$HOME` are not expanded. diff --git a/src/dotenv/main.py b/src/dotenv/main.py index ea523d48..b366b18e 100644 --- a/src/dotenv/main.py +++ b/src/dotenv/main.py @@ -43,13 +43,14 @@ def with_warn_for_invalid_lines(mappings): class DotEnv(): - def __init__(self, dotenv_path, verbose=False, encoding=None, interpolate=True): - # type: (Union[Text, _PathLike, _StringIO], bool, Union[None, Text], bool) -> None + def __init__(self, dotenv_path, verbose=False, encoding=None, interpolate=True, override=True): + # type: (Union[Text, _PathLike, _StringIO], bool, Union[None, Text], bool, bool) -> None self.dotenv_path = dotenv_path # type: Union[Text,_PathLike, _StringIO] self._dict = None # type: Optional[Dict[Text, Optional[Text]]] self.verbose = verbose # type: bool self.encoding = encoding # type: Union[None, Text] self.interpolate = interpolate # type: bool + self.override = override # type: bool @contextmanager def _get_stream(self): @@ -73,7 +74,7 @@ def dict(self): raw_values = self.parse() if self.interpolate: - self._dict = OrderedDict(resolve_variables(raw_values)) + self._dict = OrderedDict(resolve_variables(raw_values, override=self.override)) else: self._dict = OrderedDict(raw_values) @@ -86,13 +87,13 @@ def parse(self): if mapping.key is not None: yield mapping.key, mapping.value - def set_as_environment_variables(self, override=False): - # type: (bool) -> bool + def set_as_environment_variables(self): + # type: () -> bool """ Load the current dotenv as system environemt variable. """ for k, v in self.dict().items(): - if k in os.environ and not override: + if k in os.environ and not self.override: continue if v is not None: os.environ[to_env(k)] = to_env(v) @@ -205,8 +206,8 @@ def unset_key(dotenv_path, key_to_unset, quote_mode="always"): return removed, key_to_unset -def resolve_variables(values): - # type: (Iterable[Tuple[Text, Optional[Text]]]) -> Mapping[Text, Optional[Text]] +def resolve_variables(values, override): + # type: (Iterable[Tuple[Text, Optional[Text]]], bool) -> Mapping[Text, Optional[Text]] new_values = {} # type: Dict[Text, Optional[Text]] @@ -216,8 +217,12 @@ def resolve_variables(values): else: atoms = parse_variables(value) env = {} # type: Dict[Text, Optional[Text]] - env.update(os.environ) # type: ignore - env.update(new_values) + if override: + env.update(os.environ) # type: ignore + env.update(new_values) + else: + env.update(new_values) + env.update(os.environ) # type: ignore result = "".join(atom.resolve(env) for atom in atoms) new_values[name] = result @@ -299,10 +304,11 @@ def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False, in Defaults to `False`. """ f = dotenv_path or stream or find_dotenv() - return DotEnv(f, verbose=verbose, interpolate=interpolate, **kwargs).set_as_environment_variables(override=override) + dotenv = DotEnv(f, verbose=verbose, interpolate=interpolate, override=override, **kwargs) + return dotenv.set_as_environment_variables() def dotenv_values(dotenv_path=None, stream=None, verbose=False, interpolate=True, **kwargs): # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, Union[None, Text]) -> Dict[Text, Optional[Text]] # noqa: E501 f = dotenv_path or stream or find_dotenv() - return DotEnv(f, verbose=verbose, interpolate=interpolate, **kwargs).dict() + return DotEnv(f, verbose=verbose, interpolate=interpolate, override=True, **kwargs).dict() diff --git a/tests/test_main.py b/tests/test_main.py index 6b9458d2..b927d7f2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -257,6 +257,28 @@ def test_load_dotenv_existing_variable_override(dotenv_file): assert os.environ == {"a": "b"} +@mock.patch.dict(os.environ, {"a": "c"}, clear=True) +def test_load_dotenv_redefine_var_used_in_file_no_override(dotenv_file): + with open(dotenv_file, "w") as f: + f.write('a=b\nd="${a}"') + + result = dotenv.load_dotenv(dotenv_file) + + assert result is True + assert os.environ == {"a": "c", "d": "c"} + + +@mock.patch.dict(os.environ, {"a": "c"}, clear=True) +def test_load_dotenv_redefine_var_used_in_file_with_override(dotenv_file): + with open(dotenv_file, "w") as f: + f.write('a=b\nd="${a}"') + + result = dotenv.load_dotenv(dotenv_file, override=True) + + assert result is True + assert os.environ == {"a": "b", "d": "b"} + + @mock.patch.dict(os.environ, {}, clear=True) def test_load_dotenv_utf_8(): stream = StringIO("a=à")