diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e54c83dac..ca7d84cb0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,12 +81,24 @@ jobs: with: files: junit/**/*.xml check_name: Test Results (Python ${{ matrix.python-version }}) + - name: "Upload coverage to Codecov" if: ${{ matrix.python-version == '3.8' }} uses: codecov/codecov-action@v3 with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml fail_ci_if_error: true + - name: Run codacy-coverage-reporter + if: ${{ matrix.python-version == '3.10' }} + uses: codacy/codacy-coverage-reporter-action@v1 + with: + project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} + # or + # api-token: ${{ secrets.CODACY_API_TOKEN }} + coverage-reports: coverage.xml + unit_tests_mac: needs: install_test strategy: diff --git a/.github/workflows/update_contributors.yml b/.github/workflows/update_contributors.yml index a793c5d47..c06bfbbae 100644 --- a/.github/workflows/update_contributors.yml +++ b/.github/workflows/update_contributors.yml @@ -11,7 +11,7 @@ jobs: - uses: minicli/action-contributors@v3 name: "Update a projects CONTRIBUTORS file" env: - CONTRIB_REPOSITORY: 'rochacbruno/dynaconf' + CONTRIB_REPOSITORY: 'dynaconf/dynaconf' CONTRIB_OUTPUT_FILE: 'CONTRIBUTORS.md' - name: Create a PR uses: peter-evans/create-pull-request@v3 diff --git a/.gitignore b/.gitignore index 24ff77c87..b2f6f8ab8 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,8 @@ junit/ # django example static files example/django_example/admin/ example/django_example/debug_toolbar/ + + +# Vendor_src exists only during build process +dynaconf/vendor_src/ +dynaconf/vendor/source diff --git a/3.x-release-notes.md b/3.x-release-notes.md index ded4b11d7..0753c639b 100644 --- a/3.x-release-notes.md +++ b/3.x-release-notes.md @@ -3,13 +3,13 @@ In Dynaconf 3.0.0 we introduced some improvements and with those improvements it comes some **breaking changes.** -Some of the changes were discussed on the **1st Dynaconf community meeting** [video is available](https://www.twitch.tv/videos/657033043) and [meeting notes #354](https://github.com/rochacbruno/dynaconf/issues/354). +Some of the changes were discussed on the **1st Dynaconf community meeting** [video is available](https://www.twitch.tv/videos/657033043) and [meeting notes #354](https://github.com/dynaconf/dynaconf/issues/354). ## Improvements -- Validators now implements `|` and `&` operators to allow `Validator() &| Validator()` and has more `operations` available such as `len_eq, len_min, len_max, startswith` [#353](https://github.com/rochacbruno/dynaconf/pull/353). -- First level variables are now allowed to be `lowercase` it is now possible to access `settings.foo` or `settings.FOO` [#357](https://github.com/rochacbruno/dynaconf/pull/357). +- Validators now implements `|` and `&` operators to allow `Validator() &| Validator()` and has more `operations` available such as `len_eq, len_min, len_max, startswith` [#353](https://github.com/dynaconf/dynaconf/pull/353). +- First level variables are now allowed to be `lowercase` it is now possible to access `settings.foo` or `settings.FOO` [#357](https://github.com/dynaconf/dynaconf/pull/357). - All Dependencies are now vendored, so when installing Dynaconf is not needed to install any dependency. - Dynaconf configuration options are now aliased so when creating an instance of `LazySettings|FlaskDynaconf|DjangoDynaconf` it is now possible to pass instead of `ENVVAR_PREFIX_FOR_DYNACONF` just `envvar_prefix` and this lowercase alias is now accepted. - Fixed bugs in `merge` and deprecated the `@reset` token. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0645a44..80efdfb9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -584,7 +584,7 @@ Changelog In order to keep the same method api, default values should be parsed and converted to Boxed objects. - https://github.com/rochacbruno/dynaconf/issues/462 + https://github.com/dynaconf/dynaconf/issues/462 - HOTFIX for 501 (#540) [Bruno Rocha] Flask still missing __contains__ @@ -828,7 +828,7 @@ Other Update docs/release_notes Fixing prospector warnings. (#425) Fix mkdocs config problem found in #423 - Signed in for https://xscode.com/rochacbruno/dynaconf (#426) + Signed in for https://xscode.com/dynaconf/dynaconf (#426) Remove links to outdated issues from guidelines Fix colors and KEyError handling on cli.py (#429) Fix #434 setenv failing to unset LazyValues (#437) @@ -858,7 +858,7 @@ Other Update docs/release_notes Fixing prospector warnings. (#425) Fix mkdocs config problem found in #423 - Signed in for https://xscode.com/rochacbruno/dynaconf (#426) + Signed in for https://xscode.com/dynaconf/dynaconf (#426) Remove links to outdated issues from guidelines Fix colors and KEyError handling on cli.py (#429) Fix #434 setenv failing to unset LazyValues (#437) @@ -931,7 +931,7 @@ Other - [Release notes](https://github.com/facelessuser/mkdocs-material-extensions/releases) - [Changelog](https://github.com/facelessuser/mkdocs-material-extensions/blob/master/changelog.md) - [Commits](https://github.com/facelessuser/mkdocs-material-extensions/compare/1.0...1.0.1) -- Signed in for https://xscode.com/rochacbruno/dynaconf (#426) [Bruno +- Signed in for https://xscode.com/dynaconf/dynaconf (#426) [Bruno Rocha] Offering paid support for dynaconf users. @@ -1171,7 +1171,7 @@ Other Bruno Rocha (10): Release version 3.0.0 Hot fix removing unused imports - Merge branch 'master' of github.com:rochacbruno/dynaconf + Merge branch 'master' of github.com:dynaconf/dynaconf Removing invalid links, adding allert on old docs fix #369 and fix #371 (#372) Fix #359 lazy template substitution on nested keys (#375) Flask fizes and other issues included. (#376) @@ -1199,7 +1199,7 @@ Other Bruno Rocha (10): Release version 3.0.0 Hot fix removing unused imports - Merge branch 'master' of github.com:rochacbruno/dynaconf + Merge branch 'master' of github.com:dynaconf/dynaconf Removing invalid links, adding allert on old docs fix #369 and fix #371 (#372) Fix #359 lazy template substitution on nested keys (#375) Flask fizes and other issues included. (#376) @@ -1247,7 +1247,7 @@ Other Rocha] - Removing invalid links, adding allert on old docs fix #369 and fix #371 (#372) [Bruno Rocha] -- Merge branch 'master' of github.com:rochacbruno/dynaconf. [Bruno +- Merge branch 'master' of github.com:dynaconf/dynaconf. [Bruno Rocha] - Fix validation of optional fields (#370) [Bruno Rocha Co-authored-by: Bruno Rocha @@ -1313,7 +1313,7 @@ Other DEPRECATED global settings object. DEPRECATED global settings object. (#356) Lowecase read allowed by default (#357) - Merge branch 'master' of github.com:rochacbruno/dynaconf + Merge branch 'master' of github.com:dynaconf/dynaconf envless by default - breaking change ⚠️ (#358) dotenv is no more loaded by default (#360) No more loading of `settings.*` by default (#361) @@ -1351,7 +1351,7 @@ Other * Fix redis and vault tests * CLI default to global instance with warnings -- Merge branch 'master' of github.com:rochacbruno/dynaconf. [Bruno +- Merge branch 'master' of github.com:dynaconf/dynaconf. [Bruno Rocha] - Lowecase read allowed by default (#357) [Bruno Rocha] @@ -2220,7 +2220,7 @@ Other Bruno Rocha (21): Merge branch 'jperras-merge-multiple-settings-files' - Merge branch 'master' of github.com:rochacbruno/dynaconf + Merge branch 'master' of github.com:dynaconf/dynaconf Fix #106 make PROJECT_ROOT_FOR_DYNACONF to work with custom paths Update dynaconf/utils/boxing.py Update dynaconf/utils/boxing.py @@ -2282,7 +2282,7 @@ Other Bruno Rocha (21): Merge branch 'jperras-merge-multiple-settings-files' - Merge branch 'master' of github.com:rochacbruno/dynaconf + Merge branch 'master' of github.com:dynaconf/dynaconf Fix #106 make PROJECT_ROOT_FOR_DYNACONF to work with custom paths Update dynaconf/utils/boxing.py Update dynaconf/utils/boxing.py @@ -2495,7 +2495,7 @@ Other Bruno Rocha (9): Merge branch 'jperras-merge-multiple-settings-files' - Merge branch 'master' of github.com:rochacbruno/dynaconf + Merge branch 'master' of github.com:dynaconf/dynaconf Fix #106 make PROJECT_ROOT_FOR_DYNACONF to work with custom paths Update dynaconf/utils/boxing.py Update dynaconf/utils/boxing.py @@ -2542,7 +2542,7 @@ Other Bruno Rocha (9): Merge branch 'jperras-merge-multiple-settings-files' - Merge branch 'master' of github.com:rochacbruno/dynaconf + Merge branch 'master' of github.com:dynaconf/dynaconf Fix #106 make PROJECT_ROOT_FOR_DYNACONF to work with custom paths Update dynaconf/utils/boxing.py Update dynaconf/utils/boxing.py @@ -2604,7 +2604,7 @@ Other Bruno Rocha (6): Merge branch 'jperras-merge-multiple-settings-files' - Merge branch 'master' of github.com:rochacbruno/dynaconf + Merge branch 'master' of github.com:dynaconf/dynaconf Fix #106 make PROJECT_ROOT_FOR_DYNACONF to work with custom paths Update dynaconf/utils/boxing.py Update dynaconf/utils/boxing.py @@ -2639,7 +2639,7 @@ Other Bruno Rocha (6): Merge branch 'jperras-merge-multiple-settings-files' - Merge branch 'master' of github.com:rochacbruno/dynaconf + Merge branch 'master' of github.com:dynaconf/dynaconf Fix #106 make PROJECT_ROOT_FOR_DYNACONF to work with custom paths Update dynaconf/utils/boxing.py Update dynaconf/utils/boxing.py @@ -2677,14 +2677,14 @@ Other Typo! - Fix small typo in README.md. [Matthias] -- Merge branch 'master' of github.com:rochacbruno/dynaconf. [Bruno +- Merge branch 'master' of github.com:dynaconf/dynaconf. [Bruno Rocha] - Python 3.4 has different error message. [Mantas] - Remove mocker fixture. [Mantas] Left this accidentally. - https://travis-ci.org/rochacbruno/dynaconf/jobs/452612532 + https://travis-ci.org/dynaconf/dynaconf/jobs/452612532 - Add INSTANCE_FOR_DYNACONF and --instance. [Mantas] There parameters allows dynaconf to use different LazySettings instance @@ -2849,7 +2849,7 @@ Other Docs have been updated to show an example of the nested validator name definition in action. - Closes rochacbruno/dynaconf#85. + Closes dynaconf/dynaconf#85. - Fix #94 setenv cleans SETTINGS_MODULE attribute. [Bruno Rocha] - Merge branch 'jperras-dot-traversal-access' [Bruno Rocha] - Merge branch 'dot-traversal-access' of @@ -2879,7 +2879,7 @@ Other - Tested [✓] - Examples added [✓] - Closes rochacbruno/dynaconf#84 + Closes dynaconf/dynaconf#84 - Merge branch 'rsnyman-merge-settings' [Bruno Rocha] - Add example for merge_configs. [Bruno Rocha] - Add setting merging. [Raoul Snyman] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e009cca8c..712fbd0f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,7 @@ This Diagram can help you understand visually what happens on Dynaconf: https:// git clone git@github.com:{$USER}/dynaconf.git # Add the upstream remote -git remote add upstream https://github.com/rochacbruno/dynaconf.git +git remote add upstream https://github.com/dynaconf/dynaconf.git # Activate your Python Environment python3.7 -m venv venv @@ -59,7 +59,7 @@ git commit -am "Changed XPTO to fix #issue_number" # Push to your own fork git push -u origin HEAD -# Open github.com/rochacbruno/dynaconf and send a Pull Request. +# Open github.com/dynaconf/dynaconf and send a Pull Request. ``` ### Run integration tests diff --git a/Makefile b/Makefile index 1708f7c23..3bac297e3 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,7 @@ test_examples: cd example/settings_file/;pwd;rm -rf /tmp/settings_file_test/settings.py;mkdir -p /tmp/settings_file_test/;echo "MESSAGE = 'Hello from tmp'" > /tmp/settings_file_test/settings.py;python app.py;rm -rf /tmp/settings_file_test/ cd example/configure/;pwd;rm -rf /tmp/configure_test/settings.py;mkdir -p /tmp/configure_test/;echo "MESSAGE = 'Hello from tmp'" > /tmp/configure_test/settings.py;python app.py;rm -rf /tmp/configure_test/ cd example/-m_case/;pwd;python -m module + cd example/custom_cast_token;pwd;python app.py @echo '############### Calling from outer folder ###############' python example/common/program.py @@ -131,6 +132,8 @@ test_examples: cd example/issues/729_use_default_when_setting_is_blank;pwd;python app.py cd example/issues/741_envvars_ignored;pwd;sh recreate.sh cd example/issues/705_flask_dynaconf_init;pwd;make test;make clean + cd example/issues/794_includes;pwd;python app.py + cd example/issues/799_negative_numbers;pwd;DYNACONF_NUM="-1" python app.py test_vault: # @cd example/vault;pwd;python write.py @@ -182,6 +185,7 @@ test: pep8 mypy test_only citest: py.test -v --cov-config .coveragerc --cov=dynaconf -l tests/ --junitxml=junit/test-results.xml + coverage xml ciinstall: python -m pip install --upgrade pip @@ -208,7 +212,9 @@ pep8: flake8 dynaconf --exclude=dynaconf/vendor* dist: clean + @make minify_vendor @python setup.py sdist bdist_wheel + @make source_vendor publish: make run-tox @@ -234,3 +240,21 @@ docs: run-tox: tox --recreate rm -rf .tox + +minify_vendor: + # Ensure vendor is source and cleanup vendor_src bck folder + ls dynaconf/vendor/source && rm -rf dynaconf/vendor_src + + # Backup dynaconf/vendor folder as dynaconf/vendor_src + mv dynaconf/vendor dynaconf/vendor_src + + # create a new dynaconf/vendor folder with minified files + ./minify.sh + + +source_vendor: + # Ensure dynaconf/vendor_src is source and cleanup vendor folder + ls dynaconf/vendor_src/source && rm -rf dynaconf/vendor + + # Restore dynaconf/vendor_src folder as dynaconf/vendor + mv dynaconf/vendor_src dynaconf/vendor diff --git a/README.md b/README.md index 5838b6bf0..24cacd9e6 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ > **dynaconf** - Configuration Management for Python. -[![MIT License](https://img.shields.io/badge/license-MIT-007EC7.svg?style=flat-square)](/LICENSE) [![PyPI](https://img.shields.io/pypi/v/dynaconf.svg)](https://pypi.python.org/pypi/dynaconf) [![PyPI](https://img.shields.io/pypi/pyversions/dynaconf.svg)]() ![PyPI - Downloads](https://img.shields.io/pypi/dm/dynaconf.svg?label=pip%20installs&logo=python) [![CI](https://github.com/rochacbruno/dynaconf/actions/workflows/main.yml/badge.svg)](https://github.com/rochacbruno/dynaconf/actions/workflows/main.yml) [![codecov](https://codecov.io/gh/rochacbruno/dynaconf/branch/master/graph/badge.svg)](https://codecov.io/gh/rochacbruno/dynaconf) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/42d2f11ef0a446808b246c8c69603f6e)](https://www.codacy.com/gh/rochacbruno/dynaconf/dashboard?utm_source=github.com&utm_medium=referral&utm_content=rochacbruno/dynaconf&utm_campaign=Badge_Grade) ![GitHub stars](https://img.shields.io/github/stars/rochacbruno/dynaconf.svg) ![GitHub Release Date](https://img.shields.io/github/release-date/rochacbruno/dynaconf.svg) ![GitHub commits since latest release](https://img.shields.io/github/commits-since/rochacbruno/dynaconf/latest.svg) ![GitHub last commit](https://img.shields.io/github/last-commit/rochacbruno/dynaconf.svg) [![Code Style Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black/) +[![MIT License](https://img.shields.io/badge/license-MIT-007EC7.svg?style=flat-square)](/LICENSE) [![PyPI](https://img.shields.io/pypi/v/dynaconf.svg)](https://pypi.python.org/pypi/dynaconf) [![PyPI](https://img.shields.io/pypi/pyversions/dynaconf.svg)]() ![PyPI - Downloads](https://img.shields.io/pypi/dm/dynaconf.svg?label=pip%20installs&logo=python) [![CI](https://github.com/dynaconf/dynaconf/actions/workflows/main.yml/badge.svg)](https://github.com/dynaconf/dynaconf/actions/workflows/main.yml) [![codecov](https://codecov.io/gh/dynaconf/dynaconf/branch/master/graph/badge.svg)](https://codecov.io/gh/dynaconf/dynaconf) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/3fb2de98464442f99a7663181803b400)](https://www.codacy.com/gh/dynaconf/dynaconf/dashboard?utm_source=github.com&utm_medium=referral&utm_content=dynaconf/dynaconf&utm_campaign=Badge_Grade) ![GitHub stars](https://img.shields.io/github/stars/dynaconf/dynaconf.svg) ![GitHub Release Date](https://img.shields.io/github/release-date/dynaconf/dynaconf.svg) ![GitHub commits since latest release](https://img.shields.io/github/commits-since/dynaconf/dynaconf/latest.svg) ![GitHub last commit](https://img.shields.io/github/last-commit/dynaconf/dynaconf.svg) [![Code Style Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black/) -![GitHub issues](https://img.shields.io/github/issues/rochacbruno/dynaconf.svg) [![User Forum](https://img.shields.io/badge/users-forum-blue.svg?logo=googlechat)](https://github.com/rochacbruno/dynaconf/discussions) [![Join the chat at https://gitter.im/dynaconf/dev](https://badges.gitter.im/dynaconf/dev.svg)](https://gitter.im/dynaconf/dev?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![ Matrix](https://img.shields.io/badge/dev-room-blue.svg?logo=matrix)](https://matrix.to/#/#dynaconf:matrix.org) +![GitHub issues](https://img.shields.io/github/issues/dynaconf/dynaconf.svg) [![User Forum](https://img.shields.io/badge/users-forum-blue.svg?logo=googlechat)](https://github.com/dynaconf/dynaconf/discussions) [![Join the chat at https://gitter.im/dynaconf/dev](https://badges.gitter.im/dynaconf/dev.svg)](https://gitter.im/dynaconf/dev?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![ Matrix](https://img.shields.io/badge/dev-room-blue.svg?logo=matrix)](https://matrix.to/#/#dynaconf:matrix.org) ## Features @@ -127,7 +127,7 @@ There is a lot more you can do, **read the docs:** http://dynaconf.com ## Contribute -Main discussions happens on [Discussions Tab](https://github.com/rochacbruno/dynaconf/discussions) learn more about how to get involved on [CONTRIBUTING.md guide](CONTRIBUTING.md) +Main discussions happens on [Discussions Tab](https://github.com/dynaconf/dynaconf/discussions) learn more about how to get involved on [CONTRIBUTING.md guide](CONTRIBUTING.md) ## More diff --git a/docs/advanced.md b/docs/advanced.md index f2f611d97..50d60380d 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -167,7 +167,7 @@ def load( obj._loaded_files.append(sops_file) ``` -See more [example/custom_loader](https://github.com/rochacbruno/dynaconf/tree/master/example/custom_loader) +See more [example/custom_loader](https://github.com/dynaconf/dynaconf/tree/master/example/custom_loader) ## Module impersonation @@ -413,7 +413,7 @@ Then your `program.py` will print `"On Testing"` red from `[testing]` environmen For pytest it is common to create fixtures to provide pre-configured settings object or to configure the settings before all the tests are collected. -Examples available on [https://github.com/rochacbruno/dynaconf/tree/master/example/pytest_example](https://github.com/rochacbruno/dynaconf/tree/master/example/pytest_example) +Examples available on [https://github.com/dynaconf/dynaconf/tree/master/example/pytest_example](https://github.com/dynaconf/dynaconf/tree/master/example/pytest_example) With `pytest` fixtures it is recommended to use the `FORCE_ENV_FOR_DYNACONF` instead of just `ENV_FOR_DYNACONF` because it has precedence. diff --git a/docs/cli.md b/docs/cli.md index 5e0ac0c08..7bbf19f18 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -243,5 +243,5 @@ $ dynaconf -i config.settings --banner ██████╔╝ ██║ ██║ ╚████║██║ ██║╚██████╗╚██████╔╝██║ ╚████║██║ ╚═════╝ ╚═╝ ╚═╝ ╚═══╝╚═╝ ╚═╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═══╝╚═╝ -Learn more at: http://github.com/rochacbruno/dynaconf +Learn more at: http://github.com/dynaconf/dynaconf ``` diff --git a/docs/configuration.md b/docs/configuration.md index 113c52979..6eabcaa5f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -208,6 +208,11 @@ After loading the files specified in `settings_files` dynaconf will load all the ``` **Includes allows the use of globs** +!!! warning + includes are loaded relative to the first loaded file, so if you have a `settings_files=['conf/settings.toml']` and `includes=["*.yaml"]` + the yaml files will be loaded relative to the `conf` folder. Unless you specify the absolute path or pass `root_path` to the Dynaconf + initializer. + --- ### **loaders** diff --git a/docs/django.md b/docs/django.md index 8634e4cf2..7c501c91b 100644 --- a/docs/django.md +++ b/docs/django.md @@ -32,7 +32,7 @@ $ dynaconf init --django yourapp/settings.py Dynaconf will append its extension loading code to the bottom of your `yourapp/settings.py` file and will create `settings.toml` and `.secrets.toml` in the current folder (the same where `manage.py` is located). -> **TIP** Take a look at [example/django_example](https://github.com/rochacbruno/dynaconf/tree/master/example/django_example) +> **TIP** Take a look at [example/django_example](https://github.com/dynaconf/dynaconf/tree/master/example/django_example) ## Using `DJANGO_` environment variables @@ -241,7 +241,7 @@ import pytest @pytest.fixture(scope="session", autouse=True) def set_test_settings(): - # https://github.com/rochacbruno/dynaconf/issues/491#issuecomment-745391955 + # https://github.com/dynaconf/dynaconf/issues/491#issuecomment-745391955 from django.conf import settings settings.setenv('testing') # force the environment to be whatever you want ``` diff --git a/docs/envvars.md b/docs/envvars.md index 0960c9dc8..ee9ef082e 100644 --- a/docs/envvars.md +++ b/docs/envvars.md @@ -144,6 +144,35 @@ export PREFIX_PATH='@format {env{"HOME"}/.config/{this.DB_NAME}' export PREFIX_PATH='@jinja {{env.HOME}}/.config/{{this.DB_NAME}} | abspath' ``` +### Adding a Custom Casting Token + +If you would like to add a custom casting token, you can do so by adding a +converter. For example, if we would like to cast strings to a pathlib.Path +object we can add in our python code: + +```python +# app.py +from pathlib import Path +from dynaconf.utils import parse_conf + +parse_conf.converters["@path"] = ( + lambda value: value.set_casting(Path) + if isinstance(value, parse_conf.Lazy) + else Path(value) +) +``` + +In the settings file we can now use teh @path casting token. Like with other +casting tokens you can also combine them: + +```toml +# settings.toml +my_path = "@path /home/foo/example.txt" +parent = "@path @format {env[HOME]}/parent" +child = "@path @format {this.parent}/child" +``` + + ## Environment variables filtering All environment variables (naturally, accounting for prefix rules) will be diff --git a/docs/index.md b/docs/index.md index e87cbbe09..ddf97a023 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,14 +7,7 @@ Configuration Management for Python.

-

MIT License PyPIcodecov GitHub Release Date GitHub last commit Discussions Demo

- - -[![Foo](https://xscode.com/assets/promo-banner.svg)](https://xscode.com/rochacbruno/dynaconf) - - -> **TIP** You can see a working demo here: https://github.com/rochacbruno/learndynaconf - +

MIT License PyPIcodecov GitHub Release Date GitHub last commit Discussions

## Features diff --git a/docs/merging.md b/docs/merging.md index c2de78039..7e4aa437b 100644 --- a/docs/merging.md +++ b/docs/merging.md @@ -533,4 +533,4 @@ The **dynaconf_merge** and **@merge** functionalities works only for the first l ## More examples -Take a look at the [example](https://github.com/rochacbruno/dynaconf/tree/master/example) folder to see some examples of use with different file formats and features. +Take a look at the [example](https://github.com/dynaconf/dynaconf/tree/master/example) folder to see some examples of use with different file formats and features. diff --git a/docs/pt-br/docs/cli.md b/docs/pt-br/docs/cli.md new file mode 100644 index 000000000..afb09e153 --- /dev/null +++ b/docs/pt-br/docs/cli.md @@ -0,0 +1,243 @@ + + +## configurando instancias + +Todo comando (com exceção do init) requer uma Instância que pode ser passada usando o parâmetro `-i` ou por variável de ambiente com `export INSTANCE_FOR_DYNACONF` + +## O dynaconf CLI (Interface de Linha de Comando) + +O comando `$ dynaconf -i config.settings` do cli provê alguns comandos bem úteis + +> **IMPORTANTE** caso usem [Flask Extension](/flask/) a variável de ambiente `FLASK_APP` deve ser definda para usar o CLI, e caso usem [Django Extension](/django/) a variável `DJANGO_SETTINGS_MODULE` deve estar definida. + +### dynaconf --help + +```bash +Uso: dynaconf [OPÇÕES] COMMAND [ARGS]... + + Dynaconf - Command Line Interface + + Documentation: https://dynaconf.com/ + +Opções: + --version Mostra a versão do dynaconf + --docs Abre a documentação no navegador + --banner Mostra um banner maneiro + -i, --instance TEXT Define a instância customizada do LazySettings + --help Mostra essa mensagem e sai. + +Commands: + get Retorna o valor bruto de uma chave de configuração + init Inicia um projeto dynaconf, por padrão cria um settings.toml ... + list Lita todos os valores de configuração definidos pelo usuário se `--all` é passado ... + validate Valida as configurações do Dynaconf baseado em regras definidas em... + write Escreve dados para o recurso específico +``` + +### dynaconf init + +Use o comando init para facilmente iniciar a configuração de sua aplicaçãoto, tendo o dynaconf instalado vá para o diretório raiz de sua aplicação e execute: + +``` +$ dynaconf init -v key=value -v foo=bar -s token=1234 +``` + +O comando acima criará no diretório corrente: + +`settings.toml` + +```ini +KEY = "value" +FOO = "bar" +``` + +also `.secrets.toml` + +```ini +TOKEN = "1234" +``` + +Bem como incluirá no arquivo `.gitignore` para que seja ignorado o arquivo `.secrets.toml`, como descrito abaixo: + +```ini +# Ignore dynaconf secret files +.secrets.* +``` + +> Reomenda-se que para dados sensíveis em produção use[Vault Server](/secrets/) + +``` +Uso: dynaconf init [OPÇÕES] + + Inicia um projeto dynaconf que por padrão cria um settings.toml e um + .secrets.toml para os ambientes [default|development|staging|testing|production|global]. + + O formato dos arquivo pode ser alterado passando --format=yaml|json|ini|py. + + Esse comando deve ser executado no diretório raiz do projeto o pode-se passar + --path=/myproject/root/folder. + + O --env/-e está depreciado ( mantido por compatibilidade, mas deve evitar o uso) + +Opções: + -f, --format [ini|toml|yaml|json|py|env] + -p, --path TEXT o padrão é o diretório corrente + -e, --env TEXT Define o ambiente de trabalho no arquivo `.env` + -v, --vars TEXT valores extras a serem gravados no arquivo de settings, ex.: + `dynaconf init -v NAME=foo -v X=2 + + -s, --secrets TEXT valores sensíveis a serem escritos no arquivo .secrets + ex.: `dynaconf init -s TOKEN=kdslmflds + + --wg / --no-wg + -y + --django TEXT + --help Mostra essa mensagem e sai. +``` + +Observe que a opção `-i`/`--instance` não pode ser usada com `init`, pois `-i` deve apontar para uma instância existente das configurações. + + +### Dynaconf get + +Pega o valor bruto para uma única chave + +```bash +Uso: dynaconf get [OPÇÕES ] KEY + + Retorna o valor bruto para a KEY na configuração + +Opções: + -d, --default TEXT Valor padrão se a configuração não existir + -e, --env TEXT Filtra o ambiente (env) para pegar os valores do ambiente especificado + -u, --unparse Não analisa (unparse) os dados pela adição de marcadores como: @none, @int etc.. + --help Mostra essa mensagem e sai. +``` + +Exemplo: + +```bash +export FOO=$(dynaconf get DATABASE_NAME -d 'default') +``` + + +### dynaconf list + +Lista todos os parâmetros definidos e, opcionalmente, exporta para um arquivo JSON. + +``` +Uso: dynaconf list [OPÇÕES] + + Lista todos os valores de configuração definidos pelo usuário e se `--all` for passado + mostra, também, as variáveis internas do dynaconf. + +Opçẽos: + -e, --env TEXT Filtra o ambiente (env) para pegar os valores + -k, --key TEXT Filtra por uma única chave (key) + -m, --more Pagina o resultado ao estilo mais|menos (more|less) + -l, --loader TEXT um identificador de conversor (loader) para filtar ex.: toml|yaml + -a, --all Mostra as configurações internas do dynaconf? + -o, --output FILE Filepath para gravar os valores listados num arquivo JSON + --output-flat O arquivo de saída é plano (flat) (i.e., não inclui o nome ambiente [env]) + --help Mostra essa mensagem e sai. +``` + +#### Exportanto o ambiente atual como um arquivo + +```bash +dynaconf list -o path/to/file.yaml +``` + +O comando acima exportará todos os itens mostrados por `dynaconf list` para o format desejado o qual é inferido pela extensão do arquivo em `-o`, formatos suportados são: `yaml, toml, ini, json, py` + +Quando se usa `py` pode-se querer uma saída plana (flat) (sem estar aninhada internamente pelo nome do ambiente (env)) + +```bash +dynaconf list -o path/to/file.py --output-flat +``` + +### dynaconf write + +``` +Uso: dynaconf write [OPÇÕES] [ini|toml|yaml|json|py|redis|vault|env] + + Escreve os dados dem uma fonte específica + +Opções: + -v, --vars TEXT Par chave=valor que será escrito no settings ex.: + `dynaconf write toml -e NAME=foo -e X=2 + + -s, --secrets TEXT Par chave=segredo que será escrito no .secrets ex: + `dynaconf write toml -s TOKEN=kdslmflds -s X=2 + + -p, --path TEXT Diretório/Arquivo no qual será escrito, sendo o padrão para diretório-atual/settings.{ext} + -e, --env TEXT Ambiente para o qual o valor será registrado o padrão é o DEVELOPMENT, para arquivos de recursos + externos como o Redis e o Vault será + DYNACONF ou o valor configurado na variável $ENVVAR_PREFIX_FOR_DYNACONF + + -y + --help Mostra essa mensagem e sai. +``` + +### dynaconf validate + +> **Novo desde a versão 1.0.1** + +Iniciado na versão 1.0.1, é possível definir validadores no arquivo **TOML** chamado **dynaconf_validators.toml** colocado na mesma pasta do seu arquivo settings. + +`dynaconf_validators.toml` equivalente ao descrito acima + +```ini +[default] + +version = {must_exist=true} +name = {must_exist=true} +password = {must_exist=false} + + [default.age] + must_exist = true + lte = 30 + gte = 10 + +[production] +project = {eq="hello_world"} +``` + +Para executar a validação use o comando abaixo: + +``` +$ dynaconf -i config.settings validate +``` + +A validação ocorrendo sem problemas retornará status 0 (sucesso) e, por isso, o comando pode ser usado na pipeline de CI/CD/Deploy. + +### dynaconf --version + +Retorna a versão do dynaconf instalada + +``` +$ dynaconf -i config.settings --version +1.0.0 +``` + +### dynaconf --docs + +Abre a documentação do Dynaconf no navegador padrão do sistema. + + +### dynaconf --banner + +Imprime esse bonito baner feito em ascci no terminal :) + +``` +$ dynaconf -i config.settings --banner + +██████╗ ██╗ ██╗███╗ ██╗ █████╗ ██████╗ ██████╗ ███╗ ██╗███████╗ +██╔══██╗╚██╗ ██╔╝████╗ ██║██╔══██╗██╔════╝██╔═══██╗████╗ ██║██╔════╝ +██║ ██║ ╚████╔╝ ██╔██╗ ██║███████║██║ ██║ ██║██╔██╗ ██║█████╗ +██║ ██║ ╚██╔╝ ██║╚██╗██║██╔══██║██║ ██║ ██║██║╚██╗██║██╔══╝ +██████╔╝ ██║ ██║ ╚████║██║ ██║╚██████╗╚██████╔╝██║ ╚████║██║ +╚═════╝ ╚═╝ ╚═╝ ╚═══╝╚═╝ ╚═╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═══╝╚═╝ + +Learn more at: http://github.com/dynaconf/dynaconf +``` diff --git a/docs/pt-br/docs/index.md b/docs/pt-br/docs/index.md new file mode 100644 index 000000000..3f804d19f --- /dev/null +++ b/docs/pt-br/docs/index.md @@ -0,0 +1,143 @@ +# Início Rápido com Dynaconf + +

+ Dynaconf +

+

+ Gereciamento de Configurações para Python. +

+ +

MIT License PyPIcodecov GitHub Release Date GitHub last commit Discussions Demo

+ + +> **DICA** Você pode ver uma exmplo funcional aqui (em inglês): https://github.com/rochacbruno/learndynaconf + +## Recursos + +- Inspirado no **[12-factor application guide](https://12factor.net/pt_br/config)** +- **Gerenciamento de configurações** (valores padrão, validação, análise (parsing), modelagem (templating)) +- Proteção de **informações sensíveis** (passwords/tokens) +- Múltiplos **formatos de arquivos** `toml|yaml|json|ini|py` e, também, carregadores (loaders) personalizáveis. +- Suporte total para **variáveis de ambiente** para substituir as configurações existentes (incluindo suporte a dotenv). +- Sistema em camadas opcionais para **vários ambientes** `[padrão, desenvolvimento, teste, produção]` (também chamado multiplos perfis) +- Suporte nativo para **Hashicorp Vault** e **Redis** para configurações e amarzenamento de segredos. +- Extensões nativas para **Django**, **Flask** e **fastAPI** web framewors. +- **CLI** para operações comuns tais como `init, list, write, validate, export`. + +## Instalação + +### Instalando via [pypi](https://pypi.org/project/dynaconf) + +```bash +pip install dynaconf +``` +### Initialize Dynaconf on your project + +???+ "Usando somente Python" + + #### Usando somente Python + + No diretório raiz de seu projeto execute o comando `dynaconf init` + + ```bash hl_lines="2" + cd path/to/your/project/ + dynaconf init -f toml + ``` + A saída do comando deverá ser: + + ```plain + + ⚙️ Configuring your Dynaconf environment + ------------------------------------------ + 🐍 The file `config.py` was generated. + + 🎛️ settings.toml created to hold your settings. + + 🔑 .secrets.toml created to hold your secrets. + + 🙈 the .secrets.* is also included in `.gitignore` + beware to not push your secrets to a public repo. + + 🎉 Dynaconf is configured! read more on https://dynaconf.com + ``` + + > ℹ️ Você pode escolher `toml|yaml|json|ini|py` em `dynaconf init -f `, **toml** é o formato padrão e o mais **recomendado** para configurações. + + O comando `init` do Dynaconf cria os seguintes arquivos + + ```plain + . + ├── config.py # No qual você importa seu objeto de configuração (obrigatório) + ├── .secrets.toml # Arquivo com dados sensíveis como senhas e tokesn (opctional) + └── settings.toml # Configurações da aplicação (opcional) + ``` + + === "your_program.py" + Em seu código você importa e usa o objeto **settings**, este é importado do arquivo **config.py** + ```python + from config import settings + + assert settings.key == "value" + assert settings.number == 789 + assert settings.a_dict.nested.other_level == "nested value" + assert settings['a_boolean'] is False + assert settings.get("DONTEXIST", default=1) == 1 + ``` + + === "config.py" + Neste arquivo a nova instância do objeto *setings* de **Dynaconf** é inicializada e configurada. + ```python + from dynaconf import Dynaconf + + settings = Dynaconf( + settings_files=['settings.toml', '.secrets.toml'], + ) + ``` + Mais opções são descritas em [Dynaconf Configuration](/configuration/) + + === "settings.toml" + **Opcionalmente** armazene as configurações em um arquivo (ou em múltiplos arquivos) + ```toml + key = "value" + a_boolean = false + number = 1234 + a_float = 56.8 + a_list = [1, 2, 3, 4] + a_dict = {hello="world"} + + [a_dict.nested] + other_level = "nested value" + ``` + Mais detalhes em [Settings Files](/settings_files/) + + === ".secrets.toml" + **Opcionalmente** armazene dados sensíveis em um único arquivo local chamado `.secrets.toml` + ```toml + password = "s3cr3t" + token = "dfgrfg5d4g56ds4gsdf5g74984we5345-" + message = "This file doesn't go to your pub repo" + ``` + + > ⚠️ O comando `dynaconf init` coloca o `.secrets.*` em seu `.gitignore` evitando que o mesmo seja exposto no repositório público mas é sua responsabilidade mantê-lo em seu ambiente local, também é recomendado que os ambientes de produção utilizem o suporte nativo para o serviço da Hashicorp Vault para senhas e tokens. + + ```ini + # Segredos não vão para o repositório público + .secrets.* + ``` + + Leia mais em [Secrets](/secrets/) + + === "env vars" + **Opcionalmente** use variáveis de ambiente com prefixos. (arquivos `.env` também são suportados) + + ```bash + export DYNACONF_NUMBER=789 + export DYNACONF_FOO=false + export DYNACONF_DATA__CAN__BE__NESTED=value + export DYNACONF_FORMATTED_KEY="@format {this.FOO}/BAR" + export DYNACONF_TEMPLATED_KEY="@jinja {{ env['HOME'] | abspath }}" + ``` + + --- + + > ℹ️ Você pode criar os arquivos em vez de utilizar o comando `dynaconf init` e dar qualquer nome que queira em vez do padrão `config.py` (o arquivo deve estar em seu `python path` para ser importado) diff --git a/docs/release_notes.md b/docs/release_notes.md index af8590e17..2fa6c1938 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -98,13 +98,13 @@ Validator("FOO", default=Lazy(empty, formatter=my_function)) In Dynaconf 3.0.0 we introduced some improvements and with those improvements it comes some **breaking changes.** -Some of the changes were discussed on the **1st Dynaconf community meeting** [video is available](https://www.twitch.tv/videos/657033043) and [meeting notes #354](https://github.com/rochacbruno/dynaconf/issues/354). +Some of the changes were discussed on the **1st Dynaconf community meeting** [video is available](https://www.twitch.tv/videos/657033043) and [meeting notes #354](https://github.com/dynaconf/dynaconf/issues/354). ## Improvements -- Validators now implements `|` and `&` operators to allow `Validator() &| Validator()` and has more `operations` available such as `len_eq, len_min, len_max, startswith` [#353](https://github.com/rochacbruno/dynaconf/pull/353). -- First level variables are now allowed to be `lowercase` it is now possible to access `settings.foo` or `settings.FOO` [#357](https://github.com/rochacbruno/dynaconf/pull/357). +- Validators now implements `|` and `&` operators to allow `Validator() &| Validator()` and has more `operations` available such as `len_eq, len_min, len_max, startswith` [#353](https://github.com/dynaconf/dynaconf/pull/353). +- First level variables are now allowed to be `lowercase` it is now possible to access `settings.foo` or `settings.FOO` [#357](https://github.com/dynaconf/dynaconf/pull/357). - All Dependencies are now vendored, so when installing Dynaconf is not needed to install any dependency. - Dynaconf configuration options are now aliased so when creating an instance of `LazySettings|FlaskDynaconf|DjangoDynaconf` it is now possible to pass instead of `ENVVAR_PREFIX_FOR_DYNACONF` just `envvar_prefix` and this lowercase alias is now accepted. - Fixed bugs in `merge` and deprecated the `@reset` token. diff --git a/docs/settings_files.md b/docs/settings_files.md index ddd885a4a..2012c192f 100644 --- a/docs/settings_files.md +++ b/docs/settings_files.md @@ -133,6 +133,10 @@ key = value anotherkey = value ``` +!!! note + The paths passed to includes are relative to the `root_path` of Dynaconf instance or if that is not set, relative + to the directory where the first loaded file is located, includes also accepts globs and absolute paths. + --- ## Layered environments on files diff --git a/dynaconf/base.py b/dynaconf/base.py index 7dadc7a46..51e09a404 100644 --- a/dynaconf/base.py +++ b/dynaconf/base.py @@ -1074,18 +1074,16 @@ def load_file(self, path=None, env=None, silent=True, key=None): # continue the loop. continue - # python 3.6 does not resolve Pathlib basedirs - # issue #494 root_dir = str(self._root_path or os.getcwd()) + + # Issue #494 if ( isinstance(_filename, Path) and str(_filename.parent) in root_dir ): # pragma: no cover filepath = str(_filename) else: - filepath = os.path.join( - self._root_path or os.getcwd(), str(_filename) - ) + filepath = os.path.join(root_dir, str(_filename)) paths = [ p @@ -1095,6 +1093,7 @@ def load_file(self, path=None, env=None, silent=True, key=None): local_paths = [ p for p in sorted(glob.glob(filepath)) if ".local." in p ] + # Handle possible *.globs sorted alphanumeric for path in paths + local_paths: if path in already_loaded: # pragma: no cover diff --git a/dynaconf/cli.py b/dynaconf/cli.py index e341c52e5..08544d78d 100644 --- a/dynaconf/cli.py +++ b/dynaconf/cli.py @@ -25,6 +25,7 @@ from dynaconf.validator import Validator from dynaconf.vendor import click from dynaconf.vendor import toml +from dynaconf.vendor import tomllib os.environ["PYTHONIOENCODING"] = "utf-8" @@ -42,6 +43,8 @@ def set_settings(ctx, instance=None): settings = None + _echo_enabled = ctx.invoked_subcommand not in ["get", None] + if instance is not None: if ctx.invoked_subcommand in ["init"]: raise click.UsageError( @@ -56,7 +59,7 @@ def set_settings(ctx, instance=None): flask_app = ScriptInfo().load_app() settings = FlaskDynaconf(flask_app, **flask_app.config).settings - click.echo( + _echo_enabled and click.echo( click.style( "Flask app detected", fg="white", bg="bright_black" ) @@ -72,7 +75,7 @@ def set_settings(ctx, instance=None): settings = LazySettings() if settings is not None: - click.echo( + _echo_enabled and click.echo( click.style( "Django app detected", fg="white", bg="bright_black" ) @@ -163,7 +166,7 @@ def show_banner(ctx, param, value): return set_settings(ctx) click.echo(settings.dynaconf_banner) - click.echo("Learn more at: http://github.com/rochacbruno/dynaconf") + click.echo("Learn more at: http://github.com/dynaconf/dynaconf") ctx.exit() @@ -695,7 +698,16 @@ def validate(path): # pragma: no cover click.echo(click.style(f"{path} not found", fg="white", bg="red")) sys.exit(1) - validation_data = toml.load(open(str(path))) + try: # try tomlib first + validation_data = tomllib.load(open(str(path), "rb")) + except UnicodeDecodeError: # fallback to legacy toml (TBR in 4.0.0) + warnings.warn( + "TOML files should have only UTF-8 encoded characters. " + "starting on 4.0.0 dynaconf will stop allowing invalid chars.", + ) + validation_data = toml.load( + open(str(path), encoding=default_settings.ENCODING_FOR_DYNACONF), + ) success = True for env, name_data in validation_data.items(): diff --git a/dynaconf/loaders/base.py b/dynaconf/loaders/base.py index 919005a39..dec5cb0af 100644 --- a/dynaconf/loaders/base.py +++ b/dynaconf/loaders/base.py @@ -21,7 +21,14 @@ class BaseLoader: """ def __init__( - self, obj, env, identifier, extensions, file_reader, string_reader + self, + obj, + env, + identifier, + extensions, + file_reader, + string_reader, + opener_params=None, ): """Instantiates a loader for different sources""" self.obj = obj @@ -30,6 +37,10 @@ def __init__( self.extensions = extensions self.file_reader = file_reader self.string_reader = string_reader + self.opener_params = opener_params or { + "mode": "r", + "encoding": obj.get("ENCODING_FOR_DYNACONF", "utf-8"), + } @staticmethod def warn_not_installed(obj, identifier): # pragma: no cover @@ -77,12 +88,7 @@ def get_source_data(self, files): for source_file in files: if source_file.endswith(self.extensions): try: - with open( - source_file, - encoding=self.obj.get( - "ENCODING_FOR_DYNACONF", "utf-8" - ), - ) as open_file: + with open(source_file, **self.opener_params) as open_file: content = self.file_reader(open_file) self.obj._loaded_files.append(source_file) if content: diff --git a/dynaconf/loaders/toml_loader.py b/dynaconf/loaders/toml_loader.py index 372e0589e..22191acc7 100644 --- a/dynaconf/loaders/toml_loader.py +++ b/dynaconf/loaders/toml_loader.py @@ -1,13 +1,14 @@ from __future__ import annotations -import io +import warnings from pathlib import Path from dynaconf import default_settings from dynaconf.constants import TOML_EXTENSIONS from dynaconf.loaders.base import BaseLoader from dynaconf.utils import object_merge -from dynaconf.vendor import toml +from dynaconf.vendor import toml # Backwards compatibility with uiri/toml +from dynaconf.vendor import tomllib # New tomllib stdlib on py3.11 def load(obj, env=None, silent=True, key=None, filename=None): @@ -22,19 +23,54 @@ def load(obj, env=None, silent=True, key=None, filename=None): :return: None """ - loader = BaseLoader( - obj=obj, - env=env, - identifier="toml", - extensions=TOML_EXTENSIONS, - file_reader=toml.load, - string_reader=toml.loads, - ) - loader.load( - filename=filename, - key=key, - silent=silent, - ) + try: + loader = BaseLoader( + obj=obj, + env=env, + identifier="toml", + extensions=TOML_EXTENSIONS, + file_reader=tomllib.load, + string_reader=tomllib.loads, + opener_params={"mode": "rb"}, + ) + loader.load( + filename=filename, + key=key, + silent=silent, + ) + except UnicodeDecodeError: # pragma: no cover + """ + NOTE: Compat functions exists to keep backwards compatibility with + the new tomllib library. The old library was called `toml` and + the new one is called `tomllib`. + + The old lib uiri/toml allowed unicode characters and readed files + as string. + + The new tomllib (stdlib) does not allow unicode characters, only + utf-8 encoded, and read files as binary. + + NOTE: In dynaconf 4.0.0 we will drop support for the old library + removing the compat functions and calling directly the new lib. + """ + loader = BaseLoader( + obj=obj, + env=env, + identifier="toml", + extensions=TOML_EXTENSIONS, + file_reader=toml.load, + string_reader=toml.loads, + ) + loader.load( + filename=filename, + key=key, + silent=silent, + ) + + warnings.warn( + "TOML files should have only UTF-8 encoded characters. " + "starting on 4.0.0 dynaconf will stop allowing invalid chars.", + ) def write(settings_path, settings_data, merge=True): @@ -46,17 +82,33 @@ def write(settings_path, settings_data, merge=True): """ settings_path = Path(settings_path) if settings_path.exists() and merge: # pragma: no cover + try: # tomllib first + with open(str(settings_path), "rb") as open_file: + object_merge(tomllib.load(open_file), settings_data) + except UnicodeDecodeError: # pragma: no cover + # uiri/toml fallback (TBR on 4.0.0) + with open( + str(settings_path), + encoding=default_settings.ENCODING_FOR_DYNACONF, + ) as open_file: + object_merge(toml.load(open_file), settings_data) + + try: # tomllib first + with open(str(settings_path), "wb") as open_file: + tomllib.dump(encode_nulls(settings_data), open_file) + except UnicodeEncodeError: # pragma: no cover + # uiri/toml fallback (TBR on 4.0.0) with open( - str(settings_path), encoding=default_settings.ENCODING_FOR_DYNACONF + str(settings_path), + "w", + encoding=default_settings.ENCODING_FOR_DYNACONF, ) as open_file: - object_merge(toml.load(open_file), settings_data) - - with open( - str(settings_path), - "w", - encoding=default_settings.ENCODING_FOR_DYNACONF, - ) as open_file: - toml.dump(encode_nulls(settings_data), open_file) + toml.dump(encode_nulls(settings_data), open_file) + + warnings.warn( + "TOML files should have only UTF-8 encoded characters. " + "starting on 4.0.0 dynaconf will stop allowing invalid chars.", + ) def encode_nulls(data): diff --git a/dynaconf/utils/__init__.py b/dynaconf/utils/__init__.py index 78a08b007..8515fee14 100644 --- a/dynaconf/utils/__init__.py +++ b/dynaconf/utils/__init__.py @@ -55,23 +55,33 @@ def object_merge( existing_value = recursive_get(old, full_path) # doesn't handle None # Need to make every `None` on `_store` to be an wrapped `LazyNone` - for key, value in old.items(): - + # data coming from source, in `new` can be mix case: KEY4|key4|Key4 + # data existing on `old` object has the correct case: key4|KEY4|Key4 + # So we need to ensure that new keys matches the existing keys + for new_key in list(new.keys()): + correct_case_key = find_the_correct_casing(new_key, old) + if correct_case_key: + new[correct_case_key] = new.pop(new_key) + + for old_key, value in old.items(): + + # This is for when the dict exists internally + # but the new value on the end of full path is the same if ( existing_value is not None - and key.lower() == full_path[-1].lower() + and old_key.lower() == full_path[-1].lower() and existing_value is value ): # Here Be The Dragons # This comparison needs to be smarter continue - if key not in new: - new[key] = value + if old_key not in new: + new[old_key] = value else: object_merge( value, - new[key], + new[old_key], full_path=full_path[1:] if full_path else None, ) @@ -89,7 +99,7 @@ def recursive_get( """ if not names: return - head, tail = names[0], names[1:] + head, *tail = names result = getattr(obj, head, None) if not tail: return result @@ -106,7 +116,6 @@ def handle_metavalues( # MetaValue instances if getattr(new[key], "_dynaconf_reset", False): # pragma: no cover # a Reset on `new` triggers reasign of existing data - # @reset is deprecated on v3.0.0 new[key] = new[key].unwrap() elif getattr(new[key], "_dynaconf_del", False): # a Del on `new` triggers deletion of existing data @@ -424,3 +433,21 @@ def isnamedtupleinstance(value): if not isinstance(f, tuple): return False return all(type(n) == str for n in f) + + +def find_the_correct_casing(key: str, data: dict[str, Any]) -> str | None: + """Given a key, find the proper casing in data + + Arguments: + key {str} -- A key to be searched in data + data {dict} -- A dict to be searched + + Returns: + str -- The proper casing of the key in data + """ + if key in data: + return key + for k in data.keys(): + if k.lower() == key.lower(): + return k + return None diff --git a/dynaconf/utils/boxing.py b/dynaconf/utils/boxing.py index 0ae93b8ee..ff78f1246 100644 --- a/dynaconf/utils/boxing.py +++ b/dynaconf/utils/boxing.py @@ -3,8 +3,8 @@ import inspect from functools import wraps +from dynaconf.utils import find_the_correct_casing from dynaconf.utils import recursively_evaluate_lazy_format -from dynaconf.utils import upperfy from dynaconf.utils.functional import empty from dynaconf.vendor.box import Box @@ -37,7 +37,7 @@ def __getattr__(self, item, *args, **kwargs): try: return super().__getattr__(item, *args, **kwargs) except (AttributeError, KeyError): - n_item = item.lower() if item.isupper() else upperfy(item) + n_item = find_the_correct_casing(item, self) or item return super().__getattr__(n_item, *args, **kwargs) @evaluate_lazy_format @@ -45,7 +45,7 @@ def __getitem__(self, item, *args, **kwargs): try: return super().__getitem__(item, *args, **kwargs) except (AttributeError, KeyError): - n_item = item.lower() if item.isupper() else upperfy(item) + n_item = find_the_correct_casing(item, self) or item return super().__getitem__(n_item, *args, **kwargs) def __copy__(self): @@ -60,22 +60,11 @@ def copy(self): box_settings=self._box_config.get("box_settings"), ) - def _case_insensitive_get(self, item, default=None): - """adds a bit of overhead but allows case insensitive get - See issue: #486 - """ - lower_self = {k.casefold(): v for k, v in self.items()} - return lower_self.get(item.casefold(), default) - @evaluate_lazy_format def get(self, item, default=None, *args, **kwargs): - if item not in self: # toggle case - item = item.lower() if item.isupper() else upperfy(item) - value = super().get(item, empty, *args, **kwargs) - if value is empty: - # see Issue: #486 - return self._case_insensitive_get(item, default) - return value + n_item = find_the_correct_casing(item, self) or item + value = super().get(n_item, empty, *args, **kwargs) + return value if value is not empty else default def __dir__(self): keys = list(self.keys()) diff --git a/dynaconf/utils/parse_conf.py b/dynaconf/utils/parse_conf.py index 902ba2725..2fcf723f6 100644 --- a/dynaconf/utils/parse_conf.py +++ b/dynaconf/utils/parse_conf.py @@ -13,6 +13,7 @@ from dynaconf.utils.boxing import DynaBox from dynaconf.utils.functional import empty from dynaconf.vendor import toml +from dynaconf.vendor import tomllib try: from jinja2 import Environment @@ -277,10 +278,22 @@ def get_converter(converter_key, value, box_settings): def parse_with_toml(data): """Uses TOML syntax to parse data""" - try: - return toml.loads(f"key={data}")["key"] - except (toml.TomlDecodeError, KeyError): - return data + try: # try tomllib first + try: + return tomllib.loads(f"key={data}")["key"] + except (tomllib.TOMLDecodeError, KeyError): + return data + except UnicodeDecodeError: # pragma: no cover + # fallback to toml (TBR in 4.0.0) + try: + return toml.loads(f"key={data}")["key"] + except (toml.TomlDecodeError, KeyError): + return data + warnings.warn( + "TOML files should have only UTF-8 encoded characters. " + "starting on 4.0.0 dynaconf will stop allowing invalid chars.", + DeprecationWarning, + ) def _parse_conf_data(data, tomlfy=False, box_settings=None): @@ -309,7 +322,8 @@ def _parse_conf_data(data, tomlfy=False, box_settings=None): ): # Check combination token is used comb_token = re.match( - r"^@(str|int|float|bool|json) @(jinja|format)", data + f"^({'|'.join(converters.keys())}) @(jinja|format)", + data, ) if comb_token: tokens = comb_token.group(0) @@ -334,7 +348,7 @@ def _parse_conf_data(data, tomlfy=False, box_settings=None): def parse_conf_data(data, tomlfy=False, box_settings=None): - # fix for https://github.com/rochacbruno/dynaconf/issues/595 + # fix for https://github.com/dynaconf/dynaconf/issues/595 if isnamedtupleinstance(data): return data @@ -342,7 +356,6 @@ def parse_conf_data(data, tomlfy=False, box_settings=None): box_settings = box_settings or {} if isinstance(data, (tuple, list)): - # recursively parse each sequence item return [ parse_conf_data(item, tomlfy=tomlfy, box_settings=box_settings) @@ -358,13 +371,6 @@ def parse_conf_data(data, tomlfy=False, box_settings=None): ) return _parsed - if ( - isinstance(data, str) - and data.startswith(("+", "-")) - and data[1:].isdigit() - ): - return data - # return parsed string value return _parse_conf_data(data, tomlfy=tomlfy, box_settings=box_settings) diff --git a/dynaconf/validator.py b/dynaconf/validator.py index f3c581d46..60dbfded8 100644 --- a/dynaconf/validator.py +++ b/dynaconf/validator.py @@ -135,6 +135,9 @@ def __init__( self.envs: Sequence[str] | None = None self.apply_default_on_none = apply_default_on_none + # See #585 + self.is_type_of = operations.get("is_type_of") + if isinstance(env, str): self.envs = [env] elif isinstance(env, (list, tuple)): @@ -243,6 +246,20 @@ def _validate_items( else: default_value = empty + # THIS IS A FIX FOR #585 in contrast with #799 + # toml considers signed strings "+-1" as integers + # however existing users are passing strings + # to default on validator (see #585) + # The solution we added on #667 introduced a new problem + # This fix here makes it to work for both cases. + if ( + isinstance(default_value, str) + and default_value.startswith(("+", "-")) + and self.is_type_of is str + ): + # avoid TOML from parsing "+-1" as integer + default_value = f"'{default_value}'" + value = self.cast( settings.setdefault( name, diff --git a/dynaconf/vendor/box/README.md b/dynaconf/vendor/box/README.md deleted file mode 100644 index 49e2185f4..000000000 --- a/dynaconf/vendor/box/README.md +++ /dev/null @@ -1,6 +0,0 @@ -## python-box - -Vendored dep taken from: https://github.com/cdgriffith/Box -Licensed under MIT: https://github.com/cdgriffith/Box/blob/master/LICENSE - -Current version: 4.2.3 diff --git a/dynaconf/vendor/box/__init__.py b/dynaconf/vendor/box/__init__.py index fb02ed843..ad571e425 100644 --- a/dynaconf/vendor/box/__init__.py +++ b/dynaconf/vendor/box/__init__.py @@ -1,8 +1,15 @@ -__author__='Chris Griffith' -__version__='4.2.3' +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +__author__ = 'Chris Griffith' +__version__ = '4.2.3' + from .box import Box from .box_list import BoxList from .config_box import ConfigBox from .shorthand_box import SBox -from .exceptions import BoxError,BoxKeyError -from .from_file import box_from_file \ No newline at end of file +from .exceptions import BoxError, BoxKeyError +from .from_file import box_from_file + + + diff --git a/dynaconf/vendor/box/box.py b/dynaconf/vendor/box/box.py index 5e14910a7..0b4c1d283 100644 --- a/dynaconf/vendor/box/box.py +++ b/dynaconf/vendor/box/box.py @@ -1,327 +1,689 @@ -_Z='keys' -_Y='box_settings' -_X='default_box_attr' -_W='Box is frozen' -_V='modify_tuples_box' -_U='box_safe_prefix' -_T='default_box_none_transform' -_S='__created' -_R='box_dots' -_Q='box_duplicates' -_P='ignore' -_O='.' -_N='strict' -_M='box_recast' -_L='box_intact_types' -_K='default_box' -_J='_' -_I='utf-8' -_H='_box_config' -_G=True -_F='camel_killer_box' -_E='conversion_box' -_D='frozen_box' -_C='__safe_keys' -_B=False -_A=None -import copy,re,string,warnings -from collections.abc import Iterable,Mapping,Callable +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# +# Copyright (c) 2017-2020 - Chris Griffith - MIT License +""" +Improved dictionary access through dot notation with additional tools. +""" +import copy +import re +import string +import warnings +from collections.abc import Iterable, Mapping, Callable from keyword import kwlist from pathlib import Path -from typing import Any,Union,Tuple,List,Dict +from typing import Any, Union, Tuple, List, Dict + from dynaconf.vendor import box -from .converters import _to_json,_from_json,_from_toml,_to_toml,_from_yaml,_to_yaml,BOX_PARAMETERS -from .exceptions import BoxError,BoxKeyError,BoxTypeError,BoxValueError,BoxWarning -__all__=['Box'] -_first_cap_re=re.compile('(.)([A-Z][a-z]+)') -_all_cap_re=re.compile('([a-z0-9])([A-Z])') -_list_pos_re=re.compile('\\[(\\d+)\\]') -NO_DEFAULT=object() -def _camel_killer(attr):D='\\1_\\2';A=attr;A=str(A);B=_first_cap_re.sub(D,A);C=_all_cap_re.sub(D,B);return re.sub(' *_+',_J,C.lower()) -def _recursive_tuples(iterable,box_class,recreate_tuples=_B,**E): - D=recreate_tuples;C=box_class;B=[] - for A in iterable: - if isinstance(A,dict):B.append(C(A,**E)) - elif isinstance(A,list)or D and isinstance(A,tuple):B.append(_recursive_tuples(A,C,D,**E)) - else:B.append(A) - return tuple(B) +from .converters import (_to_json, _from_json, _from_toml, _to_toml, _from_yaml, _to_yaml, BOX_PARAMETERS) +from .exceptions import BoxError, BoxKeyError, BoxTypeError, BoxValueError, BoxWarning + +__all__ = ['Box'] + +_first_cap_re = re.compile('(.)([A-Z][a-z]+)') +_all_cap_re = re.compile('([a-z0-9])([A-Z])') +_list_pos_re = re.compile(r'\[(\d+)\]') + +# a sentinel object for indicating no default, in order to allow users +# to pass `None` as a valid default value +NO_DEFAULT = object() + + +def _camel_killer(attr): + """ + CamelKiller, qu'est-ce que c'est? + + Taken from http://stackoverflow.com/a/1176023/3244542 + """ + attr = str(attr) + + s1 = _first_cap_re.sub(r'\1_\2', attr) + s2 = _all_cap_re.sub(r'\1_\2', s1) + return re.sub(' *_+', '_', s2.lower()) + + +def _recursive_tuples(iterable, box_class, recreate_tuples=False, **kwargs): + out_list = [] + for i in iterable: + if isinstance(i, dict): + out_list.append(box_class(i, **kwargs)) + elif isinstance(i, list) or (recreate_tuples and isinstance(i, tuple)): + out_list.append(_recursive_tuples(i, box_class, recreate_tuples, **kwargs)) + else: + out_list.append(i) + return tuple(out_list) + + def _parse_box_dots(item): - A=item - for (B,C) in enumerate(A): - if C=='[':return A[:B],A[B:] - elif C==_O:return A[:B],A[B+1:] - raise BoxError('Could not split box dots properly') -def _get_box_config():return{_S:_B,_C:{}} + for idx, char in enumerate(item): + if char == '[': + return item[:idx], item[idx:] + elif char == '.': + return item[:idx], item[idx + 1:] + raise BoxError('Could not split box dots properly') + + +def _get_box_config(): + return { + # Internal use only + '__created': False, + '__safe_keys': {} + } + + class Box(dict): - _protected_keys=['to_dict','to_json','to_yaml','from_yaml','from_json','from_toml','to_toml','merge_update']+[A for A in dir({})if not A.startswith(_J)] - def __new__(A,*D,box_settings=_A,default_box=_B,default_box_attr=NO_DEFAULT,default_box_none_transform=_G,frozen_box=_B,camel_killer_box=_B,conversion_box=_G,modify_tuples_box=_B,box_safe_prefix='x',box_duplicates=_P,box_intact_types=(),box_recast=_A,box_dots=_B,**E):C=default_box_attr;B=super(Box,A).__new__(A,*D,**E);B._box_config=_get_box_config();B._box_config.update({_K:default_box,_X:A.__class__ if C is NO_DEFAULT else C,_T:default_box_none_transform,_E:conversion_box,_U:box_safe_prefix,_D:frozen_box,_F:camel_killer_box,_V:modify_tuples_box,_Q:box_duplicates,_L:tuple(box_intact_types),_M:box_recast,_R:box_dots,_Y:box_settings or{}});return B - def __init__(A,*B,box_settings=_A,default_box=_B,default_box_attr=NO_DEFAULT,default_box_none_transform=_G,frozen_box=_B,camel_killer_box=_B,conversion_box=_G,modify_tuples_box=_B,box_safe_prefix='x',box_duplicates=_P,box_intact_types=(),box_recast=_A,box_dots=_B,**F): - E=default_box_attr;super().__init__();A._box_config=_get_box_config();A._box_config.update({_K:default_box,_X:A.__class__ if E is NO_DEFAULT else E,_T:default_box_none_transform,_E:conversion_box,_U:box_safe_prefix,_D:frozen_box,_F:camel_killer_box,_V:modify_tuples_box,_Q:box_duplicates,_L:tuple(box_intact_types),_M:box_recast,_R:box_dots,_Y:box_settings or{}}) - if not A._box_config[_E]and A._box_config[_Q]!=_P:raise BoxError('box_duplicates are only for conversion_boxes') - if len(B)==1: - if isinstance(B[0],str):raise BoxValueError('Cannot extrapolate Box from string') - if isinstance(B[0],Mapping): - for (D,C) in B[0].items(): - if C is B[0]:C=A - if C is _A and A._box_config[_K]and A._box_config[_T]:continue - A.__setitem__(D,C) - elif isinstance(B[0],Iterable): - for (D,C) in B[0]:A.__setitem__(D,C) - else:raise BoxValueError('First argument must be mapping or iterable') - elif B:raise BoxTypeError(f"Box expected at most 1 argument, got {len(B)}") - for (D,C) in F.items(): - if B and isinstance(B[0],Mapping)and C is B[0]:C=A - A.__setitem__(D,C) - A._box_config[_S]=_G - def __add__(C,other): - A=other;B=C.copy() - if not isinstance(A,dict):raise BoxTypeError(f"Box can only merge two boxes or a box and a dictionary.") - B.merge_update(A);return B - def __hash__(A): - if A._box_config[_D]: - B=54321 - for C in A.items():B^=hash(C) - return B - raise BoxTypeError('unhashable type: "Box"') - def __dir__(B): - D=string.ascii_letters+string.digits+_J;C=set(super().__dir__()) - for A in B.keys(): - A=str(A) - if' 'not in A and A[0]not in string.digits and A not in kwlist: - for E in A: - if E not in D:break - else:C.add(A) - for A in B.keys(): - if A not in C: - if B._box_config[_E]: - A=B._safe_attr(A) - if A:C.add(A) - return list(C) - def get(B,key,default=NO_DEFAULT): - C=key;A=default - if C not in B: - if A is NO_DEFAULT: - if B._box_config[_K]and B._box_config[_T]:return B.__get_default(C) - else:return _A - if isinstance(A,dict)and not isinstance(A,Box):return Box(A,box_settings=B._box_config.get(_Y)) - if isinstance(A,list)and not isinstance(A,box.BoxList):return box.BoxList(A) - return A - return B[C] - def copy(A):return Box(super().copy(),**A.__box_config()) - def __copy__(A):return Box(super().copy(),**A.__box_config()) - def __deepcopy__(A,memodict=_A): - B=memodict;E=A._box_config[_D];D=A.__box_config();D[_D]=_B;C=A.__class__(**D);B=B or{};B[id(A)]=C - for (F,G) in A.items():C[copy.deepcopy(F,B)]=copy.deepcopy(G,B) - C._box_config[_D]=E;return C - def __setstate__(A,state):B=state;A._box_config=B[_H];A.__dict__.update(B) - def keys(A):return super().keys() - def values(A):return[A[B]for B in A.keys()] - def items(A):return[(B,A[B])for B in A.keys()] - def __get_default(B,item): - A=B._box_config[_X] - if A in(B.__class__,dict):C=B.__class__(**B.__box_config()) - elif isinstance(A,dict):C=B.__class__(**B.__box_config(),**A) - elif isinstance(A,list):C=box.BoxList(**B.__box_config()) - elif isinstance(A,Callable):C=A() - elif hasattr(A,'copy'):C=A.copy() - else:C=A - B.__convert_and_store(item,C);return C - def __box_config(C): - A={} - for (B,D) in C._box_config.copy().items(): - if not B.startswith('__'):A[B]=D - return A - def __recast(A,item,value): - C=value;B=item - if A._box_config[_M]and B in A._box_config[_M]: - try:return A._box_config[_M][B](C) - except ValueError:raise BoxValueError(f"Cannot convert {C} to {A._box_config[_M][B]}") from _A - return C - def __convert_and_store(B,item,value): - C=item;A=value - if B._box_config[_E]:D=B._safe_attr(C);B._box_config[_C][D]=C - if isinstance(A,(int,float,str,bytes,bytearray,bool,complex,set,frozenset)):return super().__setitem__(C,A) - if B._box_config[_L]and isinstance(A,B._box_config[_L]):return super().__setitem__(C,A) - if isinstance(A,dict)and not isinstance(A,Box):A=B.__class__(A,**B.__box_config()) - elif isinstance(A,list)and not isinstance(A,box.BoxList): - if B._box_config[_D]:A=_recursive_tuples(A,B.__class__,recreate_tuples=B._box_config[_V],**B.__box_config()) - else:A=box.BoxList(A,box_class=B.__class__,**B.__box_config()) - elif B._box_config[_V]and isinstance(A,tuple):A=_recursive_tuples(A,B.__class__,recreate_tuples=_G,**B.__box_config()) - super().__setitem__(C,A) - def __getitem__(B,item,_ignore_default=_B): - A=item - try:return super().__getitem__(A) - except KeyError as E: - if A==_H:raise BoxKeyError('_box_config should only exist as an attribute and is never defaulted') from _A - if B._box_config[_R]and isinstance(A,str)and(_O in A or'['in A): - C,F=_parse_box_dots(A) - if C in B.keys(): - if hasattr(B[C],'__getitem__'):return B[C][F] - if B._box_config[_F]and isinstance(A,str): - D=_camel_killer(A) - if D in B.keys():return super().__getitem__(D) - if B._box_config[_K]and not _ignore_default:return B.__get_default(A) - raise BoxKeyError(str(E)) from _A - def __getattr__(A,item): - B=item - try: - try:C=A.__getitem__(B,_ignore_default=_G) - except KeyError:C=object.__getattribute__(A,B) - except AttributeError as E: - if B=='__getstate__':raise BoxKeyError(B) from _A - if B==_H:raise BoxError('_box_config key must exist') from _A - if A._box_config[_E]: - D=A._safe_attr(B) - if D in A._box_config[_C]:return A.__getitem__(A._box_config[_C][D]) - if A._box_config[_K]:return A.__get_default(B) - raise BoxKeyError(str(E)) from _A - return C - def __setitem__(A,key,value): - C=value;B=key - if B!=_H and A._box_config[_S]and A._box_config[_D]:raise BoxError(_W) - if A._box_config[_R]and isinstance(B,str)and _O in B: - D,E=_parse_box_dots(B) - if D in A.keys(): - if hasattr(A[D],'__setitem__'):return A[D].__setitem__(E,C) - C=A.__recast(B,C) - if B not in A.keys()and A._box_config[_F]: - if A._box_config[_F]and isinstance(B,str):B=_camel_killer(B) - if A._box_config[_E]and A._box_config[_Q]!=_P:A._conversion_checks(B) - A.__convert_and_store(B,C) - def __setattr__(A,key,value): - C=value;B=key - if B!=_H and A._box_config[_D]and A._box_config[_S]:raise BoxError(_W) - if B in A._protected_keys:raise BoxKeyError(f'Key name "{B}" is protected') - if B==_H:return object.__setattr__(A,B,C) - C=A.__recast(B,C);D=A._safe_attr(B) - if D in A._box_config[_C]:B=A._box_config[_C][D] - A.__setitem__(B,C) - def __delitem__(A,key): - B=key - if A._box_config[_D]:raise BoxError(_W) - if B not in A.keys()and A._box_config[_R]and isinstance(B,str)and _O in B: - C,E=B.split(_O,1) - if C in A.keys()and isinstance(A[C],dict):return A[C].__delitem__(E) - if B not in A.keys()and A._box_config[_F]: - if A._box_config[_F]and isinstance(B,str): - for D in A: - if _camel_killer(B)==D:B=D;break - super().__delitem__(B) - def __delattr__(A,item): - B=item - if A._box_config[_D]:raise BoxError(_W) - if B==_H:raise BoxError('"_box_config" is protected') - if B in A._protected_keys:raise BoxKeyError(f'Key name "{B}" is protected') - try:A.__delitem__(B) - except KeyError as D: - if A._box_config[_E]: - C=A._safe_attr(B) - if C in A._box_config[_C]:A.__delitem__(A._box_config[_C][C]);del A._box_config[_C][C];return - raise BoxKeyError(D) - def pop(B,key,*C): - A=key - if C: - if len(C)!=1:raise BoxError('pop() takes only one optional argument "default"') - try:D=B[A] - except KeyError:return C[0] - else:del B[A];return D - try:D=B[A] - except KeyError:raise BoxKeyError('{0}'.format(A)) from _A - else:del B[A];return D - def clear(A):super().clear();A._box_config[_C].clear() - def popitem(A): - try:B=next(A.__iter__()) - except StopIteration:raise BoxKeyError('Empty box') from _A - return B,A.pop(B) - def __repr__(A):return f"" - def __str__(A):return str(A.to_dict()) - def __iter__(A): - for B in A.keys():yield B - def __reversed__(A): - for B in reversed(list(A.keys())):yield B - def to_dict(D): - A=dict(D) - for (C,B) in A.items(): - if B is D:A[C]=A - elif isinstance(B,Box):A[C]=B.to_dict() - elif isinstance(B,box.BoxList):A[C]=B.to_list() - return A - def update(C,__m=_A,**D): - B=__m - if B: - if hasattr(B,_Z): - for A in B:C.__convert_and_store(A,B[A]) - else: - for (A,E) in B:C.__convert_and_store(A,E) - for A in D:C.__convert_and_store(A,D[A]) - def merge_update(A,__m=_A,**E): - C=__m - def D(k,v): - B=A._box_config[_L]and isinstance(v,A._box_config[_L]) - if isinstance(v,dict)and not B: - v=A.__class__(v,**A.__box_config()) - if k in A and isinstance(A[k],dict): - if isinstance(A[k],Box):A[k].merge_update(v) - else:A[k].update(v) - return - if isinstance(v,list)and not B:v=box.BoxList(v,**A.__box_config()) - A.__setitem__(k,v) - if C: - if hasattr(C,_Z): - for B in C:D(B,C[B]) - else: - for (B,F) in C:D(B,F) - for B in E:D(B,E[B]) - def setdefault(B,item,default=_A): - C=item;A=default - if C in B:return B[C] - if isinstance(A,dict):A=B.__class__(A,**B.__box_config()) - if isinstance(A,list):A=box.BoxList(A,box_class=B.__class__,**B.__box_config()) - B[C]=A;return A - def _safe_attr(C,attr): - B=attr;G=string.ascii_letters+string.digits+_J - if isinstance(B,tuple):B=_J.join([str(A)for A in B]) - B=B.decode(_I,_P)if isinstance(B,bytes)else str(B) - if C.__box_config()[_F]:B=_camel_killer(B) - A=[];D=0 - for (E,F) in enumerate(B): - if F in G:D=E;A.append(F) - elif not A:continue - elif D==E-1:A.append(_J) - A=''.join(A)[:D+1] - try:int(A[0]) - except (ValueError,IndexError):pass - else:A=f"{C.__box_config()[_U]}{A}" - if A in kwlist:A=f"{C.__box_config()[_U]}{A}" - return A - def _conversion_checks(A,item): - B=A._safe_attr(item) - if B in A._box_config[_C]: - C=[f"{item}({B})",f"{A._box_config[_C][B]}({B})"] - if A._box_config[_Q].startswith('warn'):warnings.warn(f"Duplicate conversion attributes exist: {C}",BoxWarning) - else:raise BoxError(f"Duplicate conversion attributes exist: {C}") - def to_json(A,filename=_A,encoding=_I,errors=_N,**B):return _to_json(A.to_dict(),filename=filename,encoding=encoding,errors=errors,**B) - @classmethod - def from_json(E,json_string=_A,filename=_A,encoding=_I,errors=_N,**A): - D={} - for B in A.copy(): - if B in BOX_PARAMETERS:D[B]=A.pop(B) - C=_from_json(json_string,filename=filename,encoding=encoding,errors=errors,**A) - if not isinstance(C,dict):raise BoxError(f"json data not returned as a dictionary, but rather a {type(C).__name__}") - return E(C,**D) - def to_yaml(A,filename=_A,default_flow_style=_B,encoding=_I,errors=_N,**B):return _to_yaml(A.to_dict(),filename=filename,default_flow_style=default_flow_style,encoding=encoding,errors=errors,**B) - @classmethod - def from_yaml(E,yaml_string=_A,filename=_A,encoding=_I,errors=_N,**A): - D={} - for B in A.copy(): - if B in BOX_PARAMETERS:D[B]=A.pop(B) - C=_from_yaml(yaml_string=yaml_string,filename=filename,encoding=encoding,errors=errors,**A) - if not isinstance(C,dict):raise BoxError(f"yaml data not returned as a dictionary but rather a {type(C).__name__}") - return E(C,**D) - def to_toml(A,filename=_A,encoding=_I,errors=_N):return _to_toml(A.to_dict(),filename=filename,encoding=encoding,errors=errors) - @classmethod - def from_toml(D,toml_string=_A,filename=_A,encoding=_I,errors=_N,**B): - C={} - for A in B.copy(): - if A in BOX_PARAMETERS:C[A]=B.pop(A) - E=_from_toml(toml_string=toml_string,filename=filename,encoding=encoding,errors=errors);return D(E,**C) \ No newline at end of file + """ + Improved dictionary access through dot notation with additional tools. + + :param default_box: Similar to defaultdict, return a default value + :param default_box_attr: Specify the default replacement. + WARNING: If this is not the default 'Box', it will not be recursive + :param default_box_none_transform: When using default_box, treat keys with none values as absent. True by default + :param frozen_box: After creation, the box cannot be modified + :param camel_killer_box: Convert CamelCase to snake_case + :param conversion_box: Check for near matching keys as attributes + :param modify_tuples_box: Recreate incoming tuples with dicts into Boxes + :param box_safe_prefix: Conversion box prefix for unsafe attributes + :param box_duplicates: "ignore", "error" or "warn" when duplicates exists in a conversion_box + :param box_intact_types: tuple of types to ignore converting + :param box_recast: cast certain keys to a specified type + :param box_dots: access nested Boxes by period separated keys in string + """ + + _protected_keys = [ + "to_dict", + "to_json", + "to_yaml", + "from_yaml", + "from_json", + "from_toml", + "to_toml", + "merge_update", + ] + [attr for attr in dir({}) if not attr.startswith("_")] + + def __new__(cls, *args: Any, box_settings: Any = None, default_box: bool = False, default_box_attr: Any = NO_DEFAULT, + default_box_none_transform: bool = True, frozen_box: bool = False, camel_killer_box: bool = False, + conversion_box: bool = True, modify_tuples_box: bool = False, box_safe_prefix: str = 'x', + box_duplicates: str = 'ignore', box_intact_types: Union[Tuple, List] = (), + box_recast: Dict = None, box_dots: bool = False, **kwargs: Any): + """ + Due to the way pickling works in python 3, we need to make sure + the box config is created as early as possible. + """ + obj = super(Box, cls).__new__(cls, *args, **kwargs) + obj._box_config = _get_box_config() + obj._box_config.update({ + 'default_box': default_box, + 'default_box_attr': cls.__class__ if default_box_attr is NO_DEFAULT else default_box_attr, + 'default_box_none_transform': default_box_none_transform, + 'conversion_box': conversion_box, + 'box_safe_prefix': box_safe_prefix, + 'frozen_box': frozen_box, + 'camel_killer_box': camel_killer_box, + 'modify_tuples_box': modify_tuples_box, + 'box_duplicates': box_duplicates, + 'box_intact_types': tuple(box_intact_types), + 'box_recast': box_recast, + 'box_dots': box_dots, + 'box_settings': box_settings or {} + }) + return obj + + def __init__(self, *args: Any, box_settings: Any = None, default_box: bool = False, default_box_attr: Any = NO_DEFAULT, + default_box_none_transform: bool = True, frozen_box: bool = False, camel_killer_box: bool = False, + conversion_box: bool = True, modify_tuples_box: bool = False, box_safe_prefix: str = 'x', + box_duplicates: str = 'ignore', box_intact_types: Union[Tuple, List] = (), + box_recast: Dict = None, box_dots: bool = False, **kwargs: Any): + super().__init__() + self._box_config = _get_box_config() + self._box_config.update({ + 'default_box': default_box, + 'default_box_attr': self.__class__ if default_box_attr is NO_DEFAULT else default_box_attr, + 'default_box_none_transform': default_box_none_transform, + 'conversion_box': conversion_box, + 'box_safe_prefix': box_safe_prefix, + 'frozen_box': frozen_box, + 'camel_killer_box': camel_killer_box, + 'modify_tuples_box': modify_tuples_box, + 'box_duplicates': box_duplicates, + 'box_intact_types': tuple(box_intact_types), + 'box_recast': box_recast, + 'box_dots': box_dots, + 'box_settings': box_settings or {} + }) + if not self._box_config['conversion_box'] and self._box_config['box_duplicates'] != 'ignore': + raise BoxError('box_duplicates are only for conversion_boxes') + if len(args) == 1: + if isinstance(args[0], str): + raise BoxValueError('Cannot extrapolate Box from string') + if isinstance(args[0], Mapping): + for k, v in args[0].items(): + if v is args[0]: + v = self + + if v is None and self._box_config['default_box'] and self._box_config['default_box_none_transform']: + continue + self.__setitem__(k, v) + elif isinstance(args[0], Iterable): + for k, v in args[0]: + self.__setitem__(k, v) + else: + raise BoxValueError('First argument must be mapping or iterable') + elif args: + raise BoxTypeError(f'Box expected at most 1 argument, got {len(args)}') + + for k, v in kwargs.items(): + if args and isinstance(args[0], Mapping) and v is args[0]: + v = self + self.__setitem__(k, v) + + self._box_config['__created'] = True + + def __add__(self, other: dict): + new_box = self.copy() + if not isinstance(other, dict): + raise BoxTypeError(f'Box can only merge two boxes or a box and a dictionary.') + new_box.merge_update(other) + return new_box + + def __hash__(self): + if self._box_config['frozen_box']: + hashing = 54321 + for item in self.items(): + hashing ^= hash(item) + return hashing + raise BoxTypeError('unhashable type: "Box"') + + def __dir__(self): + allowed = string.ascii_letters + string.digits + '_' + items = set(super().__dir__()) + # Only show items accessible by dot notation + for key in self.keys(): + key = str(key) + if ' ' not in key and key[0] not in string.digits and key not in kwlist: + for letter in key: + if letter not in allowed: + break + else: + items.add(key) + + for key in self.keys(): + if key not in items: + if self._box_config['conversion_box']: + key = self._safe_attr(key) + if key: + items.add(key) + + return list(items) + + def get(self, key, default=NO_DEFAULT): + if key not in self: + if default is NO_DEFAULT: + if self._box_config['default_box'] and self._box_config['default_box_none_transform']: + return self.__get_default(key) + else: + return None + if isinstance(default, dict) and not isinstance(default, Box): + return Box(default, box_settings=self._box_config.get("box_settings")) + if isinstance(default, list) and not isinstance(default, box.BoxList): + return box.BoxList(default) + return default + return self[key] + + def copy(self): + return Box(super().copy(), **self.__box_config()) + + def __copy__(self): + return Box(super().copy(), **self.__box_config()) + + def __deepcopy__(self, memodict=None): + frozen = self._box_config['frozen_box'] + config = self.__box_config() + config['frozen_box'] = False + out = self.__class__(**config) + memodict = memodict or {} + memodict[id(self)] = out + for k, v in self.items(): + out[copy.deepcopy(k, memodict)] = copy.deepcopy(v, memodict) + out._box_config['frozen_box'] = frozen + return out + + def __setstate__(self, state): + self._box_config = state['_box_config'] + self.__dict__.update(state) + + def keys(self): + return super().keys() + + def values(self): + return [self[x] for x in self.keys()] + + def items(self): + return [(x, self[x]) for x in self.keys()] + + def __get_default(self, item): + default_value = self._box_config['default_box_attr'] + if default_value in (self.__class__, dict): + value = self.__class__(**self.__box_config()) + elif isinstance(default_value, dict): + value = self.__class__(**self.__box_config(), **default_value) + elif isinstance(default_value, list): + value = box.BoxList(**self.__box_config()) + elif isinstance(default_value, Callable): + value = default_value() + elif hasattr(default_value, 'copy'): + value = default_value.copy() + else: + value = default_value + self.__convert_and_store(item, value) + return value + + def __box_config(self): + out = {} + for k, v in self._box_config.copy().items(): + if not k.startswith('__'): + out[k] = v + return out + + def __recast(self, item, value): + if self._box_config['box_recast'] and item in self._box_config['box_recast']: + try: + return self._box_config['box_recast'][item](value) + except ValueError: + raise BoxValueError(f'Cannot convert {value} to {self._box_config["box_recast"][item]}') from None + return value + + def __convert_and_store(self, item, value): + if self._box_config['conversion_box']: + safe_key = self._safe_attr(item) + self._box_config['__safe_keys'][safe_key] = item + if isinstance(value, (int, float, str, bytes, bytearray, bool, complex, set, frozenset)): + return super().__setitem__(item, value) + # If the value has already been converted or should not be converted, return it as-is + if self._box_config['box_intact_types'] and isinstance(value, self._box_config['box_intact_types']): + return super().__setitem__(item, value) + # This is the magic sauce that makes sub dictionaries into new box objects + if isinstance(value, dict) and not isinstance(value, Box): + value = self.__class__(value, **self.__box_config()) + elif isinstance(value, list) and not isinstance(value, box.BoxList): + if self._box_config['frozen_box']: + value = _recursive_tuples(value, + self.__class__, + recreate_tuples=self._box_config['modify_tuples_box'], + **self.__box_config()) + else: + value = box.BoxList(value, box_class=self.__class__, **self.__box_config()) + elif self._box_config['modify_tuples_box'] and isinstance(value, tuple): + value = _recursive_tuples(value, self.__class__, recreate_tuples=True, **self.__box_config()) + super().__setitem__(item, value) + + def __getitem__(self, item, _ignore_default=False): + try: + return super().__getitem__(item) + except KeyError as err: + if item == '_box_config': + raise BoxKeyError('_box_config should only exist as an attribute and is never defaulted') from None + if self._box_config['box_dots'] and isinstance(item, str) and ('.' in item or '[' in item): + first_item, children = _parse_box_dots(item) + if first_item in self.keys(): + if hasattr(self[first_item], '__getitem__'): + return self[first_item][children] + if self._box_config['camel_killer_box'] and isinstance(item, str): + converted = _camel_killer(item) + if converted in self.keys(): + return super().__getitem__(converted) + if self._box_config['default_box'] and not _ignore_default: + return self.__get_default(item) + raise BoxKeyError(str(err)) from None + + def __getattr__(self, item): + try: + try: + value = self.__getitem__(item, _ignore_default=True) + except KeyError: + value = object.__getattribute__(self, item) + except AttributeError as err: + if item == '__getstate__': + raise BoxKeyError(item) from None + if item == '_box_config': + raise BoxError('_box_config key must exist') from None + if self._box_config['conversion_box']: + safe_key = self._safe_attr(item) + if safe_key in self._box_config['__safe_keys']: + return self.__getitem__(self._box_config['__safe_keys'][safe_key]) + if self._box_config['default_box']: + return self.__get_default(item) + raise BoxKeyError(str(err)) from None + return value + + def __setitem__(self, key, value): + if key != '_box_config' and self._box_config['__created'] and self._box_config['frozen_box']: + raise BoxError('Box is frozen') + if self._box_config['box_dots'] and isinstance(key, str) and '.' in key: + first_item, children = _parse_box_dots(key) + if first_item in self.keys(): + if hasattr(self[first_item], '__setitem__'): + return self[first_item].__setitem__(children, value) + value = self.__recast(key, value) + if key not in self.keys() and self._box_config['camel_killer_box']: + if self._box_config['camel_killer_box'] and isinstance(key, str): + key = _camel_killer(key) + if self._box_config['conversion_box'] and self._box_config['box_duplicates'] != 'ignore': + self._conversion_checks(key) + self.__convert_and_store(key, value) + + def __setattr__(self, key, value): + if key != '_box_config' and self._box_config['frozen_box'] and self._box_config['__created']: + raise BoxError('Box is frozen') + if key in self._protected_keys: + raise BoxKeyError(f'Key name "{key}" is protected') + if key == '_box_config': + return object.__setattr__(self, key, value) + value = self.__recast(key, value) + safe_key = self._safe_attr(key) + if safe_key in self._box_config['__safe_keys']: + key = self._box_config['__safe_keys'][safe_key] + self.__setitem__(key, value) + + def __delitem__(self, key): + if self._box_config['frozen_box']: + raise BoxError('Box is frozen') + if key not in self.keys() and self._box_config['box_dots'] and isinstance(key, str) and '.' in key: + first_item, children = key.split('.', 1) + if first_item in self.keys() and isinstance(self[first_item], dict): + return self[first_item].__delitem__(children) + if key not in self.keys() and self._box_config['camel_killer_box']: + if self._box_config['camel_killer_box'] and isinstance(key, str): + for each_key in self: + if _camel_killer(key) == each_key: + key = each_key + break + super().__delitem__(key) + + def __delattr__(self, item): + if self._box_config['frozen_box']: + raise BoxError('Box is frozen') + if item == '_box_config': + raise BoxError('"_box_config" is protected') + if item in self._protected_keys: + raise BoxKeyError(f'Key name "{item}" is protected') + try: + self.__delitem__(item) + except KeyError as err: + if self._box_config['conversion_box']: + safe_key = self._safe_attr(item) + if safe_key in self._box_config['__safe_keys']: + self.__delitem__(self._box_config['__safe_keys'][safe_key]) + del self._box_config['__safe_keys'][safe_key] + return + raise BoxKeyError(err) + + def pop(self, key, *args): + if args: + if len(args) != 1: + raise BoxError('pop() takes only one optional argument "default"') + try: + item = self[key] + except KeyError: + return args[0] + else: + del self[key] + return item + try: + item = self[key] + except KeyError: + raise BoxKeyError('{0}'.format(key)) from None + else: + del self[key] + return item + + def clear(self): + super().clear() + self._box_config['__safe_keys'].clear() + + def popitem(self): + try: + key = next(self.__iter__()) + except StopIteration: + raise BoxKeyError('Empty box') from None + return key, self.pop(key) + + def __repr__(self): + return f'' + + def __str__(self): + return str(self.to_dict()) + + def __iter__(self): + for key in self.keys(): + yield key + + def __reversed__(self): + for key in reversed(list(self.keys())): + yield key + + def to_dict(self): + """ + Turn the Box and sub Boxes back into a native python dictionary. + + :return: python dictionary of this Box + """ + out_dict = dict(self) + for k, v in out_dict.items(): + if v is self: + out_dict[k] = out_dict + elif isinstance(v, Box): + out_dict[k] = v.to_dict() + elif isinstance(v, box.BoxList): + out_dict[k] = v.to_list() + return out_dict + + def update(self, __m=None, **kwargs): + if __m: + if hasattr(__m, 'keys'): + for k in __m: + self.__convert_and_store(k, __m[k]) + else: + for k, v in __m: + self.__convert_and_store(k, v) + for k in kwargs: + self.__convert_and_store(k, kwargs[k]) + + def merge_update(self, __m=None, **kwargs): + def convert_and_set(k, v): + intact_type = (self._box_config['box_intact_types'] and isinstance(v, self._box_config['box_intact_types'])) + if isinstance(v, dict) and not intact_type: + # Box objects must be created in case they are already + # in the `converted` box_config set + v = self.__class__(v, **self.__box_config()) + if k in self and isinstance(self[k], dict): + if isinstance(self[k], Box): + self[k].merge_update(v) + else: + self[k].update(v) + return + if isinstance(v, list) and not intact_type: + v = box.BoxList(v, **self.__box_config()) + self.__setitem__(k, v) + + if __m: + if hasattr(__m, 'keys'): + for key in __m: + convert_and_set(key, __m[key]) + else: + for key, value in __m: + convert_and_set(key, value) + for key in kwargs: + convert_and_set(key, kwargs[key]) + + def setdefault(self, item, default=None): + if item in self: + return self[item] + + if isinstance(default, dict): + default = self.__class__(default, **self.__box_config()) + if isinstance(default, list): + default = box.BoxList(default, box_class=self.__class__, **self.__box_config()) + self[item] = default + return default + + def _safe_attr(self, attr): + """Convert a key into something that is accessible as an attribute""" + allowed = string.ascii_letters + string.digits + '_' + + if isinstance(attr, tuple): + attr = "_".join([str(x) for x in attr]) + + attr = attr.decode('utf-8', 'ignore') if isinstance(attr, bytes) else str(attr) + if self.__box_config()['camel_killer_box']: + attr = _camel_killer(attr) + + out = [] + last_safe = 0 + for i, character in enumerate(attr): + if character in allowed: + last_safe = i + out.append(character) + elif not out: + continue + else: + if last_safe == i - 1: + out.append('_') + + out = "".join(out)[:last_safe + 1] + + try: + int(out[0]) + except (ValueError, IndexError): + pass + else: + out = f'{self.__box_config()["box_safe_prefix"]}{out}' + + if out in kwlist: + out = f'{self.__box_config()["box_safe_prefix"]}{out}' + + return out + + def _conversion_checks(self, item): + """ + Internal use for checking if a duplicate safe attribute already exists + + :param item: Item to see if a dup exists + :param keys: Keys to check against + """ + safe_item = self._safe_attr(item) + + if safe_item in self._box_config['__safe_keys']: + dups = [f'{item}({safe_item})', f'{self._box_config["__safe_keys"][safe_item]}({safe_item})'] + if self._box_config['box_duplicates'].startswith('warn'): + warnings.warn(f'Duplicate conversion attributes exist: {dups}', BoxWarning) + else: + raise BoxError(f'Duplicate conversion attributes exist: {dups}') + + def to_json(self, filename: Union[str, Path] = None, encoding: str = 'utf-8', errors: str = 'strict', + **json_kwargs): + """ + Transform the Box object into a JSON string. + + :param filename: If provided will save to file + :param encoding: File encoding + :param errors: How to handle encoding errors + :param json_kwargs: additional arguments to pass to json.dump(s) + :return: string of JSON (if no filename provided) + """ + return _to_json(self.to_dict(), filename=filename, encoding=encoding, errors=errors, **json_kwargs) + + @classmethod + def from_json(cls, json_string: str = None, filename: Union[str, Path] = None, encoding: str = 'utf-8', + errors: str = 'strict', **kwargs): + """ + Transform a json object string into a Box object. If the incoming + json is a list, you must use BoxList.from_json. + + :param json_string: string to pass to `json.loads` + :param filename: filename to open and pass to `json.load` + :param encoding: File encoding + :param errors: How to handle encoding errors + :param kwargs: parameters to pass to `Box()` or `json.loads` + :return: Box object from json data + """ + box_args = {} + for arg in kwargs.copy(): + if arg in BOX_PARAMETERS: + box_args[arg] = kwargs.pop(arg) + + data = _from_json(json_string, filename=filename, encoding=encoding, errors=errors, **kwargs) + + if not isinstance(data, dict): + raise BoxError(f'json data not returned as a dictionary, but rather a {type(data).__name__}') + return cls(data, **box_args) + + def to_yaml(self, filename: Union[str, Path] = None, default_flow_style: bool = False, encoding: str = 'utf-8', + errors: str = 'strict', **yaml_kwargs): + """ + Transform the Box object into a YAML string. + + :param filename: If provided will save to file + :param default_flow_style: False will recursively dump dicts + :param encoding: File encoding + :param errors: How to handle encoding errors + :param yaml_kwargs: additional arguments to pass to yaml.dump + :return: string of YAML (if no filename provided) + """ + return _to_yaml(self.to_dict(), filename=filename, default_flow_style=default_flow_style, + encoding=encoding, errors=errors, **yaml_kwargs) + + @classmethod + def from_yaml(cls, yaml_string: str = None, filename: Union[str, Path] = None, encoding: str = 'utf-8', + errors: str = 'strict', **kwargs): + """ + Transform a yaml object string into a Box object. By default will use SafeLoader. + + :param yaml_string: string to pass to `yaml.load` + :param filename: filename to open and pass to `yaml.load` + :param encoding: File encoding + :param errors: How to handle encoding errors + :param kwargs: parameters to pass to `Box()` or `yaml.load` + :return: Box object from yaml data + """ + box_args = {} + for arg in kwargs.copy(): + if arg in BOX_PARAMETERS: + box_args[arg] = kwargs.pop(arg) + + data = _from_yaml(yaml_string=yaml_string, filename=filename, encoding=encoding, errors=errors, **kwargs) + if not isinstance(data, dict): + raise BoxError(f'yaml data not returned as a dictionary but rather a {type(data).__name__}') + return cls(data, **box_args) + + def to_toml(self, filename: Union[str, Path] = None, encoding: str = 'utf-8', errors: str = 'strict'): + """ + Transform the Box object into a toml string. + + :param filename: File to write toml object too + :param encoding: File encoding + :param errors: How to handle encoding errors + :return: string of TOML (if no filename provided) + """ + return _to_toml(self.to_dict(), filename=filename, encoding=encoding, errors=errors) + + @classmethod + def from_toml(cls, toml_string: str = None, filename: Union[str, Path] = None, + encoding: str = 'utf-8', errors: str = 'strict', **kwargs): + """ + Transforms a toml string or file into a Box object + + :param toml_string: string to pass to `toml.load` + :param filename: filename to open and pass to `toml.load` + :param encoding: File encoding + :param errors: How to handle encoding errors + :param kwargs: parameters to pass to `Box()` + :return: + """ + box_args = {} + for arg in kwargs.copy(): + if arg in BOX_PARAMETERS: + box_args[arg] = kwargs.pop(arg) + + data = _from_toml(toml_string=toml_string, filename=filename, encoding=encoding, errors=errors) + return cls(data, **box_args) diff --git a/dynaconf/vendor/box/box_list.py b/dynaconf/vendor/box/box_list.py index 9c03a67a5..8687c401c 100644 --- a/dynaconf/vendor/box/box_list.py +++ b/dynaconf/vendor/box/box_list.py @@ -1,123 +1,276 @@ -_H='toml' -_G='box_dots' -_F='BoxList is frozen' -_E='frozen_box' -_D=False -_C='strict' -_B='utf-8' -_A=None -import copy,re -from typing import Iterable,Optional +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# +# Copyright (c) 2017-2020 - Chris Griffith - MIT License +import copy +import re +from typing import Iterable, Optional + + from dynaconf.vendor import box -from .converters import _to_yaml,_from_yaml,_to_json,_from_json,_to_toml,_from_toml,_to_csv,_from_csv,BOX_PARAMETERS -from .exceptions import BoxError,BoxTypeError,BoxKeyError -_list_pos_re=re.compile('\\[(\\d+)\\]') -DYNABOX_CLASS=_A +from .converters import (_to_yaml, _from_yaml, _to_json, _from_json, + _to_toml, _from_toml, _to_csv, _from_csv, BOX_PARAMETERS) +from .exceptions import BoxError, BoxTypeError, BoxKeyError + +_list_pos_re = re.compile(r'\[(\d+)\]') + + +DYNABOX_CLASS = None # a cache constant to avoid multiple imports + + def get_dynabox_class_avoiding_circular_import(): - global DYNABOX_CLASS - if DYNABOX_CLASS is _A:from dynaconf.utils.boxing import DynaBox as A;DYNABOX_CLASS=A - return DYNABOX_CLASS + """ + See dynaconf issue #462 + """ + global DYNABOX_CLASS + if DYNABOX_CLASS is None: + from dynaconf.utils.boxing import DynaBox + DYNABOX_CLASS = DynaBox + return DYNABOX_CLASS + + class BoxList(list): - def __init__(A,iterable=_A,box_class=_A,**C): - B=iterable;A.box_class=box_class or get_dynabox_class_avoiding_circular_import();A.box_options=C;A.box_org_ref=A.box_org_ref=id(B)if B else 0 - if B: - for D in B:A.append(D) - if C.get(_E): - def E(*A,**B):raise BoxError(_F) - for F in ['append','extend','insert','pop','remove','reverse','sort']:A.__setattr__(F,E) - def __getitem__(B,item): - A=item - if B.box_options.get(_G)and isinstance(A,str)and A.startswith('['): - C=_list_pos_re.search(A);D=super(BoxList,B).__getitem__(int(C.groups()[0])) - if len(C.group())==len(A):return D - return D.__getitem__(A[len(C.group()):].lstrip('.')) - return super(BoxList,B).__getitem__(A) - def __delitem__(A,key): - if A.box_options.get(_E):raise BoxError(_F) - super(BoxList,A).__delitem__(key) - def __setitem__(B,key,value): - C=value;A=key - if B.box_options.get(_E):raise BoxError(_F) - if B.box_options.get(_G)and isinstance(A,str)and A.startswith('['): - D=_list_pos_re.search(A);E=int(D.groups()[0]) - if len(D.group())==len(A):return super(BoxList,B).__setitem__(E,C) - return super(BoxList,B).__getitem__(E).__setitem__(A[len(D.group()):].lstrip('.'),C) - super(BoxList,B).__setitem__(A,C) - def _is_intact_type(A,obj): - C='box_intact_types' - try: - if A.box_options.get(C)and isinstance(obj,A.box_options[C]):return True - except AttributeError as B: - if'box_options'in A.__dict__:raise BoxKeyError(B) - return _D - def append(A,p_object): - B=p_object - if isinstance(B,dict)and not A._is_intact_type(B): - try:B=A.box_class(B,**A.box_options) - except AttributeError as C: - if'box_class'in A.__dict__:raise BoxKeyError(C) - elif isinstance(B,list)and not A._is_intact_type(B): - try:B=A if id(B)==A.box_org_ref else BoxList(B,**A.box_options) - except AttributeError as C: - if'box_org_ref'in A.__dict__:raise BoxKeyError(C) - super(BoxList,A).append(B) - def extend(A,iterable): - for B in iterable:A.append(B) - def insert(B,index,p_object): - A=p_object - if isinstance(A,dict)and not B._is_intact_type(A):A=B.box_class(A,**B.box_options) - elif isinstance(A,list)and not B._is_intact_type(A):A=B if id(A)==B.box_org_ref else BoxList(A) - super(BoxList,B).insert(index,A) - def __repr__(A):return f"" - def __str__(A):return str(A.to_list()) - def __copy__(A):return BoxList((B for B in A),A.box_class,**A.box_options) - def __deepcopy__(B,memo=_A): - A=memo;C=B.__class__();A=A or{};A[id(B)]=C - for D in B:C.append(copy.deepcopy(D,memo=A)) - return C - def __hash__(A): - if A.box_options.get(_E):B=98765;B^=hash(tuple(A));return B - raise BoxTypeError("unhashable type: 'BoxList'") - def to_list(C): - A=[] - for B in C: - if B is C:A.append(A) - elif isinstance(B,box.Box):A.append(B.to_dict()) - elif isinstance(B,BoxList):A.append(B.to_list()) - else:A.append(B) - return A - def to_json(D,filename=_A,encoding=_B,errors=_C,multiline=_D,**E): - C=errors;B=encoding;A=filename - if A and multiline: - F=[_to_json(A,filename=_D,encoding=B,errors=C,**E)for A in D] - with open(A,'w',encoding=B,errors=C)as G:G.write('\n'.join(F)) - else:return _to_json(D.to_list(),filename=A,encoding=B,errors=C,**E) - @classmethod - def from_json(E,json_string=_A,filename=_A,encoding=_B,errors=_C,multiline=_D,**A): - D={} - for B in list(A.keys()): - if B in BOX_PARAMETERS:D[B]=A.pop(B) - C=_from_json(json_string,filename=filename,encoding=encoding,errors=errors,multiline=multiline,**A) - if not isinstance(C,list):raise BoxError(f"json data not returned as a list, but rather a {type(C).__name__}") - return E(C,**D) - def to_yaml(A,filename=_A,default_flow_style=_D,encoding=_B,errors=_C,**B):return _to_yaml(A.to_list(),filename=filename,default_flow_style=default_flow_style,encoding=encoding,errors=errors,**B) - @classmethod - def from_yaml(E,yaml_string=_A,filename=_A,encoding=_B,errors=_C,**A): - D={} - for B in list(A.keys()): - if B in BOX_PARAMETERS:D[B]=A.pop(B) - C=_from_yaml(yaml_string=yaml_string,filename=filename,encoding=encoding,errors=errors,**A) - if not isinstance(C,list):raise BoxError(f"yaml data not returned as a list but rather a {type(C).__name__}") - return E(C,**D) - def to_toml(A,filename=_A,key_name=_H,encoding=_B,errors=_C):return _to_toml({key_name:A.to_list()},filename=filename,encoding=encoding,errors=errors) - @classmethod - def from_toml(F,toml_string=_A,filename=_A,key_name=_H,encoding=_B,errors=_C,**C): - A=key_name;D={} - for B in list(C.keys()): - if B in BOX_PARAMETERS:D[B]=C.pop(B) - E=_from_toml(toml_string=toml_string,filename=filename,encoding=encoding,errors=errors) - if A not in E:raise BoxError(f"{A} was not found.") - return F(E[A],**D) - def to_csv(A,filename,encoding=_B,errors=_C):_to_csv(A,filename=filename,encoding=encoding,errors=errors) - @classmethod - def from_csv(A,filename,encoding=_B,errors=_C):return A(_from_csv(filename=filename,encoding=encoding,errors=errors)) \ No newline at end of file + """ + Drop in replacement of list, that converts added objects to Box or BoxList + objects as necessary. + """ + + def __init__(self, iterable: Iterable = None, box_class : Optional[box.Box] = None, **box_options): + self.box_class = box_class or get_dynabox_class_avoiding_circular_import() + self.box_options = box_options + self.box_org_ref = self.box_org_ref = id(iterable) if iterable else 0 + if iterable: + for x in iterable: + self.append(x) + if box_options.get('frozen_box'): + def frozen(*args, **kwargs): + raise BoxError('BoxList is frozen') + + for method in ['append', 'extend', 'insert', 'pop', 'remove', 'reverse', 'sort']: + self.__setattr__(method, frozen) + + def __getitem__(self, item): + if self.box_options.get('box_dots') and isinstance(item, str) and item.startswith('['): + list_pos = _list_pos_re.search(item) + value = super(BoxList, self).__getitem__(int(list_pos.groups()[0])) + if len(list_pos.group()) == len(item): + return value + return value.__getitem__(item[len(list_pos.group()):].lstrip('.')) + return super(BoxList, self).__getitem__(item) + + def __delitem__(self, key): + if self.box_options.get('frozen_box'): + raise BoxError('BoxList is frozen') + super(BoxList, self).__delitem__(key) + + def __setitem__(self, key, value): + if self.box_options.get('frozen_box'): + raise BoxError('BoxList is frozen') + if self.box_options.get('box_dots') and isinstance(key, str) and key.startswith('['): + list_pos = _list_pos_re.search(key) + pos = int(list_pos.groups()[0]) + if len(list_pos.group()) == len(key): + return super(BoxList, self).__setitem__(pos, value) + return super(BoxList, self).__getitem__(pos).__setitem__(key[len(list_pos.group()):].lstrip('.'), value) + super(BoxList, self).__setitem__(key, value) + + def _is_intact_type(self, obj): + try: + if self.box_options.get('box_intact_types') and isinstance(obj, self.box_options['box_intact_types']): + return True + except AttributeError as err: + if 'box_options' in self.__dict__: + raise BoxKeyError(err) + return False + + def append(self, p_object): + if isinstance(p_object, dict) and not self._is_intact_type(p_object): + try: + p_object = self.box_class(p_object, **self.box_options) + except AttributeError as err: + if 'box_class' in self.__dict__: + raise BoxKeyError(err) + elif isinstance(p_object, list) and not self._is_intact_type(p_object): + try: + p_object = (self if id(p_object) == self.box_org_ref else BoxList(p_object, **self.box_options)) + except AttributeError as err: + if 'box_org_ref' in self.__dict__: + raise BoxKeyError(err) + super(BoxList, self).append(p_object) + + def extend(self, iterable): + for item in iterable: + self.append(item) + + def insert(self, index, p_object): + if isinstance(p_object, dict) and not self._is_intact_type(p_object): + p_object = self.box_class(p_object, **self.box_options) + elif isinstance(p_object, list) and not self._is_intact_type(p_object): + p_object = (self if id(p_object) == self.box_org_ref else BoxList(p_object)) + super(BoxList, self).insert(index, p_object) + + def __repr__(self): + return f'' + + def __str__(self): + return str(self.to_list()) + + def __copy__(self): + return BoxList((x for x in self), self.box_class, **self.box_options) + + def __deepcopy__(self, memo=None): + out = self.__class__() + memo = memo or {} + memo[id(self)] = out + for k in self: + out.append(copy.deepcopy(k, memo=memo)) + return out + + def __hash__(self): + if self.box_options.get('frozen_box'): + hashing = 98765 + hashing ^= hash(tuple(self)) + return hashing + raise BoxTypeError("unhashable type: 'BoxList'") + + def to_list(self): + new_list = [] + for x in self: + if x is self: + new_list.append(new_list) + elif isinstance(x, box.Box): + new_list.append(x.to_dict()) + elif isinstance(x, BoxList): + new_list.append(x.to_list()) + else: + new_list.append(x) + return new_list + + def to_json(self, filename: str = None, encoding: str = 'utf-8', errors: str = 'strict', + multiline: bool = False, **json_kwargs): + """ + Transform the BoxList object into a JSON string. + + :param filename: If provided will save to file + :param encoding: File encoding + :param errors: How to handle encoding errors + :param multiline: Put each item in list onto it's own line + :param json_kwargs: additional arguments to pass to json.dump(s) + :return: string of JSON or return of `json.dump` + """ + if filename and multiline: + lines = [_to_json(item, filename=False, encoding=encoding, errors=errors, **json_kwargs) for item in self] + with open(filename, 'w', encoding=encoding, errors=errors) as f: + f.write("\n".join(lines)) + else: + return _to_json(self.to_list(), filename=filename, encoding=encoding, errors=errors, **json_kwargs) + + @classmethod + def from_json(cls, json_string: str = None, filename: str = None, encoding: str = 'utf-8', errors: str = 'strict', + multiline: bool = False, **kwargs): + """ + Transform a json object string into a BoxList object. If the incoming + json is a dict, you must use Box.from_json. + + :param json_string: string to pass to `json.loads` + :param filename: filename to open and pass to `json.load` + :param encoding: File encoding + :param errors: How to handle encoding errors + :param multiline: One object per line + :param kwargs: parameters to pass to `Box()` or `json.loads` + :return: BoxList object from json data + """ + bx_args = {} + for arg in list(kwargs.keys()): + if arg in BOX_PARAMETERS: + bx_args[arg] = kwargs.pop(arg) + + data = _from_json(json_string, filename=filename, encoding=encoding, + errors=errors, multiline=multiline, **kwargs) + + if not isinstance(data, list): + raise BoxError(f'json data not returned as a list, but rather a {type(data).__name__}') + return cls(data, **bx_args) + + def to_yaml(self, filename: str = None, default_flow_style: bool = False, + encoding: str = 'utf-8', errors: str = 'strict', **yaml_kwargs): + """ + Transform the BoxList object into a YAML string. + + :param filename: If provided will save to file + :param default_flow_style: False will recursively dump dicts + :param encoding: File encoding + :param errors: How to handle encoding errors + :param yaml_kwargs: additional arguments to pass to yaml.dump + :return: string of YAML or return of `yaml.dump` + """ + return _to_yaml(self.to_list(), filename=filename, default_flow_style=default_flow_style, + encoding=encoding, errors=errors, **yaml_kwargs) + + @classmethod + def from_yaml(cls, yaml_string: str = None, filename: str = None, + encoding: str = 'utf-8', errors: str = 'strict', **kwargs): + """ + Transform a yaml object string into a BoxList object. + + :param yaml_string: string to pass to `yaml.load` + :param filename: filename to open and pass to `yaml.load` + :param encoding: File encoding + :param errors: How to handle encoding errors + :param kwargs: parameters to pass to `BoxList()` or `yaml.load` + :return: BoxList object from yaml data + """ + bx_args = {} + for arg in list(kwargs.keys()): + if arg in BOX_PARAMETERS: + bx_args[arg] = kwargs.pop(arg) + + data = _from_yaml(yaml_string=yaml_string, filename=filename, encoding=encoding, errors=errors, **kwargs) + if not isinstance(data, list): + raise BoxError(f'yaml data not returned as a list but rather a {type(data).__name__}') + return cls(data, **bx_args) + + def to_toml(self, filename: str = None, key_name: str = 'toml', encoding: str = 'utf-8', errors: str = 'strict'): + """ + Transform the BoxList object into a toml string. + + :param filename: File to write toml object too + :param key_name: Specify the name of the key to store the string under + (cannot directly convert to toml) + :param encoding: File encoding + :param errors: How to handle encoding errors + :return: string of TOML (if no filename provided) + """ + return _to_toml({key_name: self.to_list()}, filename=filename, encoding=encoding, errors=errors) + + @classmethod + def from_toml(cls, toml_string: str = None, filename: str = None, key_name: str = 'toml', + encoding: str = 'utf-8', errors: str = 'strict', **kwargs): + """ + Transforms a toml string or file into a BoxList object + + :param toml_string: string to pass to `toml.load` + :param filename: filename to open and pass to `toml.load` + :param key_name: Specify the name of the key to pull the list from + (cannot directly convert from toml) + :param encoding: File encoding + :param errors: How to handle encoding errors + :param kwargs: parameters to pass to `Box()` + :return: + """ + bx_args = {} + for arg in list(kwargs.keys()): + if arg in BOX_PARAMETERS: + bx_args[arg] = kwargs.pop(arg) + + data = _from_toml(toml_string=toml_string, filename=filename, encoding=encoding, errors=errors) + if key_name not in data: + raise BoxError(f'{key_name} was not found.') + return cls(data[key_name], **bx_args) + + def to_csv(self, filename, encoding: str = 'utf-8', errors: str = 'strict'): + _to_csv(self, filename=filename, encoding=encoding, errors=errors) + + @classmethod + def from_csv(cls, filename, encoding: str = 'utf-8', errors: str = 'strict'): + return cls(_from_csv(filename=filename, encoding=encoding, errors=errors)) diff --git a/dynaconf/vendor/box/config_box.py b/dynaconf/vendor/box/config_box.py index 11949633b..875699574 100644 --- a/dynaconf/vendor/box/config_box.py +++ b/dynaconf/vendor/box/config_box.py @@ -1,54 +1,133 @@ -_H='getint' -_G='getfloat' -_F='getboolean' -_E='list' -_D='float' -_C='int' -_B='bool' -_A=None +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + from dynaconf.vendor.box.box import Box + + class ConfigBox(Box): - _protected_keys=dir(Box)+[_B,_C,_D,_E,_F,_G,_H] - def __getattr__(A,item): - try:return super().__getattr__(item) - except AttributeError:return super().__getattr__(item.lower()) - def __dir__(A):return super().__dir__()+[_B,_C,_D,_E,_F,_G,_H] - def bool(C,item,default=_A): - E=False;B=default;A=item - try:A=C.__getattr__(A) - except AttributeError as D: - if B is not _A:return B - raise D - if isinstance(A,(bool,int)):return bool(A) - if isinstance(A,str)and A.lower()in('n','no','false','f','0'):return E - return True if A else E - def int(C,item,default=_A): - B=default;A=item - try:A=C.__getattr__(A) - except AttributeError as D: - if B is not _A:return B - raise D - return int(A) - def float(C,item,default=_A): - B=default;A=item - try:A=C.__getattr__(A) - except AttributeError as D: - if B is not _A:return B - raise D - return float(A) - def list(E,item,default=_A,spliter=',',strip=True,mod=_A): - C=strip;B=default;A=item - try:A=E.__getattr__(A) - except AttributeError as F: - if B is not _A:return B - raise F - if C:A=A.lstrip('[').rstrip(']') - D=[B.strip()if C else B for B in A.split(spliter)] - if mod:return list(map(mod,D)) - return D - def getboolean(A,item,default=_A):return A.bool(item,default) - def getint(A,item,default=_A):return A.int(item,default) - def getfloat(A,item,default=_A):return A.float(item,default) - def __repr__(A):return ''.format(str(A.to_dict())) - def copy(A):return ConfigBox(super().copy()) - def __copy__(A):return ConfigBox(super().copy()) \ No newline at end of file + """ + Modified box object to add object transforms. + + Allows for build in transforms like: + + cns = ConfigBox(my_bool='yes', my_int='5', my_list='5,4,3,3,2') + + cns.bool('my_bool') # True + cns.int('my_int') # 5 + cns.list('my_list', mod=lambda x: int(x)) # [5, 4, 3, 3, 2] + """ + + _protected_keys = dir(Box) + ['bool', 'int', 'float', 'list', 'getboolean', 'getfloat', 'getint'] + + def __getattr__(self, item): + """ + Config file keys are stored in lower case, be a little more + loosey goosey + """ + try: + return super().__getattr__(item) + except AttributeError: + return super().__getattr__(item.lower()) + + def __dir__(self): + return super().__dir__() + ['bool', 'int', 'float', 'list', 'getboolean', 'getfloat', 'getint'] + + def bool(self, item, default=None): + """ + Return value of key as a boolean + + :param item: key of value to transform + :param default: value to return if item does not exist + :return: approximated bool of value + """ + try: + item = self.__getattr__(item) + except AttributeError as err: + if default is not None: + return default + raise err + + if isinstance(item, (bool, int)): + return bool(item) + + if (isinstance(item, str) + and item.lower() in ('n', 'no', 'false', 'f', '0')): + return False + + return True if item else False + + def int(self, item, default=None): + """ + Return value of key as an int + + :param item: key of value to transform + :param default: value to return if item does not exist + :return: int of value + """ + try: + item = self.__getattr__(item) + except AttributeError as err: + if default is not None: + return default + raise err + return int(item) + + def float(self, item, default=None): + """ + Return value of key as a float + + :param item: key of value to transform + :param default: value to return if item does not exist + :return: float of value + """ + try: + item = self.__getattr__(item) + except AttributeError as err: + if default is not None: + return default + raise err + return float(item) + + def list(self, item, default=None, spliter=",", strip=True, mod=None): + """ + Return value of key as a list + + :param item: key of value to transform + :param mod: function to map against list + :param default: value to return if item does not exist + :param spliter: character to split str on + :param strip: clean the list with the `strip` + :return: list of items + """ + try: + item = self.__getattr__(item) + except AttributeError as err: + if default is not None: + return default + raise err + if strip: + item = item.lstrip('[').rstrip(']') + out = [x.strip() if strip else x for x in item.split(spliter)] + if mod: + return list(map(mod, out)) + return out + + # loose configparser compatibility + + def getboolean(self, item, default=None): + return self.bool(item, default) + + def getint(self, item, default=None): + return self.int(item, default) + + def getfloat(self, item, default=None): + return self.float(item, default) + + def __repr__(self): + return ''.format(str(self.to_dict())) + + def copy(self): + return ConfigBox(super().copy()) + + def __copy__(self): + return ConfigBox(super().copy()) diff --git a/dynaconf/vendor/box/converters.py b/dynaconf/vendor/box/converters.py index 93cdcfbf0..08694fe1e 100644 --- a/dynaconf/vendor/box/converters.py +++ b/dynaconf/vendor/box/converters.py @@ -1,78 +1,129 @@ -_G='r' -_F='w' -_E=False -_D=True -_C='strict' -_B='utf-8' -_A=None -import csv,json,sys,warnings +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +# Abstract converter functions for use in any Box class + +import csv +import json +import sys +import warnings from pathlib import Path + import dynaconf.vendor.ruamel.yaml as yaml -from dynaconf.vendor.box.exceptions import BoxError,BoxWarning -from dynaconf.vendor import toml -BOX_PARAMETERS='default_box','default_box_attr','conversion_box','frozen_box','camel_killer_box','box_safe_prefix','box_duplicates','ordered_box','default_box_none_transform','box_dots','modify_tuples_box','box_intact_types','box_recast' -def _exists(filename,create=_E): - A=filename;B=Path(A) - if create: - try:B.touch(exist_ok=_D) - except OSError as C:raise BoxError(f"Could not create file {A} - {C}") - else:return - if not B.exists():raise BoxError(f'File "{A}" does not exist') - if not B.is_file():raise BoxError(f"{A} is not a file") -def _to_json(obj,filename=_A,encoding=_B,errors=_C,**C): - A=filename;B=json.dumps(obj,ensure_ascii=_E,**C) - if A: - _exists(A,create=_D) - with open(A,_F,encoding=encoding,errors=errors)as D:D.write(B if sys.version_info>=(3,0)else B.decode(_B)) - else:return B -def _from_json(json_string=_A,filename=_A,encoding=_B,errors=_C,multiline=_E,**B): - D=json_string;A=filename - if A: - _exists(A) - with open(A,_G,encoding=encoding,errors=errors)as E: - if multiline:C=[json.loads(A.strip(),**B)for A in E if A.strip()and not A.strip().startswith('#')] - else:C=json.load(E,**B) - elif D:C=json.loads(D,**B) - else:raise BoxError('from_json requires a string or filename') - return C -def _to_yaml(obj,filename=_A,default_flow_style=_E,encoding=_B,errors=_C,**C): - B=default_flow_style;A=filename - if A: - _exists(A,create=_D) - with open(A,_F,encoding=encoding,errors=errors)as D:yaml.dump(obj,stream=D,default_flow_style=B,**C) - else:return yaml.dump(obj,default_flow_style=B,**C) -def _from_yaml(yaml_string=_A,filename=_A,encoding=_B,errors=_C,**A): - F='Loader';C=yaml_string;B=filename - if F not in A:A[F]=yaml.SafeLoader - if B: - _exists(B) - with open(B,_G,encoding=encoding,errors=errors)as E:D=yaml.load(E,**A) - elif C:D=yaml.load(C,**A) - else:raise BoxError('from_yaml requires a string or filename') - return D -def _to_toml(obj,filename=_A,encoding=_B,errors=_C): - A=filename - if A: - _exists(A,create=_D) - with open(A,_F,encoding=encoding,errors=errors)as B:toml.dump(obj,B) - else:return toml.dumps(obj) -def _from_toml(toml_string=_A,filename=_A,encoding=_B,errors=_C): - B=toml_string;A=filename - if A: - _exists(A) - with open(A,_G,encoding=encoding,errors=errors)as D:C=toml.load(D) - elif B:C=toml.loads(B) - else:raise BoxError('from_toml requires a string or filename') - return C -def _to_csv(box_list,filename,encoding=_B,errors=_C): - B=filename;A=box_list;C=list(A[0].keys()) - for E in A: - if list(E.keys())!=C:raise BoxError('BoxList must contain the same dictionary structure for every item to convert to csv') - if B: - _exists(B,create=_D) - with open(B,_F,encoding=encoding,errors=errors,newline='')as F: - D=csv.DictWriter(F,fieldnames=C);D.writeheader() - for G in A:D.writerow(G) -def _from_csv(filename,encoding=_B,errors=_C): - A=filename;_exists(A) - with open(A,_G,encoding=encoding,errors=errors,newline='')as B:C=csv.DictReader(B);return[A for A in C] \ No newline at end of file +from dynaconf.vendor.box.exceptions import BoxError, BoxWarning +from dynaconf.vendor import tomllib as toml + + +BOX_PARAMETERS = ('default_box', 'default_box_attr', 'conversion_box', + 'frozen_box', 'camel_killer_box', + 'box_safe_prefix', 'box_duplicates', 'ordered_box', + 'default_box_none_transform', 'box_dots', 'modify_tuples_box', + 'box_intact_types', 'box_recast') + + +def _exists(filename, create=False): + path = Path(filename) + if create: + try: + path.touch(exist_ok=True) + except OSError as err: + raise BoxError(f'Could not create file {filename} - {err}') + else: + return + if not path.exists(): + raise BoxError(f'File "{filename}" does not exist') + if not path.is_file(): + raise BoxError(f'{filename} is not a file') + + +def _to_json(obj, filename=None, encoding="utf-8", errors="strict", **json_kwargs): + json_dump = json.dumps(obj, ensure_ascii=False, **json_kwargs) + if filename: + _exists(filename, create=True) + with open(filename, 'w', encoding=encoding, errors=errors) as f: + f.write(json_dump if sys.version_info >= (3, 0) else json_dump.decode("utf-8")) + else: + return json_dump + + +def _from_json(json_string=None, filename=None, encoding="utf-8", errors="strict", multiline=False, **kwargs): + if filename: + _exists(filename) + with open(filename, 'r', encoding=encoding, errors=errors) as f: + if multiline: + data = [json.loads(line.strip(), **kwargs) for line in f + if line.strip() and not line.strip().startswith("#")] + else: + data = json.load(f, **kwargs) + elif json_string: + data = json.loads(json_string, **kwargs) + else: + raise BoxError('from_json requires a string or filename') + return data + + +def _to_yaml(obj, filename=None, default_flow_style=False, encoding="utf-8", errors="strict", **yaml_kwargs): + if filename: + _exists(filename, create=True) + with open(filename, 'w', + encoding=encoding, errors=errors) as f: + yaml.dump(obj, stream=f, default_flow_style=default_flow_style, **yaml_kwargs) + else: + return yaml.dump(obj, default_flow_style=default_flow_style, **yaml_kwargs) + + +def _from_yaml(yaml_string=None, filename=None, encoding="utf-8", errors="strict", **kwargs): + if 'Loader' not in kwargs: + kwargs['Loader'] = yaml.SafeLoader + if filename: + _exists(filename) + with open(filename, 'r', encoding=encoding, errors=errors) as f: + data = yaml.load(f, **kwargs) + elif yaml_string: + data = yaml.load(yaml_string, **kwargs) + else: + raise BoxError('from_yaml requires a string or filename') + return data + + +def _to_toml(obj, filename=None, encoding="utf-8", errors="strict"): + if filename: + _exists(filename, create=True) + with open(filename, 'w', encoding=encoding, errors=errors) as f: + toml.dump(obj, f) + else: + return toml.dumps(obj) + + +def _from_toml(toml_string=None, filename=None, encoding="utf-8", errors="strict"): + if filename: + _exists(filename) + with open(filename, 'r', encoding=encoding, errors=errors) as f: + data = toml.load(f) + elif toml_string: + data = toml.loads(toml_string) + else: + raise BoxError('from_toml requires a string or filename') + return data + + +def _to_csv(box_list, filename, encoding="utf-8", errors="strict"): + csv_column_names = list(box_list[0].keys()) + for row in box_list: + if list(row.keys()) != csv_column_names: + raise BoxError('BoxList must contain the same dictionary structure for every item to convert to csv') + + if filename: + _exists(filename, create=True) + with open(filename, 'w', encoding=encoding, errors=errors, newline='') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=csv_column_names) + writer.writeheader() + for data in box_list: + writer.writerow(data) + + +def _from_csv(filename, encoding="utf-8", errors="strict"): + _exists(filename) + with open(filename, 'r', encoding=encoding, errors=errors, newline='') as f: + reader = csv.DictReader(f) + return [row for row in reader] diff --git a/dynaconf/vendor/box/exceptions.py b/dynaconf/vendor/box/exceptions.py index 66199e7b8..57aeaf227 100644 --- a/dynaconf/vendor/box/exceptions.py +++ b/dynaconf/vendor/box/exceptions.py @@ -1,5 +1,22 @@ -class BoxError(Exception):0 -class BoxKeyError(BoxError,KeyError,AttributeError):0 -class BoxTypeError(BoxError,TypeError):0 -class BoxValueError(BoxError,ValueError):0 -class BoxWarning(UserWarning):0 \ No newline at end of file +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + + +class BoxError(Exception): + """Non standard dictionary exceptions""" + + +class BoxKeyError(BoxError, KeyError, AttributeError): + """Key does not exist""" + + +class BoxTypeError(BoxError, TypeError): + """Cannot handle that instance's type""" + + +class BoxValueError(BoxError, ValueError): + """Issue doing something with that value""" + + +class BoxWarning(UserWarning): + """Here be dragons""" diff --git a/dynaconf/vendor/box/from_file.py b/dynaconf/vendor/box/from_file.py index daa11378e..a82ac9659 100644 --- a/dynaconf/vendor/box/from_file.py +++ b/dynaconf/vendor/box/from_file.py @@ -1,34 +1,73 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- from json import JSONDecodeError from pathlib import Path from typing import Union -from dynaconf.vendor.toml import TomlDecodeError +from dynaconf.vendor.tomllib import TOMLDecodeError from dynaconf.vendor.ruamel.yaml import YAMLError + + from .exceptions import BoxError from .box import Box from .box_list import BoxList -__all__=['box_from_file'] + +__all__ = ['box_from_file'] + + def _to_json(data): - try:return Box.from_json(data) - except JSONDecodeError:raise BoxError('File is not JSON as expected') - except BoxError:return BoxList.from_json(data) + try: + return Box.from_json(data) + except JSONDecodeError: + raise BoxError('File is not JSON as expected') + except BoxError: + return BoxList.from_json(data) + + def _to_yaml(data): - try:return Box.from_yaml(data) - except YAMLError:raise BoxError('File is not YAML as expected') - except BoxError:return BoxList.from_yaml(data) + try: + return Box.from_yaml(data) + except YAMLError: + raise BoxError('File is not YAML as expected') + except BoxError: + return BoxList.from_yaml(data) + + def _to_toml(data): - try:return Box.from_toml(data) - except TomlDecodeError:raise BoxError('File is not TOML as expected') -def box_from_file(file,file_type=None,encoding='utf-8',errors='strict'): - C=file_type;A=file - if not isinstance(A,Path):A=Path(A) - if not A.exists():raise BoxError(f'file "{A}" does not exist') - B=A.read_text(encoding=encoding,errors=errors) - if C: - if C.lower()=='json':return _to_json(B) - if C.lower()=='yaml':return _to_yaml(B) - if C.lower()=='toml':return _to_toml(B) - raise BoxError(f'"{C}" is an unknown type, please use either toml, yaml or json') - if A.suffix in('.json','.jsn'):return _to_json(B) - if A.suffix in('.yaml','.yml'):return _to_yaml(B) - if A.suffix in('.tml','.toml'):return _to_toml(B) - raise BoxError(f"Could not determine file type based off extension, please provide file_type") \ No newline at end of file + try: + return Box.from_toml(data) + except TOMLDecodeError: + raise BoxError('File is not TOML as expected') + + +def box_from_file(file: Union[str, Path], file_type: str = None, + encoding: str = "utf-8", errors: str = "strict") -> Union[Box, BoxList]: + """ + Loads the provided file and tries to parse it into a Box or BoxList object as appropriate. + + :param file: Location of file + :param encoding: File encoding + :param errors: How to handle encoding errors + :param file_type: manually specify file type: json, toml or yaml + :return: Box or BoxList + """ + + if not isinstance(file, Path): + file = Path(file) + if not file.exists(): + raise BoxError(f'file "{file}" does not exist') + data = file.read_text(encoding=encoding, errors=errors) + if file_type: + if file_type.lower() == 'json': + return _to_json(data) + if file_type.lower() == 'yaml': + return _to_yaml(data) + if file_type.lower() == 'toml': + return _to_toml(data) + raise BoxError(f'"{file_type}" is an unknown type, please use either toml, yaml or json') + if file.suffix in ('.json', '.jsn'): + return _to_json(data) + if file.suffix in ('.yaml', '.yml'): + return _to_yaml(data) + if file.suffix in ('.tml', '.toml'): + return _to_toml(data) + raise BoxError(f'Could not determine file type based off extension, please provide file_type') diff --git a/dynaconf/vendor/box/shorthand_box.py b/dynaconf/vendor/box/shorthand_box.py index b6da19cda..746f7619a 100644 --- a/dynaconf/vendor/box/shorthand_box.py +++ b/dynaconf/vendor/box/shorthand_box.py @@ -1,14 +1,38 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + from dynaconf.vendor.box.box import Box + + class SBox(Box): - _protected_keys=dir({})+['to_dict','to_json','to_yaml','json','yaml','from_yaml','from_json','dict','toml','from_toml','to_toml'] - @property - def dict(self):return self.to_dict() - @property - def json(self):return self.to_json() - @property - def yaml(self):return self.to_yaml() - @property - def toml(self):return self.to_toml() - def __repr__(A):return ''.format(str(A.to_dict())) - def copy(A):return SBox(super(SBox,A).copy()) - def __copy__(A):return SBox(super(SBox,A).copy()) \ No newline at end of file + """ + ShorthandBox (SBox) allows for + property access of `dict` `json` and `yaml` + """ + _protected_keys = dir({}) + ['to_dict', 'to_json', 'to_yaml', 'json', 'yaml', 'from_yaml', 'from_json', + 'dict', 'toml', 'from_toml', 'to_toml'] + + @property + def dict(self): + return self.to_dict() + + @property + def json(self): + return self.to_json() + + @property + def yaml(self): + return self.to_yaml() + + @property + def toml(self): + return self.to_toml() + + def __repr__(self): + return ''.format(str(self.to_dict())) + + def copy(self): + return SBox(super(SBox, self).copy()) + + def __copy__(self): + return SBox(super(SBox, self).copy()) diff --git a/dynaconf/vendor/click/README.md b/dynaconf/vendor/click/README.md deleted file mode 100644 index 0f7bac33f..000000000 --- a/dynaconf/vendor/click/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## python-click - -Vendored dep taken from: https://github.com/pallets/click -Licensed under MIT: https://github.com/pallets/clickl/blob/master/LICENSE -Current version: 7.1.x diff --git a/dynaconf/vendor/click/__init__.py b/dynaconf/vendor/click/__init__.py index fc6520ad1..9cd0129bf 100644 --- a/dynaconf/vendor/click/__init__.py +++ b/dynaconf/vendor/click/__init__.py @@ -1,4 +1,18 @@ -from .core import Argument,BaseCommand,Command,CommandCollection,Context,Group,MultiCommand,Option,Parameter +""" +Click is a simple Python module inspired by the stdlib optparse to make +writing command line scripts fun. Unlike other modules, it's based +around a simple API that does not come with too much magic and is +composable. +""" +from .core import Argument +from .core import BaseCommand +from .core import Command +from .core import CommandCollection +from .core import Context +from .core import Group +from .core import MultiCommand +from .core import Option +from .core import Parameter from .decorators import argument from .decorators import command from .decorators import confirmation_option @@ -57,4 +71,5 @@ from .utils import get_os_args from .utils import get_text_stream from .utils import open_file -__version__='8.0.0.dev' \ No newline at end of file + +__version__ = "8.0.0.dev" diff --git a/dynaconf/vendor/click/_bashcomplete.py b/dynaconf/vendor/click/_bashcomplete.py index e27049d85..b9e4900e0 100644 --- a/dynaconf/vendor/click/_bashcomplete.py +++ b/dynaconf/vendor/click/_bashcomplete.py @@ -1,114 +1,371 @@ -_I='COMP_CWORD' -_H='COMP_WORDS' -_G='fish' -_F='zsh' -_E='bash' -_D='_' -_C=False -_B=None -_A=True -import copy,os,re +import copy +import os +import re from collections import abc + from .core import Argument from .core import MultiCommand from .core import Option from .parser import split_arg_string from .types import Choice from .utils import echo -WORDBREAK='=' -COMPLETION_SCRIPT_BASH='\n%(complete_func)s() {\n local IFS=$\'\n\'\n COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \\\n COMP_CWORD=$COMP_CWORD \\\n %(autocomplete_var)s=complete $1 ) )\n return 0\n}\n\n%(complete_func)setup() {\n local COMPLETION_OPTIONS=""\n local BASH_VERSION_ARR=(${BASH_VERSION//./ })\n # Only BASH version 4.4 and later have the nosort option.\n if [ ${BASH_VERSION_ARR[0]} -gt 4 ] || ([ ${BASH_VERSION_ARR[0]} -eq 4 ] && [ ${BASH_VERSION_ARR[1]} -ge 4 ]); then\n COMPLETION_OPTIONS="-o nosort"\n fi\n\n complete $COMPLETION_OPTIONS -F %(complete_func)s %(script_names)s\n}\n\n%(complete_func)setup\n' -COMPLETION_SCRIPT_ZSH='\n#compdef %(script_names)s\n\n%(complete_func)s() {\n local -a completions\n local -a completions_with_descriptions\n local -a response\n (( ! $+commands[%(script_names)s] )) && return 1\n\n response=("${(@f)$( env COMP_WORDS="${words[*]}" \\\n COMP_CWORD=$((CURRENT-1)) \\\n %(autocomplete_var)s="complete_zsh" \\\n %(script_names)s )}")\n\n for key descr in ${(kv)response}; do\n if [[ "$descr" == "_" ]]; then\n completions+=("$key")\n else\n completions_with_descriptions+=("$key":"$descr")\n fi\n done\n\n if [ -n "$completions_with_descriptions" ]; then\n _describe -V unsorted completions_with_descriptions -U\n fi\n\n if [ -n "$completions" ]; then\n compadd -U -V unsorted -a completions\n fi\n compstate[insert]="automenu"\n}\n\ncompdef %(complete_func)s %(script_names)s\n' -COMPLETION_SCRIPT_FISH='complete --no-files --command %(script_names)s --arguments "(env %(autocomplete_var)s=complete_fish COMP_WORDS=(commandline -cp) COMP_CWORD=(commandline -t) %(script_names)s)"' -_completion_scripts={_E:COMPLETION_SCRIPT_BASH,_F:COMPLETION_SCRIPT_ZSH,_G:COMPLETION_SCRIPT_FISH} -_invalid_ident_char_re=re.compile('[^a-zA-Z0-9_]') -def get_completion_script(prog_name,complete_var,shell):A=prog_name;B=_invalid_ident_char_re.sub('',A.replace('-',_D));C=_completion_scripts.get(shell,COMPLETION_SCRIPT_BASH);return (C%{'complete_func':f"_{B}_completion",'script_names':A,'autocomplete_var':complete_var}).strip()+';' -def resolve_ctx(cli,prog_name,args): - B=args;A=cli.make_context(prog_name,B,resilient_parsing=_A);B=A.protected_args+A.args - while B: - if isinstance(A.command,MultiCommand): - if not A.command.chain: - E,C,B=A.command.resolve_command(A,B) - if C is _B:return A - A=C.make_context(E,B,parent=A,resilient_parsing=_A);B=A.protected_args+A.args - else: - while B: - E,C,B=A.command.resolve_command(A,B) - if C is _B:return A - D=C.make_context(E,B,parent=A,allow_extra_args=_A,allow_interspersed_args=_C,resilient_parsing=_A);B=D.args - A=D;B=D.protected_args+D.args - else:break - return A -def start_of_option(param_str):A=param_str;return A and A[:1]=='-' -def is_incomplete_option(all_args,cmd_param): - A=cmd_param - if not isinstance(A,Option):return _C - if A.is_flag:return _C - B=_B - for (D,C) in enumerate(reversed([A for A in all_args if A!=WORDBREAK])): - if D+1>A.nargs:break - if start_of_option(C):B=C - return _A if B and B in A.opts else _C -def is_incomplete_argument(current_params,cmd_param): - A=cmd_param - if not isinstance(A,Argument):return _C - B=current_params[A.name] - if B is _B:return _A - if A.nargs==-1:return _A - if isinstance(B,abc.Iterable)and A.nargs>1 and len(B) cmd_param.nargs: + break + if start_of_option(arg_str): + last_option = arg_str + + return True if last_option and last_option in cmd_param.opts else False + + +def is_incomplete_argument(current_params, cmd_param): + """ + :param current_params: the current params and values for this + argument as already entered + :param cmd_param: the current command parameter + :return: whether or not the last argument is incomplete and + corresponds to this cmd_param. In other words whether or not the + this cmd_param argument can still accept values + """ + if not isinstance(cmd_param, Argument): + return False + current_param_values = current_params[cmd_param.name] + if current_param_values is None: + return True + if cmd_param.nargs == -1: + return True + if ( + isinstance(current_param_values, abc.Iterable) + and cmd_param.nargs > 1 + and len(current_param_values) < cmd_param.nargs + ): + return True + return False + + +def get_user_autocompletions(ctx, args, incomplete, cmd_param): + """ + :param ctx: context associated with the parsed command + :param args: full list of args + :param incomplete: the incomplete text to autocomplete + :param cmd_param: command definition + :return: all the possible user-specified completions for the param + """ + results = [] + if isinstance(cmd_param.type, Choice): + # Choices don't support descriptions. + results = [ + (c, None) for c in cmd_param.type.choices if str(c).startswith(incomplete) + ] + elif cmd_param.autocompletion is not None: + dynamic_completions = cmd_param.autocompletion( + ctx=ctx, args=args, incomplete=incomplete + ) + results = [ + c if isinstance(c, tuple) else (c, None) for c in dynamic_completions + ] + return results + + +def get_visible_commands_starting_with(ctx, starts_with): + """ + :param ctx: context associated with the parsed command + :starts_with: string that visible commands must start with. + :return: all visible (not hidden) commands that start with starts_with. + """ + for c in ctx.command.list_commands(ctx): + if c.startswith(starts_with): + command = ctx.command.get_command(ctx, c) + if not command.hidden: + yield command + + +def add_subcommand_completions(ctx, incomplete, completions_out): + # Add subcommand completions. + if isinstance(ctx.command, MultiCommand): + completions_out.extend( + [ + (c.name, c.get_short_help_str()) + for c in get_visible_commands_starting_with(ctx, incomplete) + ] + ) + + # Walk up the context list and add any other completion + # possibilities from chained commands + while ctx.parent is not None: + ctx = ctx.parent + if isinstance(ctx.command, MultiCommand) and ctx.command.chain: + remaining_commands = [ + c + for c in get_visible_commands_starting_with(ctx, incomplete) + if c.name not in ctx.protected_args + ] + completions_out.extend( + [(c.name, c.get_short_help_str()) for c in remaining_commands] + ) + + +def get_choices(cli, prog_name, args, incomplete): + """ + :param cli: command definition + :param prog_name: the program that is running + :param args: full list of args + :param incomplete: the incomplete text to autocomplete + :return: all the possible completions for the incomplete + """ + all_args = copy.deepcopy(args) + + ctx = resolve_ctx(cli, prog_name, args) + if ctx is None: + return [] + + has_double_dash = "--" in all_args + + # In newer versions of bash long opts with '='s are partitioned, but + # it's easier to parse without the '=' + if start_of_option(incomplete) and WORDBREAK in incomplete: + partition_incomplete = incomplete.partition(WORDBREAK) + all_args.append(partition_incomplete[0]) + incomplete = partition_incomplete[2] + elif incomplete == WORDBREAK: + incomplete = "" + + completions = [] + if not has_double_dash and start_of_option(incomplete): + # completions for partial options + for param in ctx.command.params: + if isinstance(param, Option) and not param.hidden: + param_opts = [ + param_opt + for param_opt in param.opts + param.secondary_opts + if param_opt not in all_args or param.multiple + ] + completions.extend( + [(o, param.help) for o in param_opts if o.startswith(incomplete)] + ) + return completions + # completion for option values from user supplied values + for param in ctx.command.params: + if is_incomplete_option(all_args, param): + return get_user_autocompletions(ctx, all_args, incomplete, param) + # completion for argument values from user supplied values + for param in ctx.command.params: + if is_incomplete_argument(ctx.params, param): + return get_user_autocompletions(ctx, all_args, incomplete, param) + + add_subcommand_completions(ctx, incomplete, completions) + # Sort before returning so that proper ordering can be enforced in custom types. + return sorted(completions) + + +def do_complete(cli, prog_name, include_descriptions): + cwords = split_arg_string(os.environ["COMP_WORDS"]) + cword = int(os.environ["COMP_CWORD"]) + args = cwords[1:cword] + try: + incomplete = cwords[cword] + except IndexError: + incomplete = "" + + for item in get_choices(cli, prog_name, args, incomplete): + echo(item[0]) + if include_descriptions: + # ZSH has trouble dealing with empty array parameters when + # returned from commands, use '_' to indicate no description + # is present. + echo(item[1] if item[1] else "_") + + return True + + +def do_complete_fish(cli, prog_name): + cwords = split_arg_string(os.environ["COMP_WORDS"]) + incomplete = os.environ["COMP_CWORD"] + args = cwords[1:] + + for item in get_choices(cli, prog_name, args, incomplete): + if item[1]: + echo(f"{item[0]}\t{item[1]}") + else: + echo(item[0]) + + return True + + +def bashcomplete(cli, prog_name, complete_var, complete_instr): + if "_" in complete_instr: + command, shell = complete_instr.split("_", 1) + else: + command = complete_instr + shell = "bash" + + if command == "source": + echo(get_completion_script(prog_name, complete_var, shell)) + return True + elif command == "complete": + if shell == "fish": + return do_complete_fish(cli, prog_name) + elif shell in {"bash", "zsh"}: + return do_complete(cli, prog_name, shell == "zsh") + + return False diff --git a/dynaconf/vendor/click/_compat.py b/dynaconf/vendor/click/_compat.py index f1eb2b4e8..85568ca3e 100644 --- a/dynaconf/vendor/click/_compat.py +++ b/dynaconf/vendor/click/_compat.py @@ -1,240 +1,611 @@ -_L='stderr' -_K='stdout' -_J='stdin' -_I='buffer' -_H='ascii' -_G='win' -_F='utf-8' -_E='encoding' -_D='replace' -_C=True -_B=False -_A=None -import codecs,io,os,re,sys +import codecs +import io +import os +import re +import sys from weakref import WeakKeyDictionary -CYGWIN=sys.platform.startswith('cygwin') -MSYS2=sys.platform.startswith(_G)and'GCC'in sys.version -APP_ENGINE='APPENGINE_RUNTIME'in os.environ and'Development/'in os.environ.get('SERVER_SOFTWARE','') -WIN=sys.platform.startswith(_G)and not APP_ENGINE and not MSYS2 -DEFAULT_COLUMNS=80 -auto_wrap_for_ansi=_A -colorama=_A -get_winterm_size=_A -_ansi_re=re.compile('\\033\\[[;?0-9]*[a-zA-Z]') -def get_filesystem_encoding():return sys.getfilesystemencoding()or sys.getdefaultencoding() -def _make_text_stream(stream,encoding,errors,force_readable=_B,force_writable=_B): - C=stream;B=errors;A=encoding - if A is _A:A=get_best_encoding(C) - if B is _A:B=_D - return _NonClosingTextIOWrapper(C,A,B,line_buffering=_C,force_readable=force_readable,force_writable=force_writable) + +CYGWIN = sys.platform.startswith("cygwin") +MSYS2 = sys.platform.startswith("win") and ("GCC" in sys.version) +# Determine local App Engine environment, per Google's own suggestion +APP_ENGINE = "APPENGINE_RUNTIME" in os.environ and "Development/" in os.environ.get( + "SERVER_SOFTWARE", "" +) +WIN = sys.platform.startswith("win") and not APP_ENGINE and not MSYS2 +DEFAULT_COLUMNS = 80 +auto_wrap_for_ansi = None +colorama = None +get_winterm_size = None +_ansi_re = re.compile(r"\033\[[;?0-9]*[a-zA-Z]") + + +def get_filesystem_encoding(): + return sys.getfilesystemencoding() or sys.getdefaultencoding() + + +def _make_text_stream( + stream, encoding, errors, force_readable=False, force_writable=False +): + if encoding is None: + encoding = get_best_encoding(stream) + if errors is None: + errors = "replace" + return _NonClosingTextIOWrapper( + stream, + encoding, + errors, + line_buffering=True, + force_readable=force_readable, + force_writable=force_writable, + ) + + def is_ascii_encoding(encoding): - try:return codecs.lookup(encoding).name==_H - except LookupError:return _B + """Checks if a given encoding is ascii.""" + try: + return codecs.lookup(encoding).name == "ascii" + except LookupError: + return False + + def get_best_encoding(stream): - A=getattr(stream,_E,_A)or sys.getdefaultencoding() - if is_ascii_encoding(A):return _F - return A + """Returns the default stream encoding if not found.""" + rv = getattr(stream, "encoding", None) or sys.getdefaultencoding() + if is_ascii_encoding(rv): + return "utf-8" + return rv + + class _NonClosingTextIOWrapper(io.TextIOWrapper): - def __init__(B,stream,encoding,errors,force_readable=_B,force_writable=_B,**C):A=stream;B._stream=A=_FixupStream(A,force_readable,force_writable);super().__init__(A,encoding,errors,**C) - def __del__(A): - try:A.detach() - except Exception:pass - def isatty(A):return A._stream.isatty() + def __init__( + self, + stream, + encoding, + errors, + force_readable=False, + force_writable=False, + **extra, + ): + self._stream = stream = _FixupStream(stream, force_readable, force_writable) + super().__init__(stream, encoding, errors, **extra) + + def __del__(self): + try: + self.detach() + except Exception: + pass + + def isatty(self): + # https://bitbucket.org/pypy/pypy/issue/1803 + return self._stream.isatty() + + class _FixupStream: - def __init__(A,stream,force_readable=_B,force_writable=_B):A._stream=stream;A._force_readable=force_readable;A._force_writable=force_writable - def __getattr__(A,name):return getattr(A._stream,name) - def read1(A,size): - B=getattr(A._stream,'read1',_A) - if B is not _A:return B(size) - return A._stream.read(size) - def readable(A): - if A._force_readable:return _C - B=getattr(A._stream,'readable',_A) - if B is not _A:return B() - try:A._stream.read(0) - except Exception:return _B - return _C - def writable(A): - if A._force_writable:return _C - B=getattr(A._stream,'writable',_A) - if B is not _A:return B() - try:A._stream.write('') - except Exception: - try:A._stream.write(b'') - except Exception:return _B - return _C - def seekable(A): - B=getattr(A._stream,'seekable',_A) - if B is not _A:return B() - try:A._stream.seek(A._stream.tell()) - except Exception:return _B - return _C -def is_bytes(x):return isinstance(x,(bytes,memoryview,bytearray)) -def _is_binary_reader(stream,default=_B): - try:return isinstance(stream.read(0),bytes) - except Exception:return default -def _is_binary_writer(stream,default=_B): - A=stream - try:A.write(b'') - except Exception: - try:A.write('');return _B - except Exception:pass - return default - return _C + """The new io interface needs more from streams than streams + traditionally implement. As such, this fix-up code is necessary in + some circumstances. + + The forcing of readable and writable flags are there because some tools + put badly patched objects on sys (one such offender are certain version + of jupyter notebook). + """ + + def __init__(self, stream, force_readable=False, force_writable=False): + self._stream = stream + self._force_readable = force_readable + self._force_writable = force_writable + + def __getattr__(self, name): + return getattr(self._stream, name) + + def read1(self, size): + f = getattr(self._stream, "read1", None) + if f is not None: + return f(size) + + return self._stream.read(size) + + def readable(self): + if self._force_readable: + return True + x = getattr(self._stream, "readable", None) + if x is not None: + return x() + try: + self._stream.read(0) + except Exception: + return False + return True + + def writable(self): + if self._force_writable: + return True + x = getattr(self._stream, "writable", None) + if x is not None: + return x() + try: + self._stream.write("") + except Exception: + try: + self._stream.write(b"") + except Exception: + return False + return True + + def seekable(self): + x = getattr(self._stream, "seekable", None) + if x is not None: + return x() + try: + self._stream.seek(self._stream.tell()) + except Exception: + return False + return True + + +def is_bytes(x): + return isinstance(x, (bytes, memoryview, bytearray)) + + +def _is_binary_reader(stream, default=False): + try: + return isinstance(stream.read(0), bytes) + except Exception: + return default + # This happens in some cases where the stream was already + # closed. In this case, we assume the default. + + +def _is_binary_writer(stream, default=False): + try: + stream.write(b"") + except Exception: + try: + stream.write("") + return False + except Exception: + pass + return default + return True + + def _find_binary_reader(stream): - A=stream - if _is_binary_reader(A,_B):return A - B=getattr(A,_I,_A) - if B is not _A and _is_binary_reader(B,_C):return B + # We need to figure out if the given stream is already binary. + # This can happen because the official docs recommend detaching + # the streams to get binary streams. Some code might do this, so + # we need to deal with this case explicitly. + if _is_binary_reader(stream, False): + return stream + + buf = getattr(stream, "buffer", None) + + # Same situation here; this time we assume that the buffer is + # actually binary in case it's closed. + if buf is not None and _is_binary_reader(buf, True): + return buf + + def _find_binary_writer(stream): - A=stream - if _is_binary_writer(A,_B):return A - B=getattr(A,_I,_A) - if B is not _A and _is_binary_writer(B,_C):return B -def _stream_is_misconfigured(stream):return is_ascii_encoding(getattr(stream,_E,_A)or _H) -def _is_compat_stream_attr(stream,attr,value):A=value;B=getattr(stream,attr,_A);return B==A or A is _A and B is not _A -def _is_compatible_text_stream(stream,encoding,errors):A=stream;return _is_compat_stream_attr(A,_E,encoding)and _is_compat_stream_attr(A,'errors',errors) -def _force_correct_text_stream(text_stream,encoding,errors,is_binary,find_binary,force_readable=_B,force_writable=_B): - C=encoding;B=errors;A=text_stream - if is_binary(A,_B):D=A - else: - if _is_compatible_text_stream(A,C,B)and not(C is _A and _stream_is_misconfigured(A)):return A - D=find_binary(A) - if D is _A:return A - if B is _A:B=_D - return _make_text_stream(D,C,B,force_readable=force_readable,force_writable=force_writable) -def _force_correct_text_reader(text_reader,encoding,errors,force_readable=_B):return _force_correct_text_stream(text_reader,encoding,errors,_is_binary_reader,_find_binary_reader,force_readable=force_readable) -def _force_correct_text_writer(text_writer,encoding,errors,force_writable=_B):return _force_correct_text_stream(text_writer,encoding,errors,_is_binary_writer,_find_binary_writer,force_writable=force_writable) + # We need to figure out if the given stream is already binary. + # This can happen because the official docs recommend detaching + # the streams to get binary streams. Some code might do this, so + # we need to deal with this case explicitly. + if _is_binary_writer(stream, False): + return stream + + buf = getattr(stream, "buffer", None) + + # Same situation here; this time we assume that the buffer is + # actually binary in case it's closed. + if buf is not None and _is_binary_writer(buf, True): + return buf + + +def _stream_is_misconfigured(stream): + """A stream is misconfigured if its encoding is ASCII.""" + # If the stream does not have an encoding set, we assume it's set + # to ASCII. This appears to happen in certain unittest + # environments. It's not quite clear what the correct behavior is + # but this at least will force Click to recover somehow. + return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii") + + +def _is_compat_stream_attr(stream, attr, value): + """A stream attribute is compatible if it is equal to the + desired value or the desired value is unset and the attribute + has a value. + """ + stream_value = getattr(stream, attr, None) + return stream_value == value or (value is None and stream_value is not None) + + +def _is_compatible_text_stream(stream, encoding, errors): + """Check if a stream's encoding and errors attributes are + compatible with the desired values. + """ + return _is_compat_stream_attr( + stream, "encoding", encoding + ) and _is_compat_stream_attr(stream, "errors", errors) + + +def _force_correct_text_stream( + text_stream, + encoding, + errors, + is_binary, + find_binary, + force_readable=False, + force_writable=False, +): + if is_binary(text_stream, False): + binary_reader = text_stream + else: + # If the stream looks compatible, and won't default to a + # misconfigured ascii encoding, return it as-is. + if _is_compatible_text_stream(text_stream, encoding, errors) and not ( + encoding is None and _stream_is_misconfigured(text_stream) + ): + return text_stream + + # Otherwise, get the underlying binary reader. + binary_reader = find_binary(text_stream) + + # If that's not possible, silently use the original reader + # and get mojibake instead of exceptions. + if binary_reader is None: + return text_stream + + # Default errors to replace instead of strict in order to get + # something that works. + if errors is None: + errors = "replace" + + # Wrap the binary stream in a text stream with the correct + # encoding parameters. + return _make_text_stream( + binary_reader, + encoding, + errors, + force_readable=force_readable, + force_writable=force_writable, + ) + + +def _force_correct_text_reader(text_reader, encoding, errors, force_readable=False): + return _force_correct_text_stream( + text_reader, + encoding, + errors, + _is_binary_reader, + _find_binary_reader, + force_readable=force_readable, + ) + + +def _force_correct_text_writer(text_writer, encoding, errors, force_writable=False): + return _force_correct_text_stream( + text_writer, + encoding, + errors, + _is_binary_writer, + _find_binary_writer, + force_writable=force_writable, + ) + + def get_binary_stdin(): - A=_find_binary_reader(sys.stdin) - if A is _A:raise RuntimeError('Was not able to determine binary stream for sys.stdin.') - return A + reader = _find_binary_reader(sys.stdin) + if reader is None: + raise RuntimeError("Was not able to determine binary stream for sys.stdin.") + return reader + + def get_binary_stdout(): - A=_find_binary_writer(sys.stdout) - if A is _A:raise RuntimeError('Was not able to determine binary stream for sys.stdout.') - return A + writer = _find_binary_writer(sys.stdout) + if writer is None: + raise RuntimeError("Was not able to determine binary stream for sys.stdout.") + return writer + + def get_binary_stderr(): - A=_find_binary_writer(sys.stderr) - if A is _A:raise RuntimeError('Was not able to determine binary stream for sys.stderr.') - return A -def get_text_stdin(encoding=_A,errors=_A): - B=errors;A=encoding;C=_get_windows_console_stream(sys.stdin,A,B) - if C is not _A:return C - return _force_correct_text_reader(sys.stdin,A,B,force_readable=_C) -def get_text_stdout(encoding=_A,errors=_A): - B=errors;A=encoding;C=_get_windows_console_stream(sys.stdout,A,B) - if C is not _A:return C - return _force_correct_text_writer(sys.stdout,A,B,force_writable=_C) -def get_text_stderr(encoding=_A,errors=_A): - B=errors;A=encoding;C=_get_windows_console_stream(sys.stderr,A,B) - if C is not _A:return C - return _force_correct_text_writer(sys.stderr,A,B,force_writable=_C) + writer = _find_binary_writer(sys.stderr) + if writer is None: + raise RuntimeError("Was not able to determine binary stream for sys.stderr.") + return writer + + +def get_text_stdin(encoding=None, errors=None): + rv = _get_windows_console_stream(sys.stdin, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_reader(sys.stdin, encoding, errors, force_readable=True) + + +def get_text_stdout(encoding=None, errors=None): + rv = _get_windows_console_stream(sys.stdout, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_writer(sys.stdout, encoding, errors, force_writable=True) + + +def get_text_stderr(encoding=None, errors=None): + rv = _get_windows_console_stream(sys.stderr, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_writer(sys.stderr, encoding, errors, force_writable=True) + + def filename_to_ui(value): - A=value - if isinstance(A,bytes):A=A.decode(get_filesystem_encoding(),_D) - else:A=A.encode(_F,'surrogateescape').decode(_F,_D) - return A -def get_strerror(e,default=_A): - B=default - if hasattr(e,'strerror'):A=e.strerror - elif B is not _A:A=B - else:A=str(e) - if isinstance(A,bytes):A=A.decode(_F,_D) - return A -def _wrap_io_open(file,mode,encoding,errors): - A=mode - if'b'in A:return open(file,A) - return open(file,A,encoding=encoding,errors=errors) -def open_stream(filename,mode='r',encoding=_A,errors='strict',atomic=_B): - P='x';O='a';N='w';E=errors;D=encoding;B=filename;A=mode;G='b'in A - if B=='-': - if any((B in A for B in[N,O,P])): - if G:return get_binary_stdout(),_B - return get_text_stdout(encoding=D,errors=E),_B - if G:return get_binary_stdin(),_B - return get_text_stdin(encoding=D,errors=E),_B - if not atomic:return _wrap_io_open(B,A,D,E),_C - if O in A:raise ValueError("Appending to an existing file is not supported, because that would involve an expensive `copy`-operation to a temporary file. Open the file in normal `w`-mode and copy explicitly if that's what you're after.") - if P in A:raise ValueError('Use the `overwrite`-parameter instead.') - if N not in A:raise ValueError('Atomic writes only make sense with `w`-mode.') - import errno as I,random as K - try:C=os.stat(B).st_mode - except OSError:C=_A - J=os.O_RDWR|os.O_CREAT|os.O_EXCL - if G:J|=getattr(os,'O_BINARY',0) - while _C: - H=os.path.join(os.path.dirname(B),f".__atomic-write{K.randrange(1<<32):08x}") - try:L=os.open(H,J,438 if C is _A else C);break - except OSError as F: - if F.errno==I.EEXIST or os.name=='nt'and F.errno==I.EACCES and os.path.isdir(F.filename)and os.access(F.filename,os.W_OK):continue - raise - if C is not _A:os.chmod(H,C) - M=_wrap_io_open(L,A,D,E);return _AtomicFile(M,H,os.path.realpath(B)),_C + if isinstance(value, bytes): + value = value.decode(get_filesystem_encoding(), "replace") + else: + value = value.encode("utf-8", "surrogateescape").decode("utf-8", "replace") + return value + + +def get_strerror(e, default=None): + if hasattr(e, "strerror"): + msg = e.strerror + else: + if default is not None: + msg = default + else: + msg = str(e) + if isinstance(msg, bytes): + msg = msg.decode("utf-8", "replace") + return msg + + +def _wrap_io_open(file, mode, encoding, errors): + """Handles not passing ``encoding`` and ``errors`` in binary mode.""" + if "b" in mode: + return open(file, mode) + + return open(file, mode, encoding=encoding, errors=errors) + + +def open_stream(filename, mode="r", encoding=None, errors="strict", atomic=False): + binary = "b" in mode + + # Standard streams first. These are simple because they don't need + # special handling for the atomic flag. It's entirely ignored. + if filename == "-": + if any(m in mode for m in ["w", "a", "x"]): + if binary: + return get_binary_stdout(), False + return get_text_stdout(encoding=encoding, errors=errors), False + if binary: + return get_binary_stdin(), False + return get_text_stdin(encoding=encoding, errors=errors), False + + # Non-atomic writes directly go out through the regular open functions. + if not atomic: + return _wrap_io_open(filename, mode, encoding, errors), True + + # Some usability stuff for atomic writes + if "a" in mode: + raise ValueError( + "Appending to an existing file is not supported, because that" + " would involve an expensive `copy`-operation to a temporary" + " file. Open the file in normal `w`-mode and copy explicitly" + " if that's what you're after." + ) + if "x" in mode: + raise ValueError("Use the `overwrite`-parameter instead.") + if "w" not in mode: + raise ValueError("Atomic writes only make sense with `w`-mode.") + + # Atomic writes are more complicated. They work by opening a file + # as a proxy in the same folder and then using the fdopen + # functionality to wrap it in a Python file. Then we wrap it in an + # atomic file that moves the file over on close. + import errno + import random + + try: + perm = os.stat(filename).st_mode + except OSError: + perm = None + + flags = os.O_RDWR | os.O_CREAT | os.O_EXCL + + if binary: + flags |= getattr(os, "O_BINARY", 0) + + while True: + tmp_filename = os.path.join( + os.path.dirname(filename), + f".__atomic-write{random.randrange(1 << 32):08x}", + ) + try: + fd = os.open(tmp_filename, flags, 0o666 if perm is None else perm) + break + except OSError as e: + if e.errno == errno.EEXIST or ( + os.name == "nt" + and e.errno == errno.EACCES + and os.path.isdir(e.filename) + and os.access(e.filename, os.W_OK) + ): + continue + raise + + if perm is not None: + os.chmod(tmp_filename, perm) # in case perm includes bits in umask + + f = _wrap_io_open(fd, mode, encoding, errors) + return _AtomicFile(f, tmp_filename, os.path.realpath(filename)), True + + class _AtomicFile: - def __init__(A,f,tmp_filename,real_filename):A._f=f;A._tmp_filename=tmp_filename;A._real_filename=real_filename;A.closed=_B - @property - def name(self):return self._real_filename - def close(A,delete=_B): - if A.closed:return - A._f.close();os.replace(A._tmp_filename,A._real_filename);A.closed=_C - def __getattr__(A,name):return getattr(A._f,name) - def __enter__(A):return A - def __exit__(A,exc_type,exc_value,tb):A.close(delete=exc_type is not _A) - def __repr__(A):return repr(A._f) -def strip_ansi(value):return _ansi_re.sub('',value) + def __init__(self, f, tmp_filename, real_filename): + self._f = f + self._tmp_filename = tmp_filename + self._real_filename = real_filename + self.closed = False + + @property + def name(self): + return self._real_filename + + def close(self, delete=False): + if self.closed: + return + self._f.close() + os.replace(self._tmp_filename, self._real_filename) + self.closed = True + + def __getattr__(self, name): + return getattr(self._f, name) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, tb): + self.close(delete=exc_type is not None) + + def __repr__(self): + return repr(self._f) + + +def strip_ansi(value): + return _ansi_re.sub("", value) + + def _is_jupyter_kernel_output(stream): - A=stream - if WIN:return - while isinstance(A,(_FixupStream,_NonClosingTextIOWrapper)):A=A._stream - return A.__class__.__module__.startswith('ipykernel.') -def should_strip_ansi(stream=_A,color=_A): - B=color;A=stream - if B is _A: - if A is _A:A=sys.stdin - return not isatty(A)and not _is_jupyter_kernel_output(A) - return not B + if WIN: + # TODO: Couldn't test on Windows, should't try to support until + # someone tests the details wrt colorama. + return + + while isinstance(stream, (_FixupStream, _NonClosingTextIOWrapper)): + stream = stream._stream + + return stream.__class__.__module__.startswith("ipykernel.") + + +def should_strip_ansi(stream=None, color=None): + if color is None: + if stream is None: + stream = sys.stdin + return not isatty(stream) and not _is_jupyter_kernel_output(stream) + return not color + + +# If we're on Windows, we provide transparent integration through +# colorama. This will make ANSI colors through the echo function +# work automatically. if WIN: - DEFAULT_COLUMNS=79;from ._winconsole import _get_windows_console_stream - def _get_argv_encoding():import locale as A;return A.getpreferredencoding() - try:import colorama - except ImportError:pass - else: - _ansi_stream_wrappers=WeakKeyDictionary() - def auto_wrap_for_ansi(stream,color=_A): - A=stream - try:C=_ansi_stream_wrappers.get(A) - except Exception:C=_A - if C is not _A:return C - E=should_strip_ansi(A,color);D=colorama.AnsiToWin32(A,strip=E);B=D.stream;F=B.write - def G(s): - try:return F(s) - except BaseException:D.reset_all();raise - B.write=G - try:_ansi_stream_wrappers[A]=B - except Exception:pass - return B - def get_winterm_size():A=colorama.win32.GetConsoleScreenBufferInfo(colorama.win32.STDOUT).srWindow;return A.Right-A.Left,A.Bottom-A.Top + # Windows has a smaller terminal + DEFAULT_COLUMNS = 79 + + from ._winconsole import _get_windows_console_stream + + def _get_argv_encoding(): + import locale + + return locale.getpreferredencoding() + + try: + import colorama + except ImportError: + pass + else: + _ansi_stream_wrappers = WeakKeyDictionary() + + def auto_wrap_for_ansi(stream, color=None): + """This function wraps a stream so that calls through colorama + are issued to the win32 console API to recolor on demand. It + also ensures to reset the colors if a write call is interrupted + to not destroy the console afterwards. + """ + try: + cached = _ansi_stream_wrappers.get(stream) + except Exception: + cached = None + if cached is not None: + return cached + strip = should_strip_ansi(stream, color) + ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip) + rv = ansi_wrapper.stream + _write = rv.write + + def _safe_write(s): + try: + return _write(s) + except BaseException: + ansi_wrapper.reset_all() + raise + + rv.write = _safe_write + try: + _ansi_stream_wrappers[stream] = rv + except Exception: + pass + return rv + + def get_winterm_size(): + win = colorama.win32.GetConsoleScreenBufferInfo( + colorama.win32.STDOUT + ).srWindow + return win.Right - win.Left, win.Bottom - win.Top + + else: - def _get_argv_encoding():return getattr(sys.stdin,_E,_A)or get_filesystem_encoding() - def _get_windows_console_stream(f,encoding,errors):return _A -def term_len(x):return len(strip_ansi(x)) + + def _get_argv_encoding(): + return getattr(sys.stdin, "encoding", None) or get_filesystem_encoding() + + def _get_windows_console_stream(f, encoding, errors): + return None + + +def term_len(x): + return len(strip_ansi(x)) + + def isatty(stream): - try:return stream.isatty() - except Exception:return _B -def _make_cached_stream_func(src_func,wrapper_func): - C=src_func;D=WeakKeyDictionary() - def A(): - B=C() - try:A=D.get(B) - except Exception:A=_A - if A is not _A:return A - A=wrapper_func() - try:B=C();D[B]=A - except Exception:pass - return A - return A -_default_text_stdin=_make_cached_stream_func(lambda:sys.stdin,get_text_stdin) -_default_text_stdout=_make_cached_stream_func(lambda:sys.stdout,get_text_stdout) -_default_text_stderr=_make_cached_stream_func(lambda:sys.stderr,get_text_stderr) -binary_streams={_J:get_binary_stdin,_K:get_binary_stdout,_L:get_binary_stderr} -text_streams={_J:get_text_stdin,_K:get_text_stdout,_L:get_text_stderr} \ No newline at end of file + try: + return stream.isatty() + except Exception: + return False + + +def _make_cached_stream_func(src_func, wrapper_func): + cache = WeakKeyDictionary() + + def func(): + stream = src_func() + try: + rv = cache.get(stream) + except Exception: + rv = None + if rv is not None: + return rv + rv = wrapper_func() + try: + stream = src_func() # In case wrapper_func() modified the stream + cache[stream] = rv + except Exception: + pass + return rv + + return func + + +_default_text_stdin = _make_cached_stream_func(lambda: sys.stdin, get_text_stdin) +_default_text_stdout = _make_cached_stream_func(lambda: sys.stdout, get_text_stdout) +_default_text_stderr = _make_cached_stream_func(lambda: sys.stderr, get_text_stderr) + + +binary_streams = { + "stdin": get_binary_stdin, + "stdout": get_binary_stdout, + "stderr": get_binary_stderr, +} + +text_streams = { + "stdin": get_text_stdin, + "stdout": get_text_stdout, + "stderr": get_text_stderr, +} diff --git a/dynaconf/vendor/click/_termui_impl.py b/dynaconf/vendor/click/_termui_impl.py index b18a9f222..78372503d 100644 --- a/dynaconf/vendor/click/_termui_impl.py +++ b/dynaconf/vendor/click/_termui_impl.py @@ -1,262 +1,667 @@ -_H='replace' -_G='less' -_F='You need to use progress bars in a with block.' -_E=' ' -_D='\n' -_C=False -_B=True -_A=None -import contextlib,math,os,sys,time -from ._compat import _default_text_stdout,CYGWIN,get_best_encoding,isatty,open_stream,strip_ansi,term_len,WIN +""" +This module contains implementations for the termui module. To keep the +import time of Click down, some infrequently used functionality is +placed in this module and only imported as needed. +""" +import contextlib +import math +import os +import sys +import time + +from ._compat import _default_text_stdout +from ._compat import CYGWIN +from ._compat import get_best_encoding +from ._compat import isatty +from ._compat import open_stream +from ._compat import strip_ansi +from ._compat import term_len +from ._compat import WIN from .exceptions import ClickException from .utils import echo -if os.name=='nt':BEFORE_BAR='\r';AFTER_BAR=_D -else:BEFORE_BAR='\r\x1b[?25l';AFTER_BAR='\x1b[?25h\n' + +if os.name == "nt": + BEFORE_BAR = "\r" + AFTER_BAR = "\n" +else: + BEFORE_BAR = "\r\033[?25l" + AFTER_BAR = "\033[?25h\n" + + def _length_hint(obj): - B=obj - try:return len(B) - except (AttributeError,TypeError): - try:C=type(B).__length_hint__ - except AttributeError:return _A - try:A=C(B) - except TypeError:return _A - if A is NotImplemented or not isinstance(A,int)or A<0:return _A - return A + """Returns the length hint of an object.""" + try: + return len(obj) + except (AttributeError, TypeError): + try: + get_hint = type(obj).__length_hint__ + except AttributeError: + return None + try: + hint = get_hint(obj) + except TypeError: + return None + if hint is NotImplemented or not isinstance(hint, int) or hint < 0: + return None + return hint + + class ProgressBar: - def __init__(A,iterable,length=_A,fill_char='#',empty_char=_E,bar_template='%(bar)s',info_sep=' ',show_eta=_B,show_percent=_A,show_pos=_C,item_show_func=_A,label=_A,file=_A,color=_A,width=30): - E=width;D=file;C=iterable;B=length;A.fill_char=fill_char;A.empty_char=empty_char;A.bar_template=bar_template;A.info_sep=info_sep;A.show_eta=show_eta;A.show_percent=show_percent;A.show_pos=show_pos;A.item_show_func=item_show_func;A.label=label or'' - if D is _A:D=_default_text_stdout() - A.file=D;A.color=color;A.width=E;A.autowidth=E==0 - if B is _A:B=_length_hint(C) - if C is _A: - if B is _A:raise TypeError('iterable or length is required') - C=range(B) - A.iter=iter(C);A.length=B;A.length_known=B is not _A;A.pos=0;A.avg=[];A.start=A.last_eta=time.time();A.eta_known=_C;A.finished=_C;A.max_width=_A;A.entered=_C;A.current_item=_A;A.is_hidden=not isatty(A.file);A._last_line=_A;A.short_limit=0.5 - def __enter__(A):A.entered=_B;A.render_progress();return A - def __exit__(A,exc_type,exc_value,tb):A.render_finish() - def __iter__(A): - if not A.entered:raise RuntimeError(_F) - A.render_progress();return A.generator() - def __next__(A):return next(iter(A)) - def is_fast(A):return time.time()-A.start<=A.short_limit - def render_finish(A): - if A.is_hidden or A.is_fast():return - A.file.write(AFTER_BAR);A.file.flush() - @property - def pct(self): - A=self - if A.finished:return 1.0 - return min(A.pos/(float(A.length)or 1),1.0) - @property - def time_per_iteration(self): - A=self - if not A.avg:return 0.0 - return sum(A.avg)/float(len(A.avg)) - @property - def eta(self): - A=self - if A.length_known and not A.finished:return A.time_per_iteration*(A.length-A.pos) - return 0.0 - def format_eta(B): - if B.eta_known: - A=int(B.eta);C=A%60;A//=60;D=A%60;A//=60;E=A%24;A//=24 - if A>0:return f"{A}d {E:02}:{D:02}:{C:02}" - else:return f"{E:02}:{D:02}:{C:02}" - return'' - def format_pos(A): - B=str(A.pos) - if A.length_known:B+=f"/{A.length}" - return B - def format_pct(A):return f"{int(A.pct*100): 4}%"[1:] - def format_bar(A): - if A.length_known:C=int(A.pct*A.width);B=A.fill_char*C;B+=A.empty_char*(A.width-C) - elif A.finished:B=A.fill_char*A.width - else: - B=list(A.empty_char*(A.width or 1)) - if A.time_per_iteration!=0:B[int((math.cos(A.pos*A.time_per_iteration)/2.0+0.5)*A.width)]=A.fill_char - B=''.join(B) - return B - def format_progress_line(A): - C=A.show_percent;B=[] - if A.length_known and C is _A:C=not A.show_pos - if A.show_pos:B.append(A.format_pos()) - if C:B.append(A.format_pct()) - if A.show_eta and A.eta_known and not A.finished:B.append(A.format_eta()) - if A.item_show_func is not _A: - D=A.item_show_func(A.current_item) - if D is not _A:B.append(D) - return (A.bar_template%{'label':A.label,'bar':A.format_bar(),'info':A.info_sep.join(B)}).rstrip() - def render_progress(A): - from .termui import get_terminal_size as G - if A.is_hidden:return - B=[] - if A.autowidth: - H=A.width;A.width=0;I=term_len(A.format_progress_line());D=max(0,G()[0]-I) - if D=A.length:A.finished=_B - if time.time()-A.last_eta<1.0:return - A.last_eta=time.time() - if A.pos:B=(time.time()-A.start)/A.pos - else:B=time.time()-A.start - A.avg=A.avg[-6:]+[B];A.eta_known=A.length_known - def update(A,n_steps,current_item=_A): - B=current_item;A.make_step(n_steps) - if B is not _A:A.current_item=B - A.render_progress() - def finish(A):A.eta_known=0;A.current_item=_A;A.finished=_B - def generator(A): - if not A.entered:raise RuntimeError(_F) - if A.is_hidden:yield from A.iter - else: - for B in A.iter:A.current_item=B;yield B;A.update(1) - A.finish();A.render_progress() -def pager(generator,color=_A): - H='system';B=color;A=generator;C=_default_text_stdout() - if not isatty(sys.stdin)or not isatty(C):return _nullpager(C,A,B) - D=(os.environ.get('PAGER',_A)or'').strip() - if D: - if WIN:return _tempfilepager(A,D,B) - return _pipepager(A,D,B) - if os.environ.get('TERM')in('dumb','emacs'):return _nullpager(C,A,B) - if WIN or sys.platform.startswith('os2'):return _tempfilepager(A,'more <',B) - if hasattr(os,H)and os.system('(less) 2>/dev/null')==0:return _pipepager(A,_G,B) - import tempfile as F;G,E=F.mkstemp();os.close(G) - try: - if hasattr(os,H)and os.system(f'more "{E}"')==0:return _pipepager(A,'more',B) - return _nullpager(C,A,B) - finally:os.unlink(E) -def _pipepager(generator,cmd,color): - I='LESS';A=color;import subprocess as E;F=dict(os.environ);G=cmd.rsplit('/',1)[-1].split() - if A is _A and G[0]==_G: - C=f"{os.environ.get(I,'')}{_E.join(G[1:])}" - if not C:F[I]='-R';A=_B - elif'r'in C or'R'in C:A=_B - B=E.Popen(cmd,shell=_B,stdin=E.PIPE,env=F);H=get_best_encoding(B.stdin) - try: - for D in generator: - if not A:D=strip_ansi(D) - B.stdin.write(D.encode(H,_H)) - except (OSError,KeyboardInterrupt):pass - else:B.stdin.close() - while _B: - try:B.wait() - except KeyboardInterrupt:pass - else:break -def _tempfilepager(generator,cmd,color): - import tempfile as C;A=C.mktemp();B=''.join(generator) - if not color:B=strip_ansi(B) - D=get_best_encoding(sys.stdout) - with open_stream(A,'wb')[0]as E:E.write(B.encode(D)) - try:os.system(f'{cmd} "{A}"') - finally:os.unlink(A) -def _nullpager(stream,generator,color): - for A in generator: - if not color:A=strip_ansi(A) - stream.write(A) + def __init__( + self, + iterable, + length=None, + fill_char="#", + empty_char=" ", + bar_template="%(bar)s", + info_sep=" ", + show_eta=True, + show_percent=None, + show_pos=False, + item_show_func=None, + label=None, + file=None, + color=None, + width=30, + ): + self.fill_char = fill_char + self.empty_char = empty_char + self.bar_template = bar_template + self.info_sep = info_sep + self.show_eta = show_eta + self.show_percent = show_percent + self.show_pos = show_pos + self.item_show_func = item_show_func + self.label = label or "" + if file is None: + file = _default_text_stdout() + self.file = file + self.color = color + self.width = width + self.autowidth = width == 0 + + if length is None: + length = _length_hint(iterable) + if iterable is None: + if length is None: + raise TypeError("iterable or length is required") + iterable = range(length) + self.iter = iter(iterable) + self.length = length + self.length_known = length is not None + self.pos = 0 + self.avg = [] + self.start = self.last_eta = time.time() + self.eta_known = False + self.finished = False + self.max_width = None + self.entered = False + self.current_item = None + self.is_hidden = not isatty(self.file) + self._last_line = None + self.short_limit = 0.5 + + def __enter__(self): + self.entered = True + self.render_progress() + return self + + def __exit__(self, exc_type, exc_value, tb): + self.render_finish() + + def __iter__(self): + if not self.entered: + raise RuntimeError("You need to use progress bars in a with block.") + self.render_progress() + return self.generator() + + def __next__(self): + # Iteration is defined in terms of a generator function, + # returned by iter(self); use that to define next(). This works + # because `self.iter` is an iterable consumed by that generator, + # so it is re-entry safe. Calling `next(self.generator())` + # twice works and does "what you want". + return next(iter(self)) + + def is_fast(self): + return time.time() - self.start <= self.short_limit + + def render_finish(self): + if self.is_hidden or self.is_fast(): + return + self.file.write(AFTER_BAR) + self.file.flush() + + @property + def pct(self): + if self.finished: + return 1.0 + return min(self.pos / (float(self.length) or 1), 1.0) + + @property + def time_per_iteration(self): + if not self.avg: + return 0.0 + return sum(self.avg) / float(len(self.avg)) + + @property + def eta(self): + if self.length_known and not self.finished: + return self.time_per_iteration * (self.length - self.pos) + return 0.0 + + def format_eta(self): + if self.eta_known: + t = int(self.eta) + seconds = t % 60 + t //= 60 + minutes = t % 60 + t //= 60 + hours = t % 24 + t //= 24 + if t > 0: + return f"{t}d {hours:02}:{minutes:02}:{seconds:02}" + else: + return f"{hours:02}:{minutes:02}:{seconds:02}" + return "" + + def format_pos(self): + pos = str(self.pos) + if self.length_known: + pos += f"/{self.length}" + return pos + + def format_pct(self): + return f"{int(self.pct * 100): 4}%"[1:] + + def format_bar(self): + if self.length_known: + bar_length = int(self.pct * self.width) + bar = self.fill_char * bar_length + bar += self.empty_char * (self.width - bar_length) + elif self.finished: + bar = self.fill_char * self.width + else: + bar = list(self.empty_char * (self.width or 1)) + if self.time_per_iteration != 0: + bar[ + int( + (math.cos(self.pos * self.time_per_iteration) / 2.0 + 0.5) + * self.width + ) + ] = self.fill_char + bar = "".join(bar) + return bar + + def format_progress_line(self): + show_percent = self.show_percent + + info_bits = [] + if self.length_known and show_percent is None: + show_percent = not self.show_pos + + if self.show_pos: + info_bits.append(self.format_pos()) + if show_percent: + info_bits.append(self.format_pct()) + if self.show_eta and self.eta_known and not self.finished: + info_bits.append(self.format_eta()) + if self.item_show_func is not None: + item_info = self.item_show_func(self.current_item) + if item_info is not None: + info_bits.append(item_info) + + return ( + self.bar_template + % { + "label": self.label, + "bar": self.format_bar(), + "info": self.info_sep.join(info_bits), + } + ).rstrip() + + def render_progress(self): + from .termui import get_terminal_size + + if self.is_hidden: + return + + buf = [] + # Update width in case the terminal has been resized + if self.autowidth: + old_width = self.width + self.width = 0 + clutter_length = term_len(self.format_progress_line()) + new_width = max(0, get_terminal_size()[0] - clutter_length) + if new_width < old_width: + buf.append(BEFORE_BAR) + buf.append(" " * self.max_width) + self.max_width = new_width + self.width = new_width + + clear_width = self.width + if self.max_width is not None: + clear_width = self.max_width + + buf.append(BEFORE_BAR) + line = self.format_progress_line() + line_len = term_len(line) + if self.max_width is None or self.max_width < line_len: + self.max_width = line_len + + buf.append(line) + buf.append(" " * (clear_width - line_len)) + line = "".join(buf) + # Render the line only if it changed. + + if line != self._last_line and not self.is_fast(): + self._last_line = line + echo(line, file=self.file, color=self.color, nl=False) + self.file.flush() + + def make_step(self, n_steps): + self.pos += n_steps + if self.length_known and self.pos >= self.length: + self.finished = True + + if (time.time() - self.last_eta) < 1.0: + return + + self.last_eta = time.time() + + # self.avg is a rolling list of length <= 7 of steps where steps are + # defined as time elapsed divided by the total progress through + # self.length. + if self.pos: + step = (time.time() - self.start) / self.pos + else: + step = time.time() - self.start + + self.avg = self.avg[-6:] + [step] + + self.eta_known = self.length_known + + def update(self, n_steps, current_item=None): + """Update the progress bar by advancing a specified number of + steps, and optionally set the ``current_item`` for this new + position. + + :param n_steps: Number of steps to advance. + :param current_item: Optional item to set as ``current_item`` + for the updated position. + + .. versionadded:: 8.0 + Added the ``current_item`` optional parameter. + """ + self.make_step(n_steps) + if current_item is not None: + self.current_item = current_item + self.render_progress() + + def finish(self): + self.eta_known = 0 + self.current_item = None + self.finished = True + + def generator(self): + """Return a generator which yields the items added to the bar + during construction, and updates the progress bar *after* the + yielded block returns. + """ + # WARNING: the iterator interface for `ProgressBar` relies on + # this and only works because this is a simple generator which + # doesn't create or manage additional state. If this function + # changes, the impact should be evaluated both against + # `iter(bar)` and `next(bar)`. `next()` in particular may call + # `self.generator()` repeatedly, and this must remain safe in + # order for that interface to work. + if not self.entered: + raise RuntimeError("You need to use progress bars in a with block.") + + if self.is_hidden: + yield from self.iter + else: + for rv in self.iter: + self.current_item = rv + yield rv + self.update(1) + self.finish() + self.render_progress() + + +def pager(generator, color=None): + """Decide what method to use for paging through text.""" + stdout = _default_text_stdout() + if not isatty(sys.stdin) or not isatty(stdout): + return _nullpager(stdout, generator, color) + pager_cmd = (os.environ.get("PAGER", None) or "").strip() + if pager_cmd: + if WIN: + return _tempfilepager(generator, pager_cmd, color) + return _pipepager(generator, pager_cmd, color) + if os.environ.get("TERM") in ("dumb", "emacs"): + return _nullpager(stdout, generator, color) + if WIN or sys.platform.startswith("os2"): + return _tempfilepager(generator, "more <", color) + if hasattr(os, "system") and os.system("(less) 2>/dev/null") == 0: + return _pipepager(generator, "less", color) + + import tempfile + + fd, filename = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, "system") and os.system(f'more "{filename}"') == 0: + return _pipepager(generator, "more", color) + return _nullpager(stdout, generator, color) + finally: + os.unlink(filename) + + +def _pipepager(generator, cmd, color): + """Page through text by feeding it to another program. Invoking a + pager through this might support colors. + """ + import subprocess + + env = dict(os.environ) + + # If we're piping to less we might support colors under the + # condition that + cmd_detail = cmd.rsplit("/", 1)[-1].split() + if color is None and cmd_detail[0] == "less": + less_flags = f"{os.environ.get('LESS', '')}{' '.join(cmd_detail[1:])}" + if not less_flags: + env["LESS"] = "-R" + color = True + elif "r" in less_flags or "R" in less_flags: + color = True + + c = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, env=env) + encoding = get_best_encoding(c.stdin) + try: + for text in generator: + if not color: + text = strip_ansi(text) + + c.stdin.write(text.encode(encoding, "replace")) + except (OSError, KeyboardInterrupt): + pass + else: + c.stdin.close() + + # Less doesn't respect ^C, but catches it for its own UI purposes (aborting + # search or other commands inside less). + # + # That means when the user hits ^C, the parent process (click) terminates, + # but less is still alive, paging the output and messing up the terminal. + # + # If the user wants to make the pager exit on ^C, they should set + # `LESS='-K'`. It's not our decision to make. + while True: + try: + c.wait() + except KeyboardInterrupt: + pass + else: + break + + +def _tempfilepager(generator, cmd, color): + """Page through text by invoking a program on a temporary file.""" + import tempfile + + filename = tempfile.mktemp() + # TODO: This never terminates if the passed generator never terminates. + text = "".join(generator) + if not color: + text = strip_ansi(text) + encoding = get_best_encoding(sys.stdout) + with open_stream(filename, "wb")[0] as f: + f.write(text.encode(encoding)) + try: + os.system(f'{cmd} "{filename}"') + finally: + os.unlink(filename) + + +def _nullpager(stream, generator, color): + """Simply print unformatted text. This is the ultimate fallback.""" + for text in generator: + if not color: + text = strip_ansi(text) + stream.write(text) + + class Editor: - def __init__(A,editor=_A,env=_A,require_save=_B,extension='.txt'):A.editor=editor;A.env=env;A.require_save=require_save;A.extension=extension - def get_editor(A): - if A.editor is not _A:return A.editor - for D in ('VISUAL','EDITOR'): - B=os.environ.get(D) - if B:return B - if WIN:return'notepad' - for C in ('sensible-editor','vim','nano'): - if os.system(f"which {C} >/dev/null 2>&1")==0:return C - return'vi' - def edit_file(A,filename): - import subprocess as D;B=A.get_editor() - if A.env:C=os.environ.copy();C.update(A.env) - else:C=_A - try: - E=D.Popen(f'{B} "{filename}"',env=C,shell=_B);F=E.wait() - if F!=0:raise ClickException(f"{B}: Editing failed!") - except OSError as G:raise ClickException(f"{B}: Editing failed: {G}") - def edit(D,text): - L='\r\n';K='utf-8-sig';A=text;import tempfile as H;A=A or'';E=type(A)in[bytes,bytearray] - if not E and A and not A.endswith(_D):A+=_D - I,B=H.mkstemp(prefix='editor-',suffix=D.extension) - try: - if not E: - if WIN:F=K;A=A.replace(_D,L) - else:F='utf-8' - A=A.encode(F) - C=os.fdopen(I,'wb');C.write(A);C.close();J=os.path.getmtime(B);D.edit_file(B) - if D.require_save and os.path.getmtime(B)==J:return _A - C=open(B,'rb') - try:G=C.read() - finally:C.close() - if E:return G - else:return G.decode(K).replace(L,_D) - finally:os.unlink(B) -def open_url(url,wait=_C,locate=_C): - F='"';D=locate;C=wait;A=url;import subprocess as G - def E(url): - A=url;import urllib as B - if A.startswith('file://'):A=B.unquote(A[7:]) - return A - if sys.platform=='darwin': - B=['open'] - if C:B.append('-W') - if D:B.append('-R') - B.append(E(A));H=open('/dev/null','w') - try:return G.Popen(B,stderr=H).wait() - finally:H.close() - elif WIN: - if D:A=E(A.replace(F,''));B=f'explorer /select,"{A}"' - else:A=A.replace(F,'');C='/WAIT'if C else'';B=f'start {C} "" "{A}"' - return os.system(B) - elif CYGWIN: - if D:A=os.path.dirname(E(A).replace(F,''));B=f'cygstart "{A}"' - else:A=A.replace(F,'');C='-w'if C else'';B=f'cygstart {C} "{A}"' - return os.system(B) - try: - if D:A=os.path.dirname(E(A))or'.' - else:A=E(A) - I=G.Popen(['xdg-open',A]) - if C:return I.wait() - return 0 - except OSError: - if A.startswith(('http://','https://'))and not D and not C:import webbrowser as J;J.open(A);return 0 - return 1 + def __init__(self, editor=None, env=None, require_save=True, extension=".txt"): + self.editor = editor + self.env = env + self.require_save = require_save + self.extension = extension + + def get_editor(self): + if self.editor is not None: + return self.editor + for key in "VISUAL", "EDITOR": + rv = os.environ.get(key) + if rv: + return rv + if WIN: + return "notepad" + for editor in "sensible-editor", "vim", "nano": + if os.system(f"which {editor} >/dev/null 2>&1") == 0: + return editor + return "vi" + + def edit_file(self, filename): + import subprocess + + editor = self.get_editor() + if self.env: + environ = os.environ.copy() + environ.update(self.env) + else: + environ = None + try: + c = subprocess.Popen(f'{editor} "{filename}"', env=environ, shell=True) + exit_code = c.wait() + if exit_code != 0: + raise ClickException(f"{editor}: Editing failed!") + except OSError as e: + raise ClickException(f"{editor}: Editing failed: {e}") + + def edit(self, text): + import tempfile + + text = text or "" + binary_data = type(text) in [bytes, bytearray] + + if not binary_data and text and not text.endswith("\n"): + text += "\n" + + fd, name = tempfile.mkstemp(prefix="editor-", suffix=self.extension) + try: + if not binary_data: + if WIN: + encoding = "utf-8-sig" + text = text.replace("\n", "\r\n") + else: + encoding = "utf-8" + text = text.encode(encoding) + + f = os.fdopen(fd, "wb") + f.write(text) + f.close() + timestamp = os.path.getmtime(name) + + self.edit_file(name) + + if self.require_save and os.path.getmtime(name) == timestamp: + return None + + f = open(name, "rb") + try: + rv = f.read() + finally: + f.close() + if binary_data: + return rv + else: + return rv.decode("utf-8-sig").replace("\r\n", "\n") + finally: + os.unlink(name) + + +def open_url(url, wait=False, locate=False): + import subprocess + + def _unquote_file(url): + import urllib + + if url.startswith("file://"): + url = urllib.unquote(url[7:]) + return url + + if sys.platform == "darwin": + args = ["open"] + if wait: + args.append("-W") + if locate: + args.append("-R") + args.append(_unquote_file(url)) + null = open("/dev/null", "w") + try: + return subprocess.Popen(args, stderr=null).wait() + finally: + null.close() + elif WIN: + if locate: + url = _unquote_file(url.replace('"', "")) + args = f'explorer /select,"{url}"' + else: + url = url.replace('"', "") + wait = "/WAIT" if wait else "" + args = f'start {wait} "" "{url}"' + return os.system(args) + elif CYGWIN: + if locate: + url = os.path.dirname(_unquote_file(url).replace('"', "")) + args = f'cygstart "{url}"' + else: + url = url.replace('"', "") + wait = "-w" if wait else "" + args = f'cygstart {wait} "{url}"' + return os.system(args) + + try: + if locate: + url = os.path.dirname(_unquote_file(url)) or "." + else: + url = _unquote_file(url) + c = subprocess.Popen(["xdg-open", url]) + if wait: + return c.wait() + return 0 + except OSError: + if url.startswith(("http://", "https://")) and not locate and not wait: + import webbrowser + + webbrowser.open(url) + return 0 + return 1 + + def _translate_ch_to_exc(ch): - if ch=='\x03':raise KeyboardInterrupt() - if ch=='\x04'and not WIN:raise EOFError() - if ch=='\x1a'and WIN:raise EOFError() + if ch == "\x03": + raise KeyboardInterrupt() + if ch == "\x04" and not WIN: # Unix-like, Ctrl+D + raise EOFError() + if ch == "\x1a" and WIN: # Windows, Ctrl+Z + raise EOFError() + + if WIN: - import msvcrt - @contextlib.contextmanager - def raw_terminal():yield - def getchar(echo): - if echo:B=msvcrt.getwche - else:B=msvcrt.getwch - A=B() - if A in('\x00','à'):A+=B() - _translate_ch_to_exc(A);return A + import msvcrt + + @contextlib.contextmanager + def raw_terminal(): + yield + + def getchar(echo): + # The function `getch` will return a bytes object corresponding to + # the pressed character. Since Windows 10 build 1803, it will also + # return \x00 when called a second time after pressing a regular key. + # + # `getwch` does not share this probably-bugged behavior. Moreover, it + # returns a Unicode object by default, which is what we want. + # + # Either of these functions will return \x00 or \xe0 to indicate + # a special key, and you need to call the same function again to get + # the "rest" of the code. The fun part is that \u00e0 is + # "latin small letter a with grave", so if you type that on a French + # keyboard, you _also_ get a \xe0. + # E.g., consider the Up arrow. This returns \xe0 and then \x48. The + # resulting Unicode string reads as "a with grave" + "capital H". + # This is indistinguishable from when the user actually types + # "a with grave" and then "capital H". + # + # When \xe0 is returned, we assume it's part of a special-key sequence + # and call `getwch` again, but that means that when the user types + # the \u00e0 character, `getchar` doesn't return until a second + # character is typed. + # The alternative is returning immediately, but that would mess up + # cross-platform handling of arrow keys and others that start with + # \xe0. Another option is using `getch`, but then we can't reliably + # read non-ASCII characters, because return values of `getch` are + # limited to the current 8-bit codepage. + # + # Anyway, Click doesn't claim to do this Right(tm), and using `getwch` + # is doing the right thing in more situations than with `getch`. + if echo: + func = msvcrt.getwche + else: + func = msvcrt.getwch + + rv = func() + if rv in ("\x00", "\xe0"): + # \x00 and \xe0 are control characters that indicate special key, + # see above. + rv += func() + _translate_ch_to_exc(rv) + return rv + + else: - import tty,termios - @contextlib.contextmanager - def raw_terminal(): - if not isatty(sys.stdin):B=open('/dev/tty');A=B.fileno() - else:A=sys.stdin.fileno();B=_A - try: - C=termios.tcgetattr(A) - try:tty.setraw(A);yield A - finally: - termios.tcsetattr(A,termios.TCSADRAIN,C);sys.stdout.flush() - if B is not _A:B.close() - except termios.error:pass - def getchar(echo): - with raw_terminal()as B: - A=os.read(B,32);A=A.decode(get_best_encoding(sys.stdin),_H) - if echo and isatty(sys.stdout):sys.stdout.write(A) - _translate_ch_to_exc(A);return A \ No newline at end of file + import tty + import termios + + @contextlib.contextmanager + def raw_terminal(): + if not isatty(sys.stdin): + f = open("/dev/tty") + fd = f.fileno() + else: + fd = sys.stdin.fileno() + f = None + try: + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(fd) + yield fd + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + sys.stdout.flush() + if f is not None: + f.close() + except termios.error: + pass + + def getchar(echo): + with raw_terminal() as fd: + ch = os.read(fd, 32) + ch = ch.decode(get_best_encoding(sys.stdin), "replace") + if echo and isatty(sys.stdout): + sys.stdout.write(ch) + _translate_ch_to_exc(ch) + return ch diff --git a/dynaconf/vendor/click/_textwrap.py b/dynaconf/vendor/click/_textwrap.py index b02fced8d..7a052b70d 100644 --- a/dynaconf/vendor/click/_textwrap.py +++ b/dynaconf/vendor/click/_textwrap.py @@ -1,19 +1,37 @@ import textwrap from contextlib import contextmanager + + class TextWrapper(textwrap.TextWrapper): - def _handle_long_word(E,reversed_chunks,cur_line,cur_len,width): - B=cur_line;A=reversed_chunks;C=max(width-cur_len,1) - if E.break_long_words:D=A[-1];F=D[:C];G=D[C:];B.append(F);A[-1]=G - elif not B:B.append(A.pop()) - @contextmanager - def extra_indent(self,indent): - B=indent;A=self;C=A.initial_indent;D=A.subsequent_indent;A.initial_indent+=B;A.subsequent_indent+=B - try:yield - finally:A.initial_indent=C;A.subsequent_indent=D - def indent_only(A,text): - B=[] - for (D,E) in enumerate(text.splitlines()): - C=A.initial_indent - if D>0:C=A.subsequent_indent - B.append(f"{C}{E}") - return '\n'.join(B) \ No newline at end of file + def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): + space_left = max(width - cur_len, 1) + + if self.break_long_words: + last = reversed_chunks[-1] + cut = last[:space_left] + res = last[space_left:] + cur_line.append(cut) + reversed_chunks[-1] = res + elif not cur_line: + cur_line.append(reversed_chunks.pop()) + + @contextmanager + def extra_indent(self, indent): + old_initial_indent = self.initial_indent + old_subsequent_indent = self.subsequent_indent + self.initial_indent += indent + self.subsequent_indent += indent + try: + yield + finally: + self.initial_indent = old_initial_indent + self.subsequent_indent = old_subsequent_indent + + def indent_only(self, text): + rv = [] + for idx, line in enumerate(text.splitlines()): + indent = self.initial_indent + if idx > 0: + indent = self.subsequent_indent + rv.append(f"{indent}{line}") + return "\n".join(rv) diff --git a/dynaconf/vendor/click/_unicodefun.py b/dynaconf/vendor/click/_unicodefun.py index 792053f31..53ec9d267 100644 --- a/dynaconf/vendor/click/_unicodefun.py +++ b/dynaconf/vendor/click/_unicodefun.py @@ -1,28 +1,82 @@ -import codecs,os +import codecs +import os + + def _verify_python_env(): - M='.utf8';L='.utf-8';J=None;I='ascii' - try:import locale as A;G=codecs.lookup(A.getpreferredencoding()).name - except Exception:G=I - if G!=I:return - B='' - if os.name=='posix': - import subprocess as D - try:C=D.Popen(['locale','-a'],stdout=D.PIPE,stderr=D.PIPE).communicate()[0] - except OSError:C=b'' - E=set();H=False - if isinstance(C,bytes):C=C.decode(I,'replace') - for K in C.splitlines(): - A=K.strip() - if A.lower().endswith((L,M)): - E.add(A) - if A.lower()in('c.utf8','c.utf-8'):H=True - B+='\n\n' - if not E:B+='Additional information: on this system no suitable UTF-8 locales were discovered. This most likely requires resolving by reconfiguring the locale system.' - elif H:B+='This system supports the C.UTF-8 locale which is recommended. You might be able to resolve your issue by exporting the following environment variables:\n\n export LC_ALL=C.UTF-8\n export LANG=C.UTF-8' - else:B+=f"This system lists some UTF-8 supporting locales that you can pick from. The following suitable locales were discovered: {', '.join(sorted(E))}" - F=J - for A in (os.environ.get('LC_ALL'),os.environ.get('LANG')): - if A and A.lower().endswith((L,M)):F=A - if A is not J:break - if F is not J:B+=f"\n\nClick discovered that you exported a UTF-8 locale but the locale system could not pick up from it because it does not exist. The exported locale is {F!r} but it is not supported" - raise RuntimeError(f"Click will abort further execution because Python was configured to use ASCII as encoding for the environment. Consult https://click.palletsprojects.com/unicode-support/ for mitigation steps.{B}") \ No newline at end of file + """Ensures that the environment is good for Unicode.""" + try: + import locale + + fs_enc = codecs.lookup(locale.getpreferredencoding()).name + except Exception: + fs_enc = "ascii" + if fs_enc != "ascii": + return + + extra = "" + if os.name == "posix": + import subprocess + + try: + rv = subprocess.Popen( + ["locale", "-a"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ).communicate()[0] + except OSError: + rv = b"" + good_locales = set() + has_c_utf8 = False + + # Make sure we're operating on text here. + if isinstance(rv, bytes): + rv = rv.decode("ascii", "replace") + + for line in rv.splitlines(): + locale = line.strip() + if locale.lower().endswith((".utf-8", ".utf8")): + good_locales.add(locale) + if locale.lower() in ("c.utf8", "c.utf-8"): + has_c_utf8 = True + + extra += "\n\n" + if not good_locales: + extra += ( + "Additional information: on this system no suitable" + " UTF-8 locales were discovered. This most likely" + " requires resolving by reconfiguring the locale" + " system." + ) + elif has_c_utf8: + extra += ( + "This system supports the C.UTF-8 locale which is" + " recommended. You might be able to resolve your issue" + " by exporting the following environment variables:\n\n" + " export LC_ALL=C.UTF-8\n" + " export LANG=C.UTF-8" + ) + else: + extra += ( + "This system lists some UTF-8 supporting locales that" + " you can pick from. The following suitable locales" + f" were discovered: {', '.join(sorted(good_locales))}" + ) + + bad_locale = None + for locale in os.environ.get("LC_ALL"), os.environ.get("LANG"): + if locale and locale.lower().endswith((".utf-8", ".utf8")): + bad_locale = locale + if locale is not None: + break + if bad_locale is not None: + extra += ( + "\n\nClick discovered that you exported a UTF-8 locale" + " but the locale system could not pick up from it" + " because it does not exist. The exported locale is" + f" {bad_locale!r} but it is not supported" + ) + + raise RuntimeError( + "Click will abort further execution because Python was" + " configured to use ASCII as encoding for the environment." + " Consult https://click.palletsprojects.com/unicode-support/" + f" for mitigation steps.{extra}" + ) diff --git a/dynaconf/vendor/click/_winconsole.py b/dynaconf/vendor/click/_winconsole.py index 316b2525b..923fdba65 100644 --- a/dynaconf/vendor/click/_winconsole.py +++ b/dynaconf/vendor/click/_winconsole.py @@ -1,108 +1,308 @@ -_E=False -_D='strict' -_C='utf-16-le' -_B=True -_A=None -import ctypes,io,time -from ctypes import byref,c_char,c_char_p,c_int,c_ssize_t,c_ulong,c_void_p,POINTER,py_object,windll,WINFUNCTYPE +# This module is based on the excellent work by Adam Bartoš who +# provided a lot of what went into the implementation here in +# the discussion to issue1602 in the Python bug tracker. +# +# There are some general differences in regards to how this works +# compared to the original patches as we do not need to patch +# the entire interpreter but just work in our little world of +# echo and prompt. +import ctypes +import io +import time +from ctypes import byref +from ctypes import c_char +from ctypes import c_char_p +from ctypes import c_int +from ctypes import c_ssize_t +from ctypes import c_ulong +from ctypes import c_void_p +from ctypes import POINTER +from ctypes import py_object +from ctypes import windll +from ctypes import WINFUNCTYPE from ctypes.wintypes import DWORD from ctypes.wintypes import HANDLE from ctypes.wintypes import LPCWSTR from ctypes.wintypes import LPWSTR + import msvcrt + from ._compat import _NonClosingTextIOWrapper -try:from ctypes import pythonapi -except ImportError:pythonapi=_A -else:PyObject_GetBuffer=pythonapi.PyObject_GetBuffer;PyBuffer_Release=pythonapi.PyBuffer_Release -c_ssize_p=POINTER(c_ssize_t) -kernel32=windll.kernel32 -GetStdHandle=kernel32.GetStdHandle -ReadConsoleW=kernel32.ReadConsoleW -WriteConsoleW=kernel32.WriteConsoleW -GetConsoleMode=kernel32.GetConsoleMode -GetLastError=kernel32.GetLastError -GetCommandLineW=WINFUNCTYPE(LPWSTR)(('GetCommandLineW',windll.kernel32)) -CommandLineToArgvW=WINFUNCTYPE(POINTER(LPWSTR),LPCWSTR,POINTER(c_int))(('CommandLineToArgvW',windll.shell32)) -LocalFree=WINFUNCTYPE(ctypes.c_void_p,ctypes.c_void_p)(('LocalFree',windll.kernel32)) -STDIN_HANDLE=GetStdHandle(-10) -STDOUT_HANDLE=GetStdHandle(-11) -STDERR_HANDLE=GetStdHandle(-12) -PyBUF_SIMPLE=0 -PyBUF_WRITABLE=1 -ERROR_SUCCESS=0 -ERROR_NOT_ENOUGH_MEMORY=8 -ERROR_OPERATION_ABORTED=995 -STDIN_FILENO=0 -STDOUT_FILENO=1 -STDERR_FILENO=2 -EOF=b'\x1a' -MAX_BYTES_WRITTEN=32767 -class Py_buffer(ctypes.Structure):_fields_=[('buf',c_void_p),('obj',py_object),('len',c_ssize_t),('itemsize',c_ssize_t),('readonly',c_int),('ndim',c_int),('format',c_char_p),('shape',c_ssize_p),('strides',c_ssize_p),('suboffsets',c_ssize_p),('internal',c_void_p)] -if pythonapi is _A:get_buffer=_A + +try: + from ctypes import pythonapi +except ImportError: + pythonapi = None else: - def get_buffer(obj,writable=_E): - A=Py_buffer();B=PyBUF_WRITABLE if writable else PyBUF_SIMPLE;PyObject_GetBuffer(py_object(obj),byref(A),B) - try:C=c_char*A.len;return C.from_address(A.buf) - finally:PyBuffer_Release(byref(A)) + PyObject_GetBuffer = pythonapi.PyObject_GetBuffer + PyBuffer_Release = pythonapi.PyBuffer_Release + + +c_ssize_p = POINTER(c_ssize_t) + +kernel32 = windll.kernel32 +GetStdHandle = kernel32.GetStdHandle +ReadConsoleW = kernel32.ReadConsoleW +WriteConsoleW = kernel32.WriteConsoleW +GetConsoleMode = kernel32.GetConsoleMode +GetLastError = kernel32.GetLastError +GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) +CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( + ("CommandLineToArgvW", windll.shell32) +) +LocalFree = WINFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)( + ("LocalFree", windll.kernel32) +) + + +STDIN_HANDLE = GetStdHandle(-10) +STDOUT_HANDLE = GetStdHandle(-11) +STDERR_HANDLE = GetStdHandle(-12) + + +PyBUF_SIMPLE = 0 +PyBUF_WRITABLE = 1 + +ERROR_SUCCESS = 0 +ERROR_NOT_ENOUGH_MEMORY = 8 +ERROR_OPERATION_ABORTED = 995 + +STDIN_FILENO = 0 +STDOUT_FILENO = 1 +STDERR_FILENO = 2 + +EOF = b"\x1a" +MAX_BYTES_WRITTEN = 32767 + + +class Py_buffer(ctypes.Structure): + _fields_ = [ + ("buf", c_void_p), + ("obj", py_object), + ("len", c_ssize_t), + ("itemsize", c_ssize_t), + ("readonly", c_int), + ("ndim", c_int), + ("format", c_char_p), + ("shape", c_ssize_p), + ("strides", c_ssize_p), + ("suboffsets", c_ssize_p), + ("internal", c_void_p), + ] + + +# On PyPy we cannot get buffers so our ability to operate here is +# severely limited. +if pythonapi is None: + get_buffer = None +else: + + def get_buffer(obj, writable=False): + buf = Py_buffer() + flags = PyBUF_WRITABLE if writable else PyBUF_SIMPLE + PyObject_GetBuffer(py_object(obj), byref(buf), flags) + try: + buffer_type = c_char * buf.len + return buffer_type.from_address(buf.buf) + finally: + PyBuffer_Release(byref(buf)) + + class _WindowsConsoleRawIOBase(io.RawIOBase): - def __init__(A,handle):A.handle=handle - def isatty(A):io.RawIOBase.isatty(A);return _B + def __init__(self, handle): + self.handle = handle + + def isatty(self): + io.RawIOBase.isatty(self) + return True + + class _WindowsConsoleReader(_WindowsConsoleRawIOBase): - def readable(A):return _B - def readinto(D,b): - A=len(b) - if not A:return 0 - elif A%2:raise ValueError('cannot read odd number of bytes from UTF-16-LE encoded console') - B=get_buffer(b,writable=_B);E=A//2;C=c_ulong();F=ReadConsoleW(HANDLE(D.handle),B,E,byref(C),_A) - if GetLastError()==ERROR_OPERATION_ABORTED:time.sleep(0.1) - if not F:raise OSError(f"Windows error: {GetLastError()}") - if B[0]==EOF:return 0 - return 2*C.value + def readable(self): + return True + + def readinto(self, b): + bytes_to_be_read = len(b) + if not bytes_to_be_read: + return 0 + elif bytes_to_be_read % 2: + raise ValueError( + "cannot read odd number of bytes from UTF-16-LE encoded console" + ) + + buffer = get_buffer(b, writable=True) + code_units_to_be_read = bytes_to_be_read // 2 + code_units_read = c_ulong() + + rv = ReadConsoleW( + HANDLE(self.handle), + buffer, + code_units_to_be_read, + byref(code_units_read), + None, + ) + if GetLastError() == ERROR_OPERATION_ABORTED: + # wait for KeyboardInterrupt + time.sleep(0.1) + if not rv: + raise OSError(f"Windows error: {GetLastError()}") + + if buffer[0] == EOF: + return 0 + return 2 * code_units_read.value + + class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): - def writable(A):return _B - @staticmethod - def _get_error_message(errno): - A=errno - if A==ERROR_SUCCESS:return'ERROR_SUCCESS' - elif A==ERROR_NOT_ENOUGH_MEMORY:return'ERROR_NOT_ENOUGH_MEMORY' - return f"Windows error {A}" - def write(A,b): - B=len(b);E=get_buffer(b);F=min(B,MAX_BYTES_WRITTEN)//2;C=c_ulong();WriteConsoleW(HANDLE(A.handle),E,F,byref(C),_A);D=2*C.value - if D==0 and B>0:raise OSError(A._get_error_message(GetLastError())) - return D + def writable(self): + return True + + @staticmethod + def _get_error_message(errno): + if errno == ERROR_SUCCESS: + return "ERROR_SUCCESS" + elif errno == ERROR_NOT_ENOUGH_MEMORY: + return "ERROR_NOT_ENOUGH_MEMORY" + return f"Windows error {errno}" + + def write(self, b): + bytes_to_be_written = len(b) + buf = get_buffer(b) + code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2 + code_units_written = c_ulong() + + WriteConsoleW( + HANDLE(self.handle), + buf, + code_units_to_be_written, + byref(code_units_written), + None, + ) + bytes_written = 2 * code_units_written.value + + if bytes_written == 0 and bytes_to_be_written > 0: + raise OSError(self._get_error_message(GetLastError())) + return bytes_written + + class ConsoleStream: - def __init__(A,text_stream,byte_stream):A._text_stream=text_stream;A.buffer=byte_stream - @property - def name(self):return self.buffer.name - def write(A,x): - if isinstance(x,str):return A._text_stream.write(x) - try:A.flush() - except Exception:pass - return A.buffer.write(x) - def writelines(A,lines): - for B in lines:A.write(B) - def __getattr__(A,name):return getattr(A._text_stream,name) - def isatty(A):return A.buffer.isatty() - def __repr__(A):return f"" + def __init__(self, text_stream, byte_stream): + self._text_stream = text_stream + self.buffer = byte_stream + + @property + def name(self): + return self.buffer.name + + def write(self, x): + if isinstance(x, str): + return self._text_stream.write(x) + try: + self.flush() + except Exception: + pass + return self.buffer.write(x) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def __getattr__(self, name): + return getattr(self._text_stream, name) + + def isatty(self): + return self.buffer.isatty() + + def __repr__(self): + return f"" + + class WindowsChunkedWriter: - def __init__(A,wrapped):A.__wrapped=wrapped - def __getattr__(A,name):return getattr(A.__wrapped,name) - def write(D,text): - B=len(text);A=0 - while A