From 317f7e0fc83373d6eb4a50410c9e756f2209b610 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 5 Mar 2021 20:30:21 +0000 Subject: [PATCH 01/20] docs: minor formatting fix --- tqdm/dask.py | 2 +- tqdm/keras.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tqdm/dask.py b/tqdm/dask.py index 730a5e647..6fc7504c7 100644 --- a/tqdm/dask.py +++ b/tqdm/dask.py @@ -17,7 +17,7 @@ def __init__(self, start=None, pretask=None, tqdm_class=tqdm_auto, """ Parameters ---------- - tqdm_class : optional + tqdm_class : optional `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. tqdm_kwargs : optional Any other arguments used for all bars. diff --git a/tqdm/keras.py b/tqdm/keras.py index 4a808f15f..45caf6189 100644 --- a/tqdm/keras.py +++ b/tqdm/keras.py @@ -45,7 +45,7 @@ def __init__(self, epochs=None, data_size=None, batch_size=None, verbose=1, 0: epoch, 1: batch (transient), 2: batch. [default: 1]. Will be set to `0` unless both `data_size` and `batch_size` are given. - tqdm_class : optional + tqdm_class : optional `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. tqdm_kwargs : optional Any other arguments used for all bars. From 8243ba52c99f8db0bf95c7212daef31cdc156ec0 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 8 Mar 2021 22:26:04 +0000 Subject: [PATCH 02/20] tests: notebook: initial nbval config --- .meta/requirements-test.txt | 2 ++ Makefile | 5 +++ environment.yml | 2 ++ setup.cfg | 2 +- tests/tests_notebook.ipynb | 61 +++++++++++++++++++++++++++++++++++++ tox.ini | 4 +++ 6 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 tests/tests_notebook.ipynb diff --git a/.meta/requirements-test.txt b/.meta/requirements-test.txt index c34235b1b..05ac6d3fa 100644 --- a/.meta/requirements-test.txt +++ b/.meta/requirements-test.txt @@ -2,4 +2,6 @@ flake8 pytest pytest-cov pytest-timeout +nbval +ipywidgets # py>=37: pytest-asyncio diff --git a/Makefile b/Makefile index d0223128f..b01dea9f6 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ test pytest testsetup + testnb testcoverage testperf testtimer @@ -59,9 +60,13 @@ testsetup: python setup.py check --metadata --restructuredtext --strict python setup.py make none +testnb: + pytest --nbval --current-env -k ipynb -W=ignore + testcoverage: @make coverclean pytest -k "not perf" --cov=tqdm --cov-report=xml --cov-report=term --cov-fail-under=80 + pytest -k ipynb --cov-append --nbval --current-env -W=ignore --cov=tqdm --cov-report=xml --cov-report=term --cov-fail-under=80 testperf: # do not use coverage (which is extremely slow) diff --git a/environment.yml b/environment.yml index 2132fdc10..21e0d8a97 100644 --- a/environment.yml +++ b/environment.yml @@ -8,6 +8,7 @@ dependencies: - python=3 - pip - ipykernel +- ipywidgets - setuptools - setuptools_scm - toml @@ -20,6 +21,7 @@ dependencies: - pytest-cov - pytest-timeout - pytest-asyncio # [py>=3.7] +- nbval - flake8 - flake8-bugbear - flake8-comprehensions diff --git a/setup.cfg b/setup.cfg index a0365422c..7eb626244 100644 --- a/setup.cfg +++ b/setup.cfg @@ -119,7 +119,7 @@ log_level=INFO markers= asyncio slow -python_files=tests_*.py +python_files=tests_*.py tests_*.ipynb testpaths=tests addopts=-v --tb=short -rxs -W=error --durations=0 --durations-min=0.1 diff --git a/tests/tests_notebook.ipynb b/tests/tests_notebook.ipynb new file mode 100644 index 000000000..3df654748 --- /dev/null +++ b/tests/tests_notebook.ipynb @@ -0,0 +1,61 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm.notebook import tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function display in module tqdm.notebook:\n", + "\n", + "display(self, msg=None, pos=None, close=False, bar_style=None)\n", + " Use `self.sp` to display `msg` in the specified `pos`.\n", + " \n", + " Consider overloading this function when inheriting to use e.g.:\n", + " `self.some_frontend(**self.format_dict)` instead of `self.sp`.\n", + " \n", + " Parameters\n", + " ----------\n", + " msg : str, optional. What to display (default: `repr(self)`).\n", + " pos : int, optional. Position to `moveto`\n", + " (default: `abs(self.pos)`).\n", + "\n" + ] + } + ], + "source": [ + "help(tqdm.display)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:tqdm]", + "language": "python", + "name": "conda-env-tqdm-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython" + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tox.ini b/tox.ini index 3ecdfa0ac..e83705bf7 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,8 @@ deps= pytest-cov pytest-timeout py{37,38,39}: pytest-asyncio + ipywidgets + nbval coverage coveralls codecov @@ -36,12 +38,14 @@ deps= tf: tensorflow commands= pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" + pytest --cov=tqdm --cov-report=xml --cov-report=term -k ipynb --cov-append --nbval --current-env -W=ignore {[core]commands} allowlist_externals=codacy [testenv:py{27,py2}{,-tf}{,-keras}] commands= pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=10 + pytest --cov=tqdm --cov-report=xml --cov-report=term -k ipynb -o addopts= -v --tb=short -rxs -W=ignore --durations=10 --cov-append --nbval --current-env {[core]commands} # no cython/numpy/pandas From 9e001f657a55f367d9f65cec7f438b52a5d6a4b7 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 8 Mar 2021 22:54:39 +0000 Subject: [PATCH 03/20] fix coverage - related https://github.com/computationalmodelling/nbval/issues/107#issuecomment-463162174 --- Makefile | 6 +++--- tox.ini | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index b01dea9f6..8d79aa644 100644 --- a/Makefile +++ b/Makefile @@ -61,12 +61,12 @@ testsetup: python setup.py make none testnb: - pytest --nbval --current-env -k ipynb -W=ignore + pytest --nbval --current-env -k ipynb -W=ignore --sanitize-with=setup.cfg --cov=tqdm.notebook --cov-report=term testcoverage: @make coverclean - pytest -k "not perf" --cov=tqdm --cov-report=xml --cov-report=term --cov-fail-under=80 - pytest -k ipynb --cov-append --nbval --current-env -W=ignore --cov=tqdm --cov-report=xml --cov-report=term --cov-fail-under=80 + pytest -k ipynb --nbval --current-env --sanitize-with=setup.cfg -W=ignore --cov=tqdm --cov-report=xml --cov-report=term + pytest -k "not perf" --cov-append --cov=tqdm --cov-report=xml --cov-report=term --cov-fail-under=80 testperf: # do not use coverage (which is extremely slow) diff --git a/tox.ini b/tox.ini index e83705bf7..96e26b6db 100644 --- a/tox.ini +++ b/tox.ini @@ -37,15 +37,15 @@ deps= py{36,37,38,39}: rich tf: tensorflow commands= - pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" - pytest --cov=tqdm --cov-report=xml --cov-report=term -k ipynb --cov-append --nbval --current-env -W=ignore + pytest --cov=tqdm --cov-report=xml --cov-report=term -k ipynb --nbval --current-env -W=ignore --sanitize-with=setup.cfg + pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" --cov-append {[core]commands} allowlist_externals=codacy [testenv:py{27,py2}{,-tf}{,-keras}] commands= - pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=10 - pytest --cov=tqdm --cov-report=xml --cov-report=term -k ipynb -o addopts= -v --tb=short -rxs -W=ignore --durations=10 --cov-append --nbval --current-env + pytest --cov=tqdm --cov-report=xml --cov-report=term -k ipynb -o addopts= -v --tb=short -rxs -W=ignore --durations=10 --nbval --current-env --sanitize-with=setup.cfg + pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=10 --cov-append {[core]commands} # no cython/numpy/pandas From 5eb36df4f094c257656b3f941fe8b094a1bbfc46 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 8 Mar 2021 23:34:11 +0000 Subject: [PATCH 04/20] tests: notebook: fix paths - related https://github.com/computationalmodelling/nbval/issues/116#issuecomment-793148404 --- Makefile | 6 +++--- tests/tests_notebook.ipynb => tests_notebook.ipynb | 0 tox.ini | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) rename tests/tests_notebook.ipynb => tests_notebook.ipynb (100%) diff --git a/Makefile b/Makefile index 8d79aa644..10ee2e2ac 100644 --- a/Makefile +++ b/Makefile @@ -61,12 +61,12 @@ testsetup: python setup.py make none testnb: - pytest --nbval --current-env -k ipynb -W=ignore --sanitize-with=setup.cfg --cov=tqdm.notebook --cov-report=term + pytest tests_notebook.ipynb --nbval --current-env -W=ignore --sanitize-with=setup.cfg --cov=tqdm.notebook --cov-report=term testcoverage: @make coverclean - pytest -k ipynb --nbval --current-env --sanitize-with=setup.cfg -W=ignore --cov=tqdm --cov-report=xml --cov-report=term - pytest -k "not perf" --cov-append --cov=tqdm --cov-report=xml --cov-report=term --cov-fail-under=80 + pytest tests_notebook.ipynb --cov=tqdm --cov-report= --nbval --current-env --sanitize-with=setup.cfg -W=ignore + pytest -k "not perf" --cov=tqdm --cov-report=xml --cov-report=term --cov-append --cov-fail-under=80 testperf: # do not use coverage (which is extremely slow) diff --git a/tests/tests_notebook.ipynb b/tests_notebook.ipynb similarity index 100% rename from tests/tests_notebook.ipynb rename to tests_notebook.ipynb diff --git a/tox.ini b/tox.ini index 96e26b6db..badf64583 100644 --- a/tox.ini +++ b/tox.ini @@ -37,15 +37,15 @@ deps= py{36,37,38,39}: rich tf: tensorflow commands= - pytest --cov=tqdm --cov-report=xml --cov-report=term -k ipynb --nbval --current-env -W=ignore --sanitize-with=setup.cfg - pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" --cov-append + pytest --cov=tqdm --cov-report= tests_notebook.ipynb --nbval --current-env -W=ignore --sanitize-with=setup.cfg + pytest --cov=tqdm --cov-report=xml --cov-report=term --cov-append -k "not perf" {[core]commands} allowlist_externals=codacy [testenv:py{27,py2}{,-tf}{,-keras}] commands= - pytest --cov=tqdm --cov-report=xml --cov-report=term -k ipynb -o addopts= -v --tb=short -rxs -W=ignore --durations=10 --nbval --current-env --sanitize-with=setup.cfg - pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=10 --cov-append + pytest --cov=tqdm --cov-report= tests_notebook.ipynb -o addopts= -v --tb=short -rxs -W=ignore --durations=10 --nbval --current-env --sanitize-with=setup.cfg + pytest --cov=tqdm --cov-report=xml --cov-report=term --cov-append -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=10 {[core]commands} # no cython/numpy/pandas From 06698c5e9772c20b1ef35d3018560c48e43dc2d5 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 8 Mar 2021 23:53:50 +0000 Subject: [PATCH 05/20] tests: notebook: more tests --- setup.cfg | 3 + tests_notebook.ipynb | 203 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 204 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 7eb626244..17588b74f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -122,6 +122,9 @@ markers= python_files=tests_*.py tests_*.ipynb testpaths=tests addopts=-v --tb=short -rxs -W=error --durations=0 --durations-min=0.1 +[regex1] +regex: (?<= )[\s\d.]+(it/s|s/it) +replace: ??.??it/s [coverage:run] branch=True diff --git a/tests_notebook.ipynb b/tests_notebook.ipynb index 3df654748..64e0c706e 100644 --- a/tests_notebook.ipynb +++ b/tests_notebook.ipynb @@ -1,12 +1,29 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This file is part of the [test suite](./tests) and will be moved there when [nbval#116](https://github.com/computationalmodelling/nbval/issues/116#issuecomment-793148404) is fixed.\n", + "\n", + "See [DEMO.ipynb](DEMO.ipynb) instead for notebook examples." + ] + }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "from tqdm.notebook import tqdm" + "from functools import partial\n", + "from time import sleep\n", + "\n", + "from tqdm.notebook import tqdm_notebook\n", + "from tqdm.notebook import tnrange\n", + "\n", + "# avoid displaying widgets by default (pollutes output cells)\n", + "tqdm = partial(tqdm_notebook, display=False)\n", + "trange = partial(tnrange, display=False)" ] }, { @@ -36,7 +53,189 @@ } ], "source": [ - "help(tqdm.display)" + "help(tqdm_notebook.display)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# basic use" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7c18c038bf964b55941e228503292506", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/10 [00:00 Date: Mon, 8 Mar 2021 23:54:13 +0000 Subject: [PATCH 06/20] add nbstripout pre-commit --- .pre-commit-config.yaml | 6 ++++++ DEMO.ipynb | 7 ++----- environment.yml | 1 + 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aead6873d..3bcea8401 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,3 +52,9 @@ repos: rev: 5.7.0 hooks: - id: isort +repos: +- repo: https://github.com/kynan/nbstripout + rev: master + hooks: + - id: nbstripout + args: ['--keep-count', '--keep-output'] diff --git a/DEMO.ipynb b/DEMO.ipynb index feaab9389..82670daf2 100644 --- a/DEMO.ipynb +++ b/DEMO.ipynb @@ -1714,15 +1714,12 @@ }, "language_info": { "codemirror_mode": { - "name": "ipython", - "version": 3 + "name": "ipython" }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.8" + "nbconvert_exporter": "python" } }, "nbformat": 4, diff --git a/environment.yml b/environment.yml index 21e0d8a97..9b199cf21 100644 --- a/environment.yml +++ b/environment.yml @@ -29,6 +29,7 @@ dependencies: # extras - dask # dask - matplotlib # gui +- nbstripout # notebook editing - numpy # pandas, keras, contrib.tenumerate - pandas - tensorflow # keras From aa2e430865f385e152b9222e1c46569e80a4e7a1 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 9 Mar 2021 00:30:18 +0000 Subject: [PATCH 07/20] tests: skip py27 notebook --- tox.ini | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index badf64583..ea825bbee 100644 --- a/tox.ini +++ b/tox.ini @@ -14,8 +14,8 @@ deps= pytest-cov pytest-timeout py{37,38,39}: pytest-asyncio - ipywidgets - nbval + !py{27,py2}: ipywidgets + !py{27,py2}: nbval coverage coveralls codecov @@ -37,15 +37,14 @@ deps= py{36,37,38,39}: rich tf: tensorflow commands= - pytest --cov=tqdm --cov-report= tests_notebook.ipynb --nbval --current-env -W=ignore --sanitize-with=setup.cfg + !py{27,py2}: pytest --cov=tqdm --cov-report= tests_notebook.ipynb --nbval --current-env -W=ignore --sanitize-with=setup.cfg pytest --cov=tqdm --cov-report=xml --cov-report=term --cov-append -k "not perf" {[core]commands} allowlist_externals=codacy [testenv:py{27,py2}{,-tf}{,-keras}] commands= - pytest --cov=tqdm --cov-report= tests_notebook.ipynb -o addopts= -v --tb=short -rxs -W=ignore --durations=10 --nbval --current-env --sanitize-with=setup.cfg - pytest --cov=tqdm --cov-report=xml --cov-report=term --cov-append -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=10 + pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=10 {[core]commands} # no cython/numpy/pandas From 1ddb6745c5448ab0f741f56ed1f35ad8cb6434a1 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 9 Mar 2021 00:53:19 +0000 Subject: [PATCH 08/20] tests: notebook: ncols --- tests_notebook.ipynb | 104 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 91 insertions(+), 13 deletions(-) diff --git a/tests_notebook.ipynb b/tests_notebook.ipynb index 64e0c706e..1679b8686 100644 --- a/tests_notebook.ipynb +++ b/tests_notebook.ipynb @@ -16,7 +16,6 @@ "outputs": [], "source": [ "from functools import partial\n", - "from time import sleep\n", "\n", "from tqdm.notebook import tqdm_notebook\n", "from tqdm.notebook import tnrange\n", @@ -97,20 +96,69 @@ "8\n", "9\n" ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e29668be41ca4e40b16fb98572b333a5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/10 [00:00 Date: Tue, 9 Mar 2021 01:00:52 +0000 Subject: [PATCH 09/20] tests: notebook: allow 1sec slow tests --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 17588b74f..7c733eb5b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -125,6 +125,9 @@ addopts=-v --tb=short -rxs -W=error --durations=0 --durations-min=0.1 [regex1] regex: (?<= )[\s\d.]+(it/s|s/it) replace: ??.??it/s +[regex2] +regex: 00:0[01]<00:0[01] +replace: 00:00<00:00 [coverage:run] branch=True From 7a95276b83aaecc45bf2ef9d137806ae8656ee79 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 9 Mar 2021 15:10:25 +0000 Subject: [PATCH 10/20] tests: notebook: more coverage, better verbosity --- tests_notebook.ipynb | 268 ++++++++++++++++++++++++++++++++----------- tox.ini | 2 +- 2 files changed, 203 insertions(+), 67 deletions(-) diff --git a/tests_notebook.ipynb b/tests_notebook.ipynb index 1679b8686..a13f3a23f 100644 --- a/tests_notebook.ipynb +++ b/tests_notebook.ipynb @@ -55,13 +55,6 @@ "help(tqdm_notebook.display)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# basic use" - ] - }, { "cell_type": "code", "execution_count": 3, @@ -75,7 +68,7 @@ "version_minor": 0 }, "text/plain": [ - " 0%| | 0/10 [00:00 Date: Tue, 9 Mar 2021 15:18:53 +0000 Subject: [PATCH 11/20] tests: more py2 deprecation --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 8e525063c..4d83144cb 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -# deprecation warning: py{27,py2,34,35} +# deprecation warning: py{27,py2,34,35,36} envlist=py{27,34,35,36,37,38,39,py2,py3}{,-tf}{,-keras}, perf, setup.py isolated_build=True @@ -13,9 +13,9 @@ deps= pytest pytest-cov pytest-timeout - py{37,38,39}: pytest-asyncio - !py{27,py2}: ipywidgets - !py{27,py2}: git+https://github.com/casperdcl/nbval.git@named_cells#egg=nbval + py3{7,8,9}: pytest-asyncio + py3{6,7,8,9}: ipywidgets + py3{6,7,8,9}: git+https://github.com/casperdcl/nbval.git@named_cells#egg=nbval coverage coveralls codecov @@ -34,10 +34,10 @@ deps= numpy pandas keras: keras - py{36,37,38,39}: rich + py3{6,7,8,9}: rich tf: tensorflow commands= - !py{27,py2}: pytest --cov=tqdm --cov-report= tests_notebook.ipynb --nbval --current-env -W=ignore --sanitize-with=setup.cfg + py3{6,7,8,9}: pytest --cov=tqdm --cov-report= tests_notebook.ipynb --nbval --current-env -W=ignore --sanitize-with=setup.cfg pytest --cov=tqdm --cov-report=xml --cov-report=term --cov-append -k "not perf" {[core]commands} allowlist_externals=codacy From 9743421b94e0e9a8ba1865b99eb43b8d99af42e8 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 9 Mar 2021 15:43:20 +0000 Subject: [PATCH 12/20] pre-commit: pin nbstripout version, fix config --- .pre-commit-config.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3bcea8401..3bf198231 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,9 +52,8 @@ repos: rev: 5.7.0 hooks: - id: isort -repos: - repo: https://github.com/kynan/nbstripout - rev: master + rev: 0.3.9 hooks: - id: nbstripout args: ['--keep-count', '--keep-output'] From a162d02730fd66bbd9380ccbf2fc939d3242b466 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 9 Mar 2021 16:13:27 +0000 Subject: [PATCH 13/20] tests: notebook: fix no total/disable --- tests_notebook.ipynb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests_notebook.ipynb b/tests_notebook.ipynb index a13f3a23f..a526c0466 100644 --- a/tests_notebook.ipynb +++ b/tests_notebook.ipynb @@ -357,9 +357,9 @@ " 0%| | 0/1 [00:00 Date: Tue, 16 Mar 2021 18:47:03 +0100 Subject: [PATCH 14/20] Fix that tqdm_class cannot be passed to `tmap` and `tzip` --- tests/tests_contrib.py | 34 ++++++++++++++++++++++------------ tqdm/contrib/__init__.py | 3 ++- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/tests_contrib.py b/tests/tests_contrib.py index cf684eeae..7459d44c3 100644 --- a/tests/tests_contrib.py +++ b/tests/tests_contrib.py @@ -3,6 +3,9 @@ """ import sys +import pytest + +from tqdm import tqdm from tqdm.contrib import tenumerate, tmap, tzip from .tests_tqdm import StringIO, closing, importorskip @@ -13,49 +16,56 @@ def incr(x): return x + 1 -def test_enumerate(): +@pytest.mark.parametrize("tqdm_kwargs", [dict(), dict(tqdm_class=tqdm)]) +def test_enumerate(tqdm_kwargs): """Test contrib.tenumerate""" with closing(StringIO()) as our_file: a = range(9) - assert list(tenumerate(a, file=our_file)) == list(enumerate(a)) - assert list(tenumerate(a, 42, file=our_file)) == list(enumerate(a, 42)) + assert list(tenumerate(a, file=our_file, **tqdm_kwargs)) == list(enumerate(a)) + assert list(tenumerate(a, 42, file=our_file, **tqdm_kwargs)) == list( + enumerate(a, 42) + ) with closing(StringIO()) as our_file: - _ = list(tenumerate((i for i in a), file=our_file)) + _ = list(tenumerate((i for i in a), file=our_file, **tqdm_kwargs)) assert "100%" not in our_file.getvalue() with closing(StringIO()) as our_file: - _ = list(tenumerate((i for i in a), file=our_file, total=len(a))) + _ = list(tenumerate((i for i in a), file=our_file, total=len(a), **tqdm_kwargs)) assert "100%" in our_file.getvalue() def test_enumerate_numpy(): """Test contrib.tenumerate(numpy.ndarray)""" - np = importorskip('numpy') + np = importorskip("numpy") with closing(StringIO()) as our_file: a = np.random.random((42, 7)) assert list(tenumerate(a, file=our_file)) == list(np.ndenumerate(a)) -def test_zip(): +@pytest.mark.parametrize("tqdm_kwargs", [dict(), dict(tqdm_class=tqdm)]) +def test_zip(tqdm_kwargs): """Test contrib.tzip""" with closing(StringIO()) as our_file: a = range(9) b = [i + 1 for i in a] if sys.version_info[:1] < (3,): - assert tzip(a, b, file=our_file) == zip(a, b) + assert tzip(a, b, file=our_file, **tqdm_kwargs) == zip(a, b) else: - gen = tzip(a, b, file=our_file) + gen = tzip(a, b, file=our_file, **tqdm_kwargs) assert gen != list(zip(a, b)) assert list(gen) == list(zip(a, b)) -def test_map(): +@pytest.mark.parametrize("tqdm_kwargs", [dict(), dict(tqdm_class=tqdm)]) +def test_map(tqdm_kwargs): """Test contrib.tmap""" with closing(StringIO()) as our_file: a = range(9) b = [i + 1 for i in a] if sys.version_info[:1] < (3,): - assert tmap(lambda x: x + 1, a, file=our_file) == map(incr, a) + assert tmap(lambda x: x + 1, a, file=our_file, **tqdm_kwargs) == map( + incr, a + ) else: - gen = tmap(lambda x: x + 1, a, file=our_file) + gen = tmap(lambda x: x + 1, a, file=our_file, **tqdm_kwargs) assert gen != b assert list(gen) == b diff --git a/tqdm/contrib/__init__.py b/tqdm/contrib/__init__.py index 8a6d2474c..ac1969890 100644 --- a/tqdm/contrib/__init__.py +++ b/tqdm/contrib/__init__.py @@ -16,6 +16,7 @@ class DummyTqdmFile(ObjectWrapper): """Dummy file-like that will write to tqdm""" + def __init__(self, wrapped): super(DummyTqdmFile, self).__init__(wrapped) self._buf = [] @@ -80,7 +81,7 @@ def tzip(iter1, *iter2plus, **tqdm_kwargs): """ kwargs = tqdm_kwargs.copy() tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) - for i in zip(tqdm_class(iter1, **tqdm_kwargs), *iter2plus): + for i in zip(tqdm_class(iter1, **kwargs), *iter2plus): yield i From c633fc59b1426cdc25df544ebc1bb38634f0e19d Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Tue, 16 Mar 2021 18:59:55 +0100 Subject: [PATCH 15/20] Fix flake8 --- tests/tests_contrib.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tests_contrib.py b/tests/tests_contrib.py index 7459d44c3..3a8396251 100644 --- a/tests/tests_contrib.py +++ b/tests/tests_contrib.py @@ -16,7 +16,7 @@ def incr(x): return x + 1 -@pytest.mark.parametrize("tqdm_kwargs", [dict(), dict(tqdm_class=tqdm)]) +@pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) def test_enumerate(tqdm_kwargs): """Test contrib.tenumerate""" with closing(StringIO()) as our_file: @@ -41,7 +41,7 @@ def test_enumerate_numpy(): assert list(tenumerate(a, file=our_file)) == list(np.ndenumerate(a)) -@pytest.mark.parametrize("tqdm_kwargs", [dict(), dict(tqdm_class=tqdm)]) +@pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) def test_zip(tqdm_kwargs): """Test contrib.tzip""" with closing(StringIO()) as our_file: @@ -55,7 +55,7 @@ def test_zip(tqdm_kwargs): assert list(gen) == list(zip(a, b)) -@pytest.mark.parametrize("tqdm_kwargs", [dict(), dict(tqdm_class=tqdm)]) +@pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) def test_map(tqdm_kwargs): """Test contrib.tmap""" with closing(StringIO()) as our_file: From a22aebcd1d4b0295e8e1f4e795861a5f108f2f74 Mon Sep 17 00:00:00 2001 From: Peter Hansen Date: Mon, 8 Mar 2021 11:57:09 -0600 Subject: [PATCH 16/20] fix delay for notebooks --- tqdm/notebook.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tqdm/notebook.py b/tqdm/notebook.py index 2d73ea825..5b2debaf5 100644 --- a/tqdm/notebook.py +++ b/tqdm/notebook.py @@ -234,8 +234,10 @@ def __init__(self, *args, **kwargs): total = self.total * unit_scale if self.total else self.total self.container = self.status_printer(self.fp, total, self.desc, self.ncols) self.container.pbar = self - if display_here: + self.displayed = False + if display_here and self.delay <= 0: display(self.container) + self.displayed = True self.disp = self.display self.colour = colour @@ -256,6 +258,9 @@ def __iter__(self): # since this could be a shared bar which the user will `reset()` def update(self, n=1): + if not self.displayed and self.delay > 0: + display(self.container) + self.displayed = True try: return super(tqdm_notebook, self).update(n=n) # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt From a638d6a157543c437145dd1506a25fc18c2f1543 Mon Sep 17 00:00:00 2001 From: Peter Hansen Date: Wed, 17 Mar 2021 15:37:30 -0500 Subject: [PATCH 17/20] notebook update checks if disabled --- tqdm/notebook.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tqdm/notebook.py b/tqdm/notebook.py index 5b2debaf5..0baf6ea7c 100644 --- a/tqdm/notebook.py +++ b/tqdm/notebook.py @@ -258,6 +258,8 @@ def __iter__(self): # since this could be a shared bar which the user will `reset()` def update(self, n=1): + if self.disable: + return if not self.displayed and self.delay > 0: display(self.container) self.displayed = True From 8a66f6e35d753f58dbed4425ab2fb91641bef715 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 5 Apr 2021 16:52:03 +0100 Subject: [PATCH 18/20] notebook: fix `delay`, add tests --- tests_notebook.ipynb | 41 ++++++++++++++++++++++++++++++++++++++++- tqdm/notebook.py | 21 +++++++++++---------- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/tests_notebook.ipynb b/tests_notebook.ipynb index a526c0466..fb8227d7d 100644 --- a/tests_notebook.ipynb +++ b/tests_notebook.ipynb @@ -16,6 +16,7 @@ "outputs": [], "source": [ "from functools import partial\n", + "from time import sleep\n", "\n", "from tqdm.notebook import tqdm_notebook\n", "from tqdm.notebook import tnrange\n", @@ -36,7 +37,7 @@ "text": [ "Help on function display in module tqdm.notebook:\n", "\n", - "display(self, msg=None, pos=None, close=False, bar_style=None)\n", + "display(self, msg=None, pos=None, close=False, bar_style=None, check_delay=True)\n", " Use `self.sp` to display `msg` in the specified `pos`.\n", " \n", " Consider overloading this function when inheriting to use e.g.:\n", @@ -450,6 +451,44 @@ " print(t)\n", " assert t.colour == 'yellow'" ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "# NBVAL_TEST_NAME: delay no trigger\n", + "with tqdm_notebook(total=1, delay=10) as t:\n", + " t.update()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fe102eedbb4f437783fbd0cff32f6613", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "100%|##########| 1/1 [00:00<00:00, 7.68it/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# NBVAL_TEST_NAME: delay trigger\n", + "with tqdm_notebook(total=1, delay=0.1) as t:\n", + " sleep(0.1)\n", + " t.update()" + ] } ], "metadata": { diff --git a/tqdm/notebook.py b/tqdm/notebook.py index 0baf6ea7c..db25d8570 100644 --- a/tqdm/notebook.py +++ b/tqdm/notebook.py @@ -145,7 +145,7 @@ def status_printer(_, total=None, desc=None, ncols=None): def display(self, msg=None, pos=None, # additional signals - close=False, bar_style=None): + close=False, bar_style=None, check_delay=True): # Note: contrary to native tqdm, msg='' does NOT clear bar # goal is to keep all infos if error happens so user knows # at which iteration the loop failed. @@ -190,6 +190,10 @@ def display(self, msg=None, pos=None, except AttributeError: self.container.visible = False + if check_delay and self.delay > 0 and not self.displayed: + display(self.container) + self.displayed = True + @property def colour(self): if hasattr(self, 'container'): @@ -243,7 +247,7 @@ def __init__(self, *args, **kwargs): # Print initial bar state if not self.disable: - self.display() + self.display(check_delay=False) def __iter__(self): try: @@ -258,11 +262,6 @@ def __iter__(self): # since this could be a shared bar which the user will `reset()` def update(self, n=1): - if self.disable: - return - if not self.displayed and self.delay > 0: - display(self.container) - self.displayed = True try: return super(tqdm_notebook, self).update(n=n) # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt @@ -275,16 +274,18 @@ def update(self, n=1): # since this could be a shared bar which the user will `reset()` def close(self): + if self.disable: + return super(tqdm_notebook, self).close() # Try to detect if there was an error or KeyboardInterrupt # in manual mode: if n < total, things probably got wrong if self.total and self.n < self.total: - self.disp(bar_style='danger') + self.disp(bar_style='danger', check_delay=False) else: if self.leave: - self.disp(bar_style='success') + self.disp(bar_style='success', check_delay=False) else: - self.disp(close=True) + self.disp(close=True, check_delay=False) def clear(self, *_, **__): pass From e86b9b18a000136b85703b48504775d5aa350adc Mon Sep 17 00:00:00 2001 From: Daniel Ecer Date: Thu, 1 Apr 2021 20:32:38 +0100 Subject: [PATCH 19/20] added logging sub-module added logging sub-module python 2 compatibility fixed python 2 fix added test for custom tqdm class python 2 absolute imports (due to otherwise conflicting `logging` module) isort more tests relating to _get_first_found_console_formatter isort minor simplification test handleError test logging formatter being used minor rename to _get_first_found_console_logging_formatter test that certain exceptions are not swallowed avoid using mock.assert_called (py 3.5) moved to tqdm.contrib.logging added "Redirecting console logging to tqdm" readme removed no longer necessary absolute_import declaration minor: updated package of example in docstring --- README.rst | 53 +++++++++ tests/contrib/__init__.py | 0 tests/contrib/tests_logging.py | 182 +++++++++++++++++++++++++++++++ tqdm/contrib/logging.py | 194 +++++++++++++++++++++++++++++++++ 4 files changed, 429 insertions(+) create mode 100644 tests/contrib/__init__.py create mode 100644 tests/contrib/tests_logging.py create mode 100644 tqdm/contrib/logging.py diff --git a/README.rst b/README.rst index dd7e5a381..0ea0e98b0 100644 --- a/README.rst +++ b/README.rst @@ -1321,6 +1321,59 @@ A reusable canonical example is given below: # After the `with`, printing is restored print("Done!") +Redirecting console logging to tqdm +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Similar to redirecting ``sys.stdout`` directly as detailed in the previous section, +you may want to redirect logging that would otherwise go to the +console (``sys.stdout`` or ``sys.stderr``) to ``tqdm``. + +Note: if you are also replace ``sys.stdout`` and ``sys.stderr`` at the same time, +then the logging should be redirected first. Otherwise it won't be able to detect +the console logging handler. + +For that you may use ``redirect_logging_to_tqdm`` or ``tqdm_with_logging_redirect`` +from ``tqdm.contrib.logging``. Both methods accept the following optional parameters: + +- ``loggers``: A list of loggers to update. Defaults to ``logging.root``. +- ``tqdm``: A ``tqdm`` class. Defaults to ``tqdm.tqdm``. + +An example redirecting the console logging to tqdm: + +.. code:: python + + import logging + from tqdm.contrib.logging import redirect_logging_to_tqdm + + LOGGER = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level='INFO') + with redirect_logging_to_tqdm(): + # logging to the console is now redirected to tqdm + LOGGER.info('some message') + # logging is now restored + +An similar example, wrapping tqdm while redirecting console logging: + +.. code:: python + + import logging + from tqdm.contrib.logging import tqdm_with_logging_redirect + + LOGGER = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level='INFO') + + file_list = ['file1', 'file2'] + with tqdm_with_logging_redirect(total=len(file_list)) as pbar: + # logging to the console is now redirected to tqdm + for filename in file_list: + LOGGER.info('processing file: %s', filename) + pbar.update(1) + # logging is now restored + Monitoring thread, intervals and miniters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/contrib/__init__.py b/tests/contrib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/contrib/tests_logging.py b/tests/contrib/tests_logging.py new file mode 100644 index 000000000..0b3342257 --- /dev/null +++ b/tests/contrib/tests_logging.py @@ -0,0 +1,182 @@ +# pylint: disable=missing-module-docstring, missing-class-docstring +# pylint: disable=missing-function-docstring, no-self-use + +from __future__ import absolute_import + +import logging +import logging.handlers +import sys +from io import StringIO + +import pytest + +from tqdm import tqdm +from tqdm.contrib.logging import _get_first_found_console_logging_formatter +from tqdm.contrib.logging import _TqdmLoggingHandler as TqdmLoggingHandler +from tqdm.contrib.logging import redirect_logging_to_tqdm, tqdm_with_logging_redirect + +from ..tests_tqdm import importorskip + +LOGGER = logging.getLogger(__name__) + +TEST_LOGGING_FORMATTER = logging.Formatter() + + +class CustomTqdm(tqdm): + messages = [] + + @classmethod + def write(cls, s, **__): # pylint: disable=arguments-differ + CustomTqdm.messages.append(s) + + +class ErrorRaisingTqdm(tqdm): + exception_class = RuntimeError + + @classmethod + def write(cls, s, **__): # pylint: disable=arguments-differ + raise ErrorRaisingTqdm.exception_class('fail fast') + + +class TestTqdmLoggingHandler: + def test_should_call_tqdm_write(self): + CustomTqdm.messages = [] + logger = logging.Logger('test') + logger.handlers = [TqdmLoggingHandler(CustomTqdm)] + logger.info('test') + assert CustomTqdm.messages == ['test'] + + def test_should_call_handle_error_if_exception_was_thrown(self): + patch = importorskip('unittest.mock').patch + logger = logging.Logger('test') + ErrorRaisingTqdm.exception_class = RuntimeError + handler = TqdmLoggingHandler(ErrorRaisingTqdm) + logger.handlers = [handler] + with patch.object(handler, 'handleError') as mock: + logger.info('test') + assert mock.called + + @pytest.mark.parametrize('exception_class', [ + KeyboardInterrupt, + SystemExit + ]) + def test_should_not_swallow_certain_exceptions(self, exception_class): + logger = logging.Logger('test') + ErrorRaisingTqdm.exception_class = exception_class + handler = TqdmLoggingHandler(ErrorRaisingTqdm) + logger.handlers = [handler] + with pytest.raises(exception_class): + logger.info('test') + + +class TestGetFirstFoundConsoleLoggingFormatter: + def test_should_return_none_for_no_handlers(self): + assert _get_first_found_console_logging_formatter([]) is None + + def test_should_return_none_without_stream_handler(self): + handler = logging.handlers.MemoryHandler(capacity=1) + handler.formatter = TEST_LOGGING_FORMATTER + assert _get_first_found_console_logging_formatter([handler]) is None + + def test_should_return_none_for_stream_handler_not_stdout_or_stderr(self): + handler = logging.StreamHandler(StringIO()) + handler.formatter = TEST_LOGGING_FORMATTER + assert _get_first_found_console_logging_formatter([handler]) is None + + def test_should_return_stream_handler_formatter_if_stream_is_stdout(self): + handler = logging.StreamHandler(sys.stdout) + handler.formatter = TEST_LOGGING_FORMATTER + assert _get_first_found_console_logging_formatter( + [handler] + ) == TEST_LOGGING_FORMATTER + + def test_should_return_stream_handler_formatter_if_stream_is_stderr(self): + handler = logging.StreamHandler(sys.stderr) + handler.formatter = TEST_LOGGING_FORMATTER + assert _get_first_found_console_logging_formatter( + [handler] + ) == TEST_LOGGING_FORMATTER + + +class TestRedirectLoggingToTqdm: + def test_should_add_and_remove_tqdm_handler(self): + logger = logging.Logger('test') + with redirect_logging_to_tqdm(loggers=[logger]): + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], TqdmLoggingHandler) + assert not logger.handlers + + def test_should_remove_and_restore_console_handlers(self): + logger = logging.Logger('test') + stderr_console_handler = logging.StreamHandler(sys.stderr) + stdout_console_handler = logging.StreamHandler(sys.stderr) + logger.handlers = [stderr_console_handler, stdout_console_handler] + with redirect_logging_to_tqdm(loggers=[logger]): + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], TqdmLoggingHandler) + assert logger.handlers == [stderr_console_handler, stdout_console_handler] + + def test_should_inherit_console_logger_formatter(self): + logger = logging.Logger('test') + formatter = logging.Formatter('custom: %(message)s') + console_handler = logging.StreamHandler(sys.stderr) + console_handler.setFormatter(formatter) + logger.handlers = [console_handler] + with redirect_logging_to_tqdm(loggers=[logger]): + assert logger.handlers[0].formatter == formatter + + def test_should_not_remove_stream_handlers_not_fot_stdout_or_stderr(self): + logger = logging.Logger('test') + stream_handler = logging.StreamHandler(StringIO()) + logger.addHandler(stream_handler) + with redirect_logging_to_tqdm(loggers=[logger]): + assert len(logger.handlers) == 2 + assert logger.handlers[0] == stream_handler + assert isinstance(logger.handlers[1], TqdmLoggingHandler) + assert logger.handlers == [stream_handler] + + +class TestTqdmWithLoggingRedirect: + def test_should_add_and_remove_handler_from_root_logger_by_default(self): + original_handlers = list(logging.root.handlers) + with tqdm_with_logging_redirect(total=1) as pbar: + assert isinstance(logging.root.handlers[-1], TqdmLoggingHandler) + LOGGER.info('test') + pbar.update(1) + assert logging.root.handlers == original_handlers + + def test_should_add_and_remove_handler_from_custom_logger(self): + logger = logging.Logger('test') + with tqdm_with_logging_redirect(total=1, loggers=[logger]) as pbar: + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], TqdmLoggingHandler) + logger.info('test') + pbar.update(1) + assert not logger.handlers + + def test_should_not_fail_with_logger_without_console_handler(self): + logger = logging.Logger('test') + logger.handlers = [] + with tqdm_with_logging_redirect(total=1, loggers=[logger]): + logger.info('test') + assert not logger.handlers + + def test_should_format_message(self): + logger = logging.Logger('test') + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(logging.Formatter( + r'prefix:%(message)s' + )) + logger.handlers = [console_handler] + CustomTqdm.messages = [] + with tqdm_with_logging_redirect(loggers=[logger], tqdm=CustomTqdm): + logger.info('test') + assert CustomTqdm.messages == ['prefix:test'] + + def test_use_root_logger_by_default_and_write_to_custom_tqdm(self): + logger = logging.root + CustomTqdm.messages = [] + with tqdm_with_logging_redirect(total=1, tqdm=CustomTqdm) as pbar: + assert isinstance(pbar, CustomTqdm) + logger.info('test') + assert CustomTqdm.messages == ['test'] diff --git a/tqdm/contrib/logging.py b/tqdm/contrib/logging.py new file mode 100644 index 000000000..08c9c3c40 --- /dev/null +++ b/tqdm/contrib/logging.py @@ -0,0 +1,194 @@ + +""" +Enables multiple commonly used features relating to logging +in combination with tqdm. +""" +from __future__ import absolute_import + +import logging +import sys +from contextlib import contextmanager + +try: + from typing import Iterator, List, Optional, Type # pylint: disable=unused-import +except ImportError: + # we may ignore type hints + pass + +from ..std import tqdm as _tqdm + + +class _TqdmLoggingHandler(logging.StreamHandler): + def __init__( + self, + tqdm=None # type: Optional[Type[tqdm.tqdm]] + ): + super( # pylint: disable=super-with-arguments + _TqdmLoggingHandler, self + ).__init__() + if tqdm is None: + tqdm = _tqdm + self.tqdm = tqdm + + def emit(self, record): + try: + msg = self.format(record) + self.tqdm.write(msg) + self.flush() + except (KeyboardInterrupt, SystemExit): + raise + except: # noqa pylint: disable=bare-except + self.handleError(record) + + +def _is_console_logging_handler(handler): + return ( + isinstance(handler, logging.StreamHandler) + and handler.stream in {sys.stdout, sys.stderr} + ) + + +def _get_first_found_console_logging_formatter(handlers): + for handler in handlers: + if _is_console_logging_handler(handler): + return handler.formatter + return None + + +@contextmanager +def redirect_logging_to_tqdm( + loggers=None, # type: Optional[List[logging.Logger]], + tqdm=None # type: Optional[Type[tqdm.tqdm]] +): + # type: (...) -> Iterator[None] + """ + Context manager for redirecting logging console output to tqdm. + Logging to other logging handlers, such as a log file, + will not be affected. + + By default the, the handlers of the root logger will be amended. + (for the duration of the context) + You may also provide a list of `loggers` instead + (e.g. if a particular logger doesn't fallback to the root logger) + + Example: + + ```python + import logging + from tqdm.contrib.logging import redirect_logging_to_tqdm + + LOGGER = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level='INFO') + with redirect_logging_to_tqdm(): + # logging to the console is now redirected to tqdm + LOGGER.info('some message') + # logging is now restored + ``` + """ + if loggers is None: + loggers = [logging.root] + original_handlers_list = [ + logger.handlers for logger in loggers + ] + try: + for logger in loggers: + tqdm_handler = _TqdmLoggingHandler(tqdm) + tqdm_handler.setFormatter( + _get_first_found_console_logging_formatter( + logger.handlers + ) + ) + logger.handlers = [ + handler + for handler in logger.handlers + if not _is_console_logging_handler(handler) + ] + [tqdm_handler] + yield + finally: + for logger, original_handlers in zip(loggers, original_handlers_list): + logger.handlers = original_handlers + + +def _pop_optional( + kwargs, # type: dict + key, # type: str + default_value=None +): + try: + return kwargs.pop(key) + except KeyError: + return default_value + + +@contextmanager +def tqdm_with_logging_redirect( + *args, + # loggers=None, # type: Optional[List[logging.Logger]] + # tqdm=None, # type: Optional[Type[tqdm.tqdm]] + **kwargs +): + # type: (...) -> Iterator[None] + """ + Similar to `redirect_logging_to_tqdm`, + but provides a context manager wrapping tqdm. + + All parameters, except `loggers` and `tqdm`, will get passed on to `tqdm`. + + By default this will wrap `tqdm.tqdm`. + You may pass your own `tqdm` class if desired. + + Example: + + ```python + import logging + from tqdm.contrib.logging import tqdm_with_logging_redirect + + LOGGER = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level='INFO') + + file_list = ['file1', 'file2'] + with tqdm_with_logging_redirect(total=len(file_list)) as pbar: + # logging to the console is now redirected to tqdm + for filename in file_list: + LOGGER.info('processing file: %s', filename) + pbar.update(1) + # logging is now restored + ``` + + A more advanced example with non-default tqdm class and loggers: + + ```python + import logging + from tqdm.auto import tqdm + from tqdm.contrib.logging import tqdm_with_logging_redirect + + LOGGER = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level='INFO') + + file_list = ['file1', 'file2'] + with tqdm_with_logging_redirect( + total=len(file_list), + tqdm=tqdm, + loggers=[LOGGER] + ) as pbar: + # logging to the console is now redirected to tqdm + for filename in file_list: + LOGGER.info('processing file: %s', filename) + pbar.update(1) + # logging is now restored + ``` + + """ + loggers = _pop_optional(kwargs, 'loggers') + tqdm = _pop_optional(kwargs, 'tqdm') + if tqdm is None: + tqdm = _tqdm + with tqdm(*args, **kwargs) as pbar: + with redirect_logging_to_tqdm(loggers=loggers, tqdm=tqdm): + yield pbar From 745866669845f6f88c881253725343a0b461a743 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 6 Apr 2021 00:46:07 +0100 Subject: [PATCH 20/20] contrib.logging: cleanup & docs --- .meta/.readme.rst | 27 +++ README.rst | 58 ++----- tests/contrib/__init__.py | 0 ...ts_logging.py => tests_contrib_logging.py} | 23 ++- tqdm/contrib/logging.py | 164 +++++------------- 5 files changed, 102 insertions(+), 170 deletions(-) delete mode 100644 tests/contrib/__init__.py rename tests/{contrib/tests_logging.py => tests_contrib_logging.py} (90%) diff --git a/.meta/.readme.rst b/.meta/.readme.rst index 97bf461fc..d299164b9 100644 --- a/.meta/.readme.rst +++ b/.meta/.readme.rst @@ -1102,6 +1102,33 @@ A reusable canonical example is given below: # After the `with`, printing is restored print("Done!") +Redirecting ``logging`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Similar to ``sys.stdout``/``sys.stderr`` as detailed above, console ``logging`` +may also be redirected to ``tqdm.write()``. + +Warning: if also redirecting ``sys.stdout``/``sys.stderr``, make sure to +redirect ``logging`` first if needed. + +Helper methods are available in ``tqdm.contrib.logging``. For example: + +.. code:: python + + import logging + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm + + LOG = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored + Monitoring thread, intervals and miniters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index 0ea0e98b0..c906e60cf 100644 --- a/README.rst +++ b/README.rst @@ -1321,58 +1321,32 @@ A reusable canonical example is given below: # After the `with`, printing is restored print("Done!") -Redirecting console logging to tqdm -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Redirecting ``logging`` +~~~~~~~~~~~~~~~~~~~~~~~ -Similar to redirecting ``sys.stdout`` directly as detailed in the previous section, -you may want to redirect logging that would otherwise go to the -console (``sys.stdout`` or ``sys.stderr``) to ``tqdm``. +Similar to ``sys.stdout``/``sys.stderr`` as detailed above, console ``logging`` +may also be redirected to ``tqdm.write()``. -Note: if you are also replace ``sys.stdout`` and ``sys.stderr`` at the same time, -then the logging should be redirected first. Otherwise it won't be able to detect -the console logging handler. +Warning: if also redirecting ``sys.stdout``/``sys.stderr``, make sure to +redirect ``logging`` first if needed. -For that you may use ``redirect_logging_to_tqdm`` or ``tqdm_with_logging_redirect`` -from ``tqdm.contrib.logging``. Both methods accept the following optional parameters: - -- ``loggers``: A list of loggers to update. Defaults to ``logging.root``. -- ``tqdm``: A ``tqdm`` class. Defaults to ``tqdm.tqdm``. - -An example redirecting the console logging to tqdm: +Helper methods are available in ``tqdm.contrib.logging``. For example: .. code:: python import logging - from tqdm.contrib.logging import redirect_logging_to_tqdm - - LOGGER = logging.getLogger(__name__) - - if __name__ == '__main__': - logging.basicConfig(level='INFO') - with redirect_logging_to_tqdm(): - # logging to the console is now redirected to tqdm - LOGGER.info('some message') - # logging is now restored - -An similar example, wrapping tqdm while redirecting console logging: - -.. code:: python - - import logging - from tqdm.contrib.logging import tqdm_with_logging_redirect + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm - LOGGER = logging.getLogger(__name__) + LOG = logging.getLogger(__name__) if __name__ == '__main__': - logging.basicConfig(level='INFO') - - file_list = ['file1', 'file2'] - with tqdm_with_logging_redirect(total=len(file_list)) as pbar: - # logging to the console is now redirected to tqdm - for filename in file_list: - LOGGER.info('processing file: %s', filename) - pbar.update(1) - # logging is now restored + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored Monitoring thread, intervals and miniters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/contrib/__init__.py b/tests/contrib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/contrib/tests_logging.py b/tests/tests_contrib_logging.py similarity index 90% rename from tests/contrib/tests_logging.py rename to tests/tests_contrib_logging.py index 0b3342257..e2affa786 100644 --- a/tests/contrib/tests_logging.py +++ b/tests/tests_contrib_logging.py @@ -1,6 +1,5 @@ # pylint: disable=missing-module-docstring, missing-class-docstring # pylint: disable=missing-function-docstring, no-self-use - from __future__ import absolute_import import logging @@ -13,9 +12,9 @@ from tqdm import tqdm from tqdm.contrib.logging import _get_first_found_console_logging_formatter from tqdm.contrib.logging import _TqdmLoggingHandler as TqdmLoggingHandler -from tqdm.contrib.logging import redirect_logging_to_tqdm, tqdm_with_logging_redirect +from tqdm.contrib.logging import logging_redirect_tqdm, tqdm_logging_redirect -from ..tests_tqdm import importorskip +from .tests_tqdm import importorskip LOGGER = logging.getLogger(__name__) @@ -101,7 +100,7 @@ def test_should_return_stream_handler_formatter_if_stream_is_stderr(self): class TestRedirectLoggingToTqdm: def test_should_add_and_remove_tqdm_handler(self): logger = logging.Logger('test') - with redirect_logging_to_tqdm(loggers=[logger]): + with logging_redirect_tqdm(loggers=[logger]): assert len(logger.handlers) == 1 assert isinstance(logger.handlers[0], TqdmLoggingHandler) assert not logger.handlers @@ -111,7 +110,7 @@ def test_should_remove_and_restore_console_handlers(self): stderr_console_handler = logging.StreamHandler(sys.stderr) stdout_console_handler = logging.StreamHandler(sys.stderr) logger.handlers = [stderr_console_handler, stdout_console_handler] - with redirect_logging_to_tqdm(loggers=[logger]): + with logging_redirect_tqdm(loggers=[logger]): assert len(logger.handlers) == 1 assert isinstance(logger.handlers[0], TqdmLoggingHandler) assert logger.handlers == [stderr_console_handler, stdout_console_handler] @@ -122,14 +121,14 @@ def test_should_inherit_console_logger_formatter(self): console_handler = logging.StreamHandler(sys.stderr) console_handler.setFormatter(formatter) logger.handlers = [console_handler] - with redirect_logging_to_tqdm(loggers=[logger]): + with logging_redirect_tqdm(loggers=[logger]): assert logger.handlers[0].formatter == formatter def test_should_not_remove_stream_handlers_not_fot_stdout_or_stderr(self): logger = logging.Logger('test') stream_handler = logging.StreamHandler(StringIO()) logger.addHandler(stream_handler) - with redirect_logging_to_tqdm(loggers=[logger]): + with logging_redirect_tqdm(loggers=[logger]): assert len(logger.handlers) == 2 assert logger.handlers[0] == stream_handler assert isinstance(logger.handlers[1], TqdmLoggingHandler) @@ -139,7 +138,7 @@ def test_should_not_remove_stream_handlers_not_fot_stdout_or_stderr(self): class TestTqdmWithLoggingRedirect: def test_should_add_and_remove_handler_from_root_logger_by_default(self): original_handlers = list(logging.root.handlers) - with tqdm_with_logging_redirect(total=1) as pbar: + with tqdm_logging_redirect(total=1) as pbar: assert isinstance(logging.root.handlers[-1], TqdmLoggingHandler) LOGGER.info('test') pbar.update(1) @@ -147,7 +146,7 @@ def test_should_add_and_remove_handler_from_root_logger_by_default(self): def test_should_add_and_remove_handler_from_custom_logger(self): logger = logging.Logger('test') - with tqdm_with_logging_redirect(total=1, loggers=[logger]) as pbar: + with tqdm_logging_redirect(total=1, loggers=[logger]) as pbar: assert len(logger.handlers) == 1 assert isinstance(logger.handlers[0], TqdmLoggingHandler) logger.info('test') @@ -157,7 +156,7 @@ def test_should_add_and_remove_handler_from_custom_logger(self): def test_should_not_fail_with_logger_without_console_handler(self): logger = logging.Logger('test') logger.handlers = [] - with tqdm_with_logging_redirect(total=1, loggers=[logger]): + with tqdm_logging_redirect(total=1, loggers=[logger]): logger.info('test') assert not logger.handlers @@ -169,14 +168,14 @@ def test_should_format_message(self): )) logger.handlers = [console_handler] CustomTqdm.messages = [] - with tqdm_with_logging_redirect(loggers=[logger], tqdm=CustomTqdm): + with tqdm_logging_redirect(loggers=[logger], tqdm_class=CustomTqdm): logger.info('test') assert CustomTqdm.messages == ['prefix:test'] def test_use_root_logger_by_default_and_write_to_custom_tqdm(self): logger = logging.root CustomTqdm.messages = [] - with tqdm_with_logging_redirect(total=1, tqdm=CustomTqdm) as pbar: + with tqdm_logging_redirect(total=1, tqdm_class=CustomTqdm) as pbar: assert isinstance(pbar, CustomTqdm) logger.info('test') assert CustomTqdm.messages == ['test'] diff --git a/tqdm/contrib/logging.py b/tqdm/contrib/logging.py index 08c9c3c40..5f70944dc 100644 --- a/tqdm/contrib/logging.py +++ b/tqdm/contrib/logging.py @@ -1,7 +1,5 @@ - """ -Enables multiple commonly used features relating to logging -in combination with tqdm. +Helper functionality for interoperability with stdlib `logging`. """ from __future__ import absolute_import @@ -12,28 +10,23 @@ try: from typing import Iterator, List, Optional, Type # pylint: disable=unused-import except ImportError: - # we may ignore type hints pass -from ..std import tqdm as _tqdm +from ..std import tqdm as std_tqdm class _TqdmLoggingHandler(logging.StreamHandler): def __init__( self, - tqdm=None # type: Optional[Type[tqdm.tqdm]] + tqdm_class=std_tqdm # type: Type[std_tqdm] ): - super( # pylint: disable=super-with-arguments - _TqdmLoggingHandler, self - ).__init__() - if tqdm is None: - tqdm = _tqdm - self.tqdm = tqdm + super(_TqdmLoggingHandler, self).__init__() + self.tqdm_class = tqdm_class def emit(self, record): try: msg = self.format(record) - self.tqdm.write(msg) + self.tqdm_class.write(msg) self.flush() except (KeyboardInterrupt, SystemExit): raise @@ -42,88 +35,69 @@ def emit(self, record): def _is_console_logging_handler(handler): - return ( - isinstance(handler, logging.StreamHandler) - and handler.stream in {sys.stdout, sys.stderr} - ) + return (isinstance(handler, logging.StreamHandler) + and handler.stream in {sys.stdout, sys.stderr}) def _get_first_found_console_logging_formatter(handlers): for handler in handlers: if _is_console_logging_handler(handler): return handler.formatter - return None @contextmanager -def redirect_logging_to_tqdm( +def logging_redirect_tqdm( loggers=None, # type: Optional[List[logging.Logger]], - tqdm=None # type: Optional[Type[tqdm.tqdm]] + tqdm_class=std_tqdm # type: Type[std_tqdm] ): # type: (...) -> Iterator[None] """ - Context manager for redirecting logging console output to tqdm. - Logging to other logging handlers, such as a log file, - will not be affected. - - By default the, the handlers of the root logger will be amended. - (for the duration of the context) - You may also provide a list of `loggers` instead - (e.g. if a particular logger doesn't fallback to the root logger) + Context manager redirecting console logging to `tqdm.write()`, leaving + other logging handlers (e.g. log files) unaffected. - Example: + Parameters + ---------- + loggers : list, optional + Which handlers to redirect (default: [logging.root]). + tqdm_class : optional + Example + ------- ```python import logging - from tqdm.contrib.logging import redirect_logging_to_tqdm + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm - LOGGER = logging.getLogger(__name__) + LOG = logging.getLogger(__name__) if __name__ == '__main__': - logging.basicConfig(level='INFO') - with redirect_logging_to_tqdm(): - # logging to the console is now redirected to tqdm - LOGGER.info('some message') - # logging is now restored + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored ``` """ if loggers is None: loggers = [logging.root] - original_handlers_list = [ - logger.handlers for logger in loggers - ] + original_handlers_list = [logger.handlers for logger in loggers] try: for logger in loggers: - tqdm_handler = _TqdmLoggingHandler(tqdm) + tqdm_handler = _TqdmLoggingHandler(tqdm_class) tqdm_handler.setFormatter( - _get_first_found_console_logging_formatter( - logger.handlers - ) - ) + _get_first_found_console_logging_formatter(logger.handlers)) logger.handlers = [ - handler - for handler in logger.handlers - if not _is_console_logging_handler(handler) - ] + [tqdm_handler] + handler for handler in logger.handlers + if not _is_console_logging_handler(handler)] + [tqdm_handler] yield finally: for logger, original_handlers in zip(loggers, original_handlers_list): logger.handlers = original_handlers -def _pop_optional( - kwargs, # type: dict - key, # type: str - default_value=None -): - try: - return kwargs.pop(key) - except KeyError: - return default_value - - @contextmanager -def tqdm_with_logging_redirect( +def tqdm_logging_redirect( *args, # loggers=None, # type: Optional[List[logging.Logger]] # tqdm=None, # type: Optional[Type[tqdm.tqdm]] @@ -131,64 +105,22 @@ def tqdm_with_logging_redirect( ): # type: (...) -> Iterator[None] """ - Similar to `redirect_logging_to_tqdm`, - but provides a context manager wrapping tqdm. - - All parameters, except `loggers` and `tqdm`, will get passed on to `tqdm`. - - By default this will wrap `tqdm.tqdm`. - You may pass your own `tqdm` class if desired. - - Example: - + Convenience shortcut for: ```python - import logging - from tqdm.contrib.logging import tqdm_with_logging_redirect - - LOGGER = logging.getLogger(__name__) - - if __name__ == '__main__': - logging.basicConfig(level='INFO') - - file_list = ['file1', 'file2'] - with tqdm_with_logging_redirect(total=len(file_list)) as pbar: - # logging to the console is now redirected to tqdm - for filename in file_list: - LOGGER.info('processing file: %s', filename) - pbar.update(1) - # logging is now restored - ``` - - A more advanced example with non-default tqdm class and loggers: - - ```python - import logging - from tqdm.auto import tqdm - from tqdm.contrib.logging import tqdm_with_logging_redirect - - LOGGER = logging.getLogger(__name__) - - if __name__ == '__main__': - logging.basicConfig(level='INFO') - - file_list = ['file1', 'file2'] - with tqdm_with_logging_redirect( - total=len(file_list), - tqdm=tqdm, - loggers=[LOGGER] - ) as pbar: - # logging to the console is now redirected to tqdm - for filename in file_list: - LOGGER.info('processing file: %s', filename) - pbar.update(1) - # logging is now restored + with tqdm_class(*args, **tqdm_kwargs) as pbar: + with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class): + yield pbar ``` + Parameters + ---------- + tqdm_class : optional, (default: tqdm.std.tqdm). + loggers : optional, list. + **tqdm_kwargs : passed to `tqdm_class`. """ - loggers = _pop_optional(kwargs, 'loggers') - tqdm = _pop_optional(kwargs, 'tqdm') - if tqdm is None: - tqdm = _tqdm - with tqdm(*args, **kwargs) as pbar: - with redirect_logging_to_tqdm(loggers=loggers, tqdm=tqdm): + tqdm_kwargs = kwargs.copy() + loggers = tqdm_kwargs.pop('loggers', None) + tqdm_class = tqdm_kwargs.pop('tqdm_class', std_tqdm) + with tqdm_class(*args, **tqdm_kwargs) as pbar: + with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class): yield pbar