From f16c8886ae16b74bc4aaad9c835bf29956639066 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 24 Nov 2020 11:35:23 +0000 Subject: [PATCH 1/5] add `dask` submodule - fixes #278 - replaces/closes #279 - based on `tqdm.keras` --- tqdm/dask.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tqdm/dask.py diff --git a/tqdm/dask.py b/tqdm/dask.py new file mode 100644 index 000000000..d4413bca5 --- /dev/null +++ b/tqdm/dask.py @@ -0,0 +1,45 @@ +from __future__ import absolute_import +from .auto import tqdm as tqdm_auto +from functools import partial +from dask.callbacks import Callback +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['TqdmCallback'] + + +class TqdmCallback(Callback): + """`dask` callback for task progress""" + def __init__(self, start=None, start_state=None, pretask=None, + posttask=None, finish=None, tqdm_class=tqdm_auto, + **tqdm_kwargs): + """ + Parameters + ---------- + tqdm_class : optional + `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. + tqdm_kwargs : optional + Any other arguments used for all bars. + """ + super(TqdmCallback, self).__init__( + start=start, start_state=start_state, pretask=pretask, + posttask=posttask, finish=finish) + if tqdm_kwargs: + tqdm_class = partial(tqdm_class, **tqdm_kwargs) + self.tqdm_class = tqdm_class + + def _start_state(self, _, state): + self.pbar = self.tqdm_class(total=sum( + len(state[k]) for k in ['ready', 'waiting', 'running', 'finished'])) + + def _posttask(self, *args, **kwargs): + self.pbar.update() + + def _finish(self, *args, **kwargs): + self.pbar.close() + + def display(self): + """displays in the current cell in Notebooks""" + container = getattr(self.bar, 'container', None) + if container is None: + return + from .notebook import display + display(container) From 1e50916fdc4e5a86a65f95e239a64b019a4bc24a Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 22 Feb 2021 22:53:38 +0000 Subject: [PATCH 2/5] dask: purge unused args --- tqdm/dask.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tqdm/dask.py b/tqdm/dask.py index d4413bca5..f07d1b9ba 100644 --- a/tqdm/dask.py +++ b/tqdm/dask.py @@ -8,8 +8,7 @@ class TqdmCallback(Callback): """`dask` callback for task progress""" - def __init__(self, start=None, start_state=None, pretask=None, - posttask=None, finish=None, tqdm_class=tqdm_auto, + def __init__(self, start=None, pretask=None, tqdm_class=tqdm_auto, **tqdm_kwargs): """ Parameters @@ -19,9 +18,7 @@ def __init__(self, start=None, start_state=None, pretask=None, tqdm_kwargs : optional Any other arguments used for all bars. """ - super(TqdmCallback, self).__init__( - start=start, start_state=start_state, pretask=pretask, - posttask=posttask, finish=finish) + super(TqdmCallback, self).__init__(start=start, pretask=pretask) if tqdm_kwargs: tqdm_class = partial(tqdm_class, **tqdm_kwargs) self.tqdm_class = tqdm_class @@ -30,10 +27,10 @@ def _start_state(self, _, state): self.pbar = self.tqdm_class(total=sum( len(state[k]) for k in ['ready', 'waiting', 'running', 'finished'])) - def _posttask(self, *args, **kwargs): + def _posttask(self, *_, **__): self.pbar.update() - def _finish(self, *args, **kwargs): + def _finish(self, *_, **__): self.pbar.close() def display(self): From 28823f42884d48050e1a0a62a14b09005492051d Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 5 Mar 2021 13:05:00 +0000 Subject: [PATCH 3/5] misc documentation updates --- .meta/.readme.rst | 11 +++++++---- README.rst | 11 +++++++---- benchmarks/README.md | 2 +- tqdm/dask.py | 10 +++++++--- tqdm/gui.py | 6 +----- tqdm/keras.py | 4 ++-- tqdm/rich.py | 6 +----- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/.meta/.readme.rst b/.meta/.readme.rst index 4b607bec8..d231ffb69 100644 --- a/.meta/.readme.rst +++ b/.meta/.readme.rst @@ -140,7 +140,7 @@ Changelog The list of all changes is available either on GitHub's Releases: |GitHub-Status|, on the `wiki `__, or on the -`website `__. +`website `__. Usage @@ -460,7 +460,10 @@ Submodules """`rich.progress` version.""" class tqdm.keras.TqdmCallback(keras.callbacks.Callback): - """`keras` callback for epoch and batch progress.""" + """Keras callback for epoch and batch progress.""" + + class tqdm.dask.TqdmCallback(dask.callbacks.Callback): + """Dask callback for task progress.""" ``contrib`` @@ -470,8 +473,8 @@ The ``tqdm.contrib`` package also contains experimental modules: - ``tqdm.contrib.itertools``: Thin wrappers around ``itertools`` - ``tqdm.contrib.concurrent``: Thin wrappers around ``concurrent.futures`` -- ``tqdm.contrib.discord``: Posts to `Discord `__ bots -- ``tqdm.contrib.telegram``: Posts to `Telegram `__ bots +- ``tqdm.contrib.discord``: Posts to `Discord `__ bots +- ``tqdm.contrib.telegram``: Posts to `Telegram `__ bots - ``tqdm.contrib.bells``: Automagically enables all optional features * ``auto``, ``pandas``, ``discord``, ``telegram`` diff --git a/README.rst b/README.rst index b80c2d637..97674698a 100644 --- a/README.rst +++ b/README.rst @@ -140,7 +140,7 @@ Changelog The list of all changes is available either on GitHub's Releases: |GitHub-Status|, on the `wiki `__, or on the -`website `__. +`website `__. Usage @@ -679,7 +679,10 @@ Submodules """`rich.progress` version.""" class tqdm.keras.TqdmCallback(keras.callbacks.Callback): - """`keras` callback for epoch and batch progress.""" + """Keras callback for epoch and batch progress.""" + + class tqdm.dask.TqdmCallback(dask.callbacks.Callback): + """Dask callback for task progress.""" ``contrib`` @@ -689,8 +692,8 @@ The ``tqdm.contrib`` package also contains experimental modules: - ``tqdm.contrib.itertools``: Thin wrappers around ``itertools`` - ``tqdm.contrib.concurrent``: Thin wrappers around ``concurrent.futures`` -- ``tqdm.contrib.discord``: Posts to `Discord `__ bots -- ``tqdm.contrib.telegram``: Posts to `Telegram `__ bots +- ``tqdm.contrib.discord``: Posts to `Discord `__ bots +- ``tqdm.contrib.telegram``: Posts to `Telegram `__ bots - ``tqdm.contrib.bells``: Automagically enables all optional features * ``auto``, ``pandas``, ``discord``, ``telegram`` diff --git a/benchmarks/README.md b/benchmarks/README.md index a4e2e5289..235cb3197 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -11,7 +11,7 @@ These benchmarks serve two purposes: - [`progressbar2`](https://pypi.org/project/progressbar2) - [`alive-progress`](https://pypi.org/project/alive-progress) -Performance graphs are available at +Performance graphs are available at ## Running diff --git a/tqdm/dask.py b/tqdm/dask.py index f07d1b9ba..730a5e647 100644 --- a/tqdm/dask.py +++ b/tqdm/dask.py @@ -1,13 +1,17 @@ from __future__ import absolute_import -from .auto import tqdm as tqdm_auto + from functools import partial + from dask.callbacks import Callback + +from .auto import tqdm as tqdm_auto + __author__ = {"github.com/": ["casperdcl"]} __all__ = ['TqdmCallback'] class TqdmCallback(Callback): - """`dask` callback for task progress""" + """Dask callback for task progress.""" def __init__(self, start=None, pretask=None, tqdm_class=tqdm_auto, **tqdm_kwargs): """ @@ -34,7 +38,7 @@ def _finish(self, *_, **__): self.pbar.close() def display(self): - """displays in the current cell in Notebooks""" + """Displays in the current cell in Notebooks.""" container = getattr(self.bar, 'container', None) if container is None: return diff --git a/tqdm/gui.py b/tqdm/gui.py index 1daab1ef9..4612701d2 100644 --- a/tqdm/gui.py +++ b/tqdm/gui.py @@ -24,12 +24,8 @@ class tqdm_gui(std_tqdm): # pragma: no cover - """ - Experimental Matplotlib GUI version of tqdm! - """ - + """Experimental Matplotlib GUI version of tqdm!""" # TODO: @classmethod: write() on GUI? - def __init__(self, *args, **kwargs): from collections import deque diff --git a/tqdm/keras.py b/tqdm/keras.py index 8587bb09e..4a808f15f 100644 --- a/tqdm/keras.py +++ b/tqdm/keras.py @@ -17,7 +17,7 @@ class TqdmCallback(keras.callbacks.Callback): - """`keras` callback for epoch and batch progress""" + """Keras callback for epoch and batch progress.""" @staticmethod def bar2callback(bar, pop=None, delta=(lambda logs: 1)): def callback(_, logs=None): @@ -98,7 +98,7 @@ def on_train_end(self, *_, **__): self.epoch_bar.close() def display(self): - """displays in the current cell in Notebooks""" + """Displays in the current cell in Notebooks.""" container = getattr(self.epoch_bar, 'container', None) if container is None: return diff --git a/tqdm/rich.py b/tqdm/rich.py index 944691b9a..0ad6545a6 100644 --- a/tqdm/rich.py +++ b/tqdm/rich.py @@ -74,12 +74,8 @@ def render(self, task): class tqdm_rich(std_tqdm): # pragma: no cover - """ - Experimental rich.progress GUI version of tqdm! - """ - + """Experimental rich.progress GUI version of tqdm!""" # TODO: @classmethod: write()? - def __init__(self, *args, **kwargs): """ This class accepts the following parameters *in addition* to From 3cab8835ad70ef0e7395f0b4fdd2cec1251e0996 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 5 Mar 2021 13:17:39 +0000 Subject: [PATCH 4/5] tests: add `dask` --- environment.yml | 1 + tests/tests_dask.py | 19 +++++++++++++++++++ tox.ini | 1 + 3 files changed, 21 insertions(+) create mode 100644 tests/tests_dask.py diff --git a/environment.yml b/environment.yml index a88f03f14..2132fdc10 100644 --- a/environment.yml +++ b/environment.yml @@ -25,6 +25,7 @@ dependencies: - flake8-comprehensions - coverage # extras +- dask # dask - matplotlib # gui - numpy # pandas, keras, contrib.tenumerate - pandas diff --git a/tests/tests_dask.py b/tests/tests_dask.py new file mode 100644 index 000000000..2834abd61 --- /dev/null +++ b/tests/tests_dask.py @@ -0,0 +1,19 @@ +from __future__ import division + +from time import sleep + +from .tests_tqdm import importorskip, mark + +pytestmark = mark.slow + + +def test_dask(capsys): + """Test tqdm.dask.TqdmCallback""" + ProgressBar = importorskip('tqdm.dask').TqdmCallback + dask = importorskip('dask') + + schedule = [dask.delayed(sleep)(i / 10) for i in range(5)] + with ProgressBar(): + dask.compute(schedule) + _, err = capsys.readouterr() + assert '5/5' in err diff --git a/tox.ini b/tox.ini index 18f043e71..3ecdfa0ac 100644 --- a/tox.ini +++ b/tox.ini @@ -27,6 +27,7 @@ passenv=TOXENV CI GITHUB_* CODECOV_* COVERALLS_* CODACY_* HOME deps= {[core]deps} cython + dask[delayed] matplotlib numpy pandas From 7c42d809db512292f4e0b46e97320f3c66633ccf Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 5 Mar 2021 13:35:53 +0000 Subject: [PATCH 5/5] document `dask` --- .meta/.readme.rst | 18 ++++++++++++++++++ README.rst | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/.meta/.readme.rst b/.meta/.readme.rst index d231ffb69..97bf461fc 100644 --- a/.meta/.readme.rst +++ b/.meta/.readme.rst @@ -832,6 +832,24 @@ A ``keras`` callback is also available: model.fit(..., verbose=0, callbacks=[TqdmCallback()]) +Dask Integration +~~~~~~~~~~~~~~~~ + +A ``dask`` callback is also available: + +.. code:: python + + from tqdm.dask import TqdmCallback + + with TqdmCallback(desc="compute"): + ... + arr.compute() + + # or use callback globally + cb = TqdmCallback(desc="global") + cb.register() + arr.compute() + IPython/Jupyter Integration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index 97674698a..dd7e5a381 100644 --- a/README.rst +++ b/README.rst @@ -1051,6 +1051,24 @@ A ``keras`` callback is also available: model.fit(..., verbose=0, callbacks=[TqdmCallback()]) +Dask Integration +~~~~~~~~~~~~~~~~ + +A ``dask`` callback is also available: + +.. code:: python + + from tqdm.dask import TqdmCallback + + with TqdmCallback(desc="compute"): + ... + arr.compute() + + # or use callback globally + cb = TqdmCallback(desc="global") + cb.register() + arr.compute() + IPython/Jupyter Integration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~