diff --git a/.readme.rst b/.readme.rst new file mode 100644 index 000000000..7e38f4cab --- /dev/null +++ b/.readme.rst @@ -0,0 +1,851 @@ +|Logo| + +tqdm +==== + +|PyPI-Versions| |PyPI-Status| |Conda-Forge-Status| + +|Build-Status| |Coverage-Status| |Branch-Coverage-Status| |Codacy-Grade| |Libraries-Rank| |PyPI-Downloads| + +|DOI-URI| |LICENCE| |OpenHub-Status| |interactive-demo| + + +``tqdm`` means "progress" in Arabic (taqadum, تقدّم) +and is an abbreviation for "I love you so much" in Spanish (te quiero demasiado). + +Instantly make your loops show a smart progress meter - just wrap any +iterable with ``tqdm(iterable)``, and you're done! + +.. code:: python + + from tqdm import tqdm + for i in tqdm(range(10000)): + ... + +``76%|████████████████████████████         | 7568/10000 [00:33<00:10, 229.00it/s]`` + +``trange(N)`` can be also used as a convenient shortcut for +``tqdm(xrange(N))``. + +|Screenshot| + REPL: `ptpython `__ + +It can also be executed as a module with pipes: + +.. code:: sh + + $ seq 9999999 | tqdm --bytes | wc -l + 75.2MB [00:00, 217MB/s] + 9999999 + $ 7z a -bd -r backup.7z docs/ | grep Compressing | \ + tqdm --total $(find docs/ -type f | wc -l) --unit files >> backup.log + 100%|███████████████████████████████▉| 8014/8014 [01:37<00:00, 82.29files/s] + +Overhead is low -- about 60ns per iteration (80ns with ``tqdm_gui``), and is +unit tested against performance regression. +By comparison, the well-established +`ProgressBar `__ has +an 800ns/iter overhead. + +In addition to its low overhead, ``tqdm`` uses smart algorithms to predict +the remaining time and to skip unnecessary iteration displays, which allows +for a negligible overhead in most cases. + +``tqdm`` works on any platform +(Linux, Windows, Mac, FreeBSD, NetBSD, Solaris/SunOS), +in any console or in a GUI, and is also friendly with IPython/Jupyter notebooks. + +``tqdm`` does not require any dependencies (not even ``curses``!), just +Python and an environment supporting ``carriage return \r`` and +``line feed \n`` control characters. + +------------------------------------------ + +.. contents:: Table of contents + :backlinks: top + :local: + + +Installation +------------ + +Latest PyPI stable release +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +|PyPI-Status| |PyPI-Downloads| |Libraries-Dependents| + +.. code:: sh + + pip install tqdm + +Latest development release on GitHub +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +|GitHub-Status| |GitHub-Stars| |GitHub-Commits| |GitHub-Forks| |GitHub-Updated| + +Pull and install in the current directory: + +.. code:: sh + + pip install -e git+https://github.com/tqdm/tqdm.git@master#egg=tqdm + +Latest Conda release +~~~~~~~~~~~~~~~~~~~~ + +|Conda-Forge-Status| + +.. code:: sh + + conda install -c conda-forge tqdm + + +Changelog +--------- + +The list of all changes is available either on GitHub's Releases: +|GitHub-Status|, on the +`wiki `__, on the +`website `__, or on crawlers such as +`allmychanges.com `_. + + +Usage +----- + +``tqdm`` is very versatile and can be used in a number of ways. +The three main ones are given below. + +Iterable-based +~~~~~~~~~~~~~~ + +Wrap ``tqdm()`` around any iterable: + +.. code:: python + + from tqdm import tqdm + import time + + text = "" + for char in tqdm(["a", "b", "c", "d"]): + time.sleep(0.25) + text = text + char + +``trange(i)`` is a special optimised instance of ``tqdm(range(i))``: + +.. code:: python + + for i in trange(100): + time.sleep(0.01) + +Instantiation outside of the loop allows for manual control over ``tqdm()``: + +.. code:: python + + pbar = tqdm(["a", "b", "c", "d"]) + for char in pbar: + time.sleep(0.25) + pbar.set_description("Processing %s" % char) + +Manual +~~~~~~ + +Manual control on ``tqdm()`` updates by using a ``with`` statement: + +.. code:: python + + with tqdm(total=100) as pbar: + for i in range(10): + time.sleep(0.1) + pbar.update(10) + +If the optional variable ``total`` (or an iterable with ``len()``) is +provided, predictive stats are displayed. + +``with`` is also optional (you can just assign ``tqdm()`` to a variable, +but in this case don't forget to ``del`` or ``close()`` at the end: + +.. code:: python + + pbar = tqdm(total=100) + for i in range(10): + time.sleep(0.1) + pbar.update(10) + pbar.close() + +Module +~~~~~~ + +Perhaps the most wonderful use of ``tqdm`` is in a script or on the command +line. Simply inserting ``tqdm`` (or ``python -m tqdm``) between pipes will pass +through all ``stdin`` to ``stdout`` while printing progress to ``stderr``. + +The example below demonstrated counting the number of lines in all Python files +in the current directory, with timing information included. + +.. code:: sh + + $ time find . -name '*.py' -type f -exec cat \{} \; | wc -l + 857365 + + real 0m3.458s + user 0m0.274s + sys 0m3.325s + + $ time find . -name '*.py' -type f -exec cat \{} \; | tqdm | wc -l + 857366it [00:03, 246471.31it/s] + 857365 + + real 0m3.585s + user 0m0.862s + sys 0m3.358s + +Note that the usual arguments for ``tqdm`` can also be specified. + +.. code:: sh + + $ find . -name '*.py' -type f -exec cat \{} \; | + tqdm --unit loc --unit_scale --total 857366 >> /dev/null + 100%|███████████████████████████████████| 857K/857K [00:04<00:00, 246Kloc/s] + +Backing up a large directory? + +.. code:: sh + + $ 7z a -bd -r backup.7z docs/ | grep Compressing | + tqdm --total $(find docs/ -type f | wc -l) --unit files >> backup.log + 100%|███████████████████████████████▉| 8014/8014 [01:37<00:00, 82.29files/s] + + +FAQ and Known Issues +-------------------- + +|GitHub-Issues| + +The most common issues relate to excessive output on multiple lines, instead +of a neat one-line progress bar. + +- Consoles in general: require support for carriage return (``CR``, ``\r``). +- Nested progress bars: + * Consoles in general: require support for moving cursors up to the + previous line. For example, + `IDLE `__, + `ConEmu `__ and + `PyCharm `__ (also + `here `__, + `here `__, and + `here `__) + lack full support. + * Windows: additionally may require the Python module ``colorama`` + to ensure nested bars stay within their respective lines. +- Unicode: + * Environments which report that they support unicode will have solid smooth + progressbars. The fallback is an ```ascii``-only bar. + * Windows consoles often only partially support unicode and thus + `often require explicit ascii=True `__ + (also `here `__). This is due to + either normal-width unicode characters being incorrectly displayed as + "wide", or some unicode characters not rendering. +- Wrapping enumerated iterables: use ``enumerate(tqdm(...))`` instead of + ``tqdm(enumerate(...))``. The same applies to ``numpy.ndenumerate``. + This is because enumerate functions tend to hide the length of iterables. + ``tqdm`` does not. +- Wrapping zipped iterables has similar issues due to internal optimisations. + ``tqdm(zip(a, b))`` should be replaced with ``zip(tqdm(a), b)`` or even + ``zip(tqdm(a), tqdm(b))``. +- `Hanging pipes in python2 `__: + when using ``tqdm`` on the CLI, you may need to use python 3.5+ for correct + buffering. + +If you come across any other difficulties, browse and file |GitHub-Issues|. + +Documentation +------------- + +|PyPI-Versions| |README-Hits| (Since 19 May 2016) + +.. code:: python + + class tqdm(object): + """{DOC_tqdm}""" + + def __init__(self, iterable=None, desc=None, total=None, leave=True, + file=None, ncols=None, mininterval=0.1, + maxinterval=10.0, miniters=None, ascii=None, disable=False, + unit='it', unit_scale=False, dynamic_ncols=False, + smoothing=0.3, bar_format=None, initial=0, position=None, + postfix=None, unit_divisor=1000): + +Parameters +~~~~~~~~~~ + +{DOC_tqdm.tqdm.__init__.Parameters} +Extra CLI Options +~~~~~~~~~~~~~~~~~ + +{DOC_tqdm._main.CLI_EXTRA_DOC} +Returns +~~~~~~~ + +{DOC_tqdm.tqdm.__init__.Returns} +.. code:: python + + def update(self, n=1): + """ + Manually update the progress bar, useful for streams + such as reading files. + E.g.: + >>> t = tqdm(total=filesize) # Initialise + >>> for current_buffer in stream: + ... ... + ... t.update(len(current_buffer)) + >>> t.close() + The last line is highly recommended, but possibly not necessary if + ``t.update()`` will be called in such a way that ``filesize`` will be + exactly reached and printed. + + Parameters + ---------- + n : int, optional + Increment to add to the internal counter of iterations + [default: 1]. + """ + + def close(self): + """ + Cleanup and (if leave=False) close the progressbar. + """ + + def unpause(self): + """ + Restart tqdm timer from last print time. + """ + + def clear(self, nomove=False): + """ + Clear current bar display + """ + + def refresh(self): + """ + Force refresh the display of this bar + """ + + def write(cls, s, file=sys.stdout, end="\n"): + """ + Print a message via tqdm (without overlap with bars) + """ + + def set_description(self, desc=None, refresh=True): + """ + Set/modify description of the progress bar. + + Parameters + ---------- + desc : str, optional + refresh : bool, optional + Forces refresh [default: True]. + """ + + def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): + """ + Set/modify postfix (additional stats) + with automatic formatting based on datatype. + + Parameters + ---------- + refresh : bool, optional + Forces refresh [default: True]. + """ + + def trange(*args, **kwargs): + """ + A shortcut for tqdm(xrange(*args), **kwargs). + On Python3+ range is used instead of xrange. + """ + + class tqdm_gui(tqdm): + """ + Experimental GUI version of tqdm! + """ + + def tgrange(*args, **kwargs): + """ + Experimental GUI version of trange! + """ + + class tqdm_notebook(tqdm): + """ + Experimental IPython/Jupyter Notebook widget using tqdm! + """ + + def tnrange(*args, **kwargs): + """ + Experimental IPython/Jupyter Notebook widget using tqdm! + """ + + +Examples and Advanced Usage +--------------------------- + +- See the `examples `__ + folder; +- import the module and run ``help()``; +- consult the `wiki `__. + - this has an + `excellent article `__ + on how to make a **great** progressbar, or +- run the |interactive-demo|. + +Description and additional stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Custom information can be displayed and updated dynamically on ``tqdm`` bars +with the ``desc`` and ``postfix`` arguments: + +.. code:: python + + from tqdm import trange + from random import random, randint + from time import sleep + + with trange(10) as t: + for i in t: + # Description will be displayed on the left + t.set_description('GEN %i' % i) + # Postfix will be displayed on the right, + # formatted automatically based on argument's datatype + t.set_postfix(loss=random(), gen=randint(1,999), str='h', + lst=[1, 2]) + sleep(0.1) + + with tqdm(total=10, bar_format="{postfix[0]} {postfix[1][value]:>8.2g}", + postfix=["Batch", dict(value=0)]) as t: + for i in range(10): + sleep(0.1) + t.postfix[1]["value"] = i / 2 + t.update() + +Points to remember when using ``{postfix[...]}`` in the ``bar_format`` string: + +- ``postfix`` also needs to be passed as an initial argument in a compatible + format, and +- ``postfix`` will be auto-converted to a string if it is a ``dict``-like + object. To prevent this behaviour, insert an extra item into the dictionary + where the key is not a string. + +Additional ``bar_format`` parameters may also be defined by overriding +``format_dict``, and the bar itself may be modified using ``ascii``: + +.. code:: python + + from tqdm import tqdm + class TqdmExtraFormat(tqdm): + """Provides a `total_time` format parameter""" + @property + def format_dict(self): + d = super(TqdmExtraFormat, self).format_dict + total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) + d.update(total_time=self.format_interval(total_time) + " in total") + return d + + for i in TqdmExtraFormat( + range(10), ascii=" .oO0", + bar_format="{total_time}: {percentage:.0f}%|{bar}{r_bar}"): + pass + +.. code:: + + 00:01 in total: 40%|000o | 4/10 [00:00<00:00, 9.96it/s] + +Nested progress bars +~~~~~~~~~~~~~~~~~~~~ + +``tqdm`` supports nested progress bars. Here's an example: + +.. code:: python + + from tqdm import trange + from time import sleep + + for i in trange(4, desc='1st loop'): + for j in trange(5, desc='2nd loop'): + for k in trange(50, desc='3nd loop', leave=False): + sleep(0.01) + +On Windows `colorama `__ will be used if +available to keep nested bars on their respective lines. + +For manual control over positioning (e.g. for multi-threaded use), +you may specify ``position=n`` where ``n=0`` for the outermost bar, +``n=1`` for the next, and so on: + +.. code:: python + + from time import sleep + from tqdm import trange, tqdm + from multiprocessing import Pool, freeze_support, RLock + + L = list(range(9)) + + def progresser(n): + interval = 0.001 / (n + 2) + total = 5000 + text = "#{}, est. {:<04.2}s".format(n, interval * total) + for i in trange(total, desc=text, position=n): + sleep(interval) + + if __name__ == '__main__': + freeze_support() # for Windows support + p = Pool(len(L), + # again, for Windows support + initializer=tqdm.set_lock, initargs=(RLock(),)) + p.map(progresser, L) + print("\n" * (len(L) - 2)) + +Hooks and callbacks +~~~~~~~~~~~~~~~~~~~ + +``tqdm`` can easily support callbacks/hooks and manual updates. +Here's an example with ``urllib``: + +**urllib.urlretrieve documentation** + + | [...] + | If present, the hook function will be called once + | on establishment of the network connection and once after each block read + | thereafter. The hook will be passed three arguments; a count of blocks + | transferred so far, a block size in bytes, and the total size of the file. + | [...] + +.. code:: python + + import urllib, os + from tqdm import tqdm + + class TqdmUpTo(tqdm): + """Provides `update_to(n)` which uses `tqdm.update(delta_n)`.""" + def update_to(self, b=1, bsize=1, tsize=None): + """ + b : int, optional + Number of blocks transferred so far [default: 1]. + bsize : int, optional + Size of each block (in tqdm units) [default: 1]. + tsize : int, optional + Total size (in tqdm units). If [default: None] remains unchanged. + """ + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) # will also set self.n = b * bsize + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + with TqdmUpTo(unit='B', unit_scale=True, miniters=1, + desc=eg_link.split('/')[-1]) as t: # all optional kwargs + urllib.urlretrieve(eg_link, filename=os.devnull, + reporthook=t.update_to, data=None) + +Inspired by `twine#242 `__. +Functional alternative in +`examples/tqdm_wget.py `__. + +It is recommend to use ``miniters=1`` whenever there is potentially +large differences in iteration speed (e.g. downloading a file over +a patchy connection). + +Pandas Integration +~~~~~~~~~~~~~~~~~~ + +Due to popular demand we've added support for ``pandas`` -- here's an example +for ``DataFrame.progress_apply`` and ``DataFrameGroupBy.progress_apply``: + +.. code:: python + + import pandas as pd + import numpy as np + from tqdm import tqdm + + df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) + + # Register `pandas.progress_apply` and `pandas.Series.map_apply` with `tqdm` + # (can use `tqdm_gui`, `tqdm_notebook`, optional kwargs, etc.) + tqdm.pandas(desc="my bar!") + + # Now you can use `progress_apply` instead of `apply` + # and `progress_map` instead of `map` + df.progress_apply(lambda x: x**2) + # can also groupby: + # df.groupby(0).progress_apply(lambda x: x**2) + +In case you're interested in how this works (and how to modify it for your +own callbacks), see the +`examples `__ +folder or import the module and run ``help()``. + +IPython/Jupyter Integration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +IPython/Jupyter is supported via the ``tqdm_notebook`` submodule: + +.. code:: python + + from tqdm import tnrange, tqdm_notebook + from time import sleep + + for i in tnrange(3, desc='1st loop'): + for j in tqdm_notebook(range(100), desc='2nd loop'): + sleep(0.01) + +In addition to ``tqdm`` features, the submodule provides a native Jupyter +widget (compatible with IPython v1-v4 and Jupyter), fully working nested bars +and colour hints (blue: normal, green: completed, red: error/interrupt, +light blue: no ETA); as demonstrated below. + +|Screenshot-Jupyter1| +|Screenshot-Jupyter2| +|Screenshot-Jupyter3| + +It is also possible to let ``tqdm`` automatically choose between +console or notebook versions by using the ``autonotebook`` submodule: + +.. code:: python + + from tqdm.autonotebook import tqdm + tqdm.pandas() + +Note that this will issue a ``TqdmExperimentalWarning`` if run in a notebook +since it is not meant to be possible to distinguish between ``jupyter notebook`` +and ``jupyter console``. Use ``auto`` instead of ``autonotebook`` to suppress +this warning. + +Custom Integration +~~~~~~~~~~~~~~~~~~ + +``tqdm`` may be inherited from to create custom callbacks (as with the +``TqdmUpTo`` example `above <#hooks-and-callbacks>`__) or for custom frontends +(e.g. GUIs such as notebook or plotting packages). In the latter case: + +1. ``def __init__()`` to call ``super().__init__(..., gui=True)`` to disable + terminal ``status_printer`` creation. +2. Redefine: ``close()``, ``clear()``, ``display()``. + +Consider overloading ``display()`` to use e.g. +``self.frontend(**self.format_dict)`` instead of ``self.sp(repr(self))``. + +Writing messages +~~~~~~~~~~~~~~~~ + +Since ``tqdm`` uses a simple printing mechanism to display progress bars, +you should not write any message in the terminal using ``print()`` while +a progressbar is open. + +To write messages in the terminal without any collision with ``tqdm`` bar +display, a ``.write()`` method is provided: + +.. code:: python + + from tqdm import tqdm, trange + from time import sleep + + bar = trange(10) + for i in bar: + # Print using tqdm class method .write() + sleep(0.1) + if not (i % 3): + tqdm.write("Done task %i" % i) + # Can also use bar.write() + +By default, this will print to standard output ``sys.stdout``. but you can +specify any file-like object using the ``file`` argument. For example, this +can be used to redirect the messages writing to a log file or class. + +Redirecting writing +~~~~~~~~~~~~~~~~~~~ + +If using a library that can print messages to the console, editing the library +by replacing ``print()`` with ``tqdm.write()`` may not be desirable. +In that case, redirecting ``sys.stdout`` to ``tqdm.write()`` is an option. + +To redirect ``sys.stdout``, create a file-like class that will write +any input string to ``tqdm.write()``, and supply the arguments +``file=sys.stdout, dynamic_ncols=True``. + +A reusable canonical example is given below: + +.. code:: python + + from time import sleep + import contextlib + import sys + from tqdm import tqdm + + class DummyTqdmFile(object): + """Dummy file-like that will write to tqdm""" + file = None + def __init__(self, file): + self.file = file + + def write(self, x): + # Avoid print() second call (useless \n) + if len(x.rstrip()) > 0: + tqdm.write(x, file=self.file) + + def flush(self): + return getattr(self.file, "flush", lambda: None)() + + @contextlib.contextmanager + def std_out_err_redirect_tqdm(): + orig_out_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = map(DummyTqdmFile, orig_out_err) + yield orig_out_err[0] + # Relay exceptions + except Exception as exc: + raise exc + # Always restore sys.stdout/err if necessary + finally: + sys.stdout, sys.stderr = orig_out_err + + def some_fun(i): + print("Fee, fi, fo,".split()[i]) + + # Redirect stdout to tqdm.write() (don't forget the `as save_stdout`) + with std_out_err_redirect_tqdm() as orig_stdout: + # tqdm needs the original stdout + # and dynamic_ncols=True to autodetect console width + for i in tqdm(range(3), file=orig_stdout, dynamic_ncols=True): + sleep(.5) + some_fun(i) + + # After the `with`, printing is restored + print("Done!") + +Monitoring thread, intervals and miniters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``tqdm`` implements a few tricks to to increase efficiency and reduce overhead. + +- Avoid unnecessary frequent bar refreshing: ``mininterval`` defines how long + to wait between each refresh. ``tqdm`` always gets updated in the background, + but it will diplay only every ``mininterval``. +- Reduce number of calls to check system clock/time. +- ``mininterval`` is more intuitive to configure than ``miniters``. + A clever adjustment system ``dynamic_miniters`` will automatically adjust + ``miniters`` to the amount of iterations that fit into time ``mininterval``. + Essentially, ``tqdm`` will check if it's time to print without actually + checking time. This behaviour can be still be bypassed by manually setting + ``miniters``. + +However, consider a case with a combination of fast and slow iterations. +After a few fast iterations, ``dynamic_miniters`` will set ``miniters`` to a +large number. When iteration rate subsequently slows, ``miniters`` will +remain large and thus reduce display update frequency. To address this: + +- ``maxinterval`` defines the maximum time between display refreshes. + A concurrent monitoring thread checks for overdue updates and forces one + where necessary. + +The monitoring thread should not have a noticeable overhead, and guarantees +updates at least every 10 seconds by default. +This value can be directly changed by setting the ``monitor_interval`` of +any ``tqdm`` instance (i.e. ``t = tqdm.tqdm(...); t.monitor_interval = 2``). +The monitor thread may be disabled application-wide by setting +``tqdm.tqdm.monitor_interval = 0`` before instantiatiation of any ``tqdm`` bar. + + +Contributions +------------- + +|GitHub-Commits| |GitHub-Issues| |GitHub-PRs| |OpenHub-Status| + +All source code is hosted on `GitHub `__. +Contributions are welcome. + +See the +`CONTRIBUTING `__ +file for more information. + +Ports to Other Languages +~~~~~~~~~~~~~~~~~~~~~~~~ + +A list is available on +`this wiki page `__. + + +LICENCE +------- + +Open Source (OSI approved): |LICENCE| + +Citation information: |DOI-URI| + + +Authors +------- + +The main developers, ranked by surviving lines of code +(`git fame -wMC --excl '\.(png|gif)$' `__), are: + +- Casper da Costa-Luis (`casperdcl `__, ~2/3, |Gift-Casper|) +- Stephen Larroque (`lrq3000 `__, ~1/5) +- Matthew Stevens (`mjstevens777 `__, ~2%) +- Noam Yorav-Raphael (`noamraph `__, ~2%, original author) +- Guangshuo Chen (`chengs `__, ~1%) +- Hadrien Mary (`hadim `__, ~1%) +- Mikhail Korobov (`kmike `__, ~1%) + +There are also many |GitHub-Contributions| which we are grateful for. + +|README-Hits| (Since 19 May 2016) + +.. |Logo| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif +.. |Screenshot| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm.gif +.. |Build-Status| image:: https://img.shields.io/travis/tqdm/tqdm/master.svg?logo=travis + :target: https://travis-ci.org/tqdm/tqdm +.. |Coverage-Status| image:: https://coveralls.io/repos/tqdm/tqdm/badge.svg?branch=master + :target: https://coveralls.io/github/tqdm/tqdm +.. |Branch-Coverage-Status| image:: https://codecov.io/gh/tqdm/tqdm/branch/master/graph/badge.svg + :target: https://codecov.io/gh/tqdm/tqdm +.. |Codacy-Grade| image:: https://api.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177 + :target: https://www.codacy.com/app/tqdm/tqdm/dashboard +.. |GitHub-Status| image:: https://img.shields.io/github/tag/tqdm/tqdm.svg?maxAge=86400&logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/releases +.. |GitHub-Forks| image:: https://img.shields.io/github/forks/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/network +.. |GitHub-Stars| image:: https://img.shields.io/github/stars/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/stargazers +.. |GitHub-Commits| image:: https://img.shields.io/github/commit-activity/y/tqdm/tqdm.svg?logo=git&logoColor=white + :target: https://github.com/tqdm/tqdm/graphs/commit-activity +.. |GitHub-Issues| image:: https://img.shields.io/github/issues-closed/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/issues +.. |GitHub-PRs| image:: https://img.shields.io/github/issues-pr-closed/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/pulls +.. |GitHub-Contributions| image:: https://img.shields.io/github/contributors/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/graphs/contributors +.. |GitHub-Updated| image:: https://img.shields.io/github/last-commit/tqdm/tqdm/master.svg?logo=github&logoColor=white&label=pushed + :target: https://github.com/tqdm/tqdm/pulse +.. |Gift-Casper| image:: https://img.shields.io/badge/gift-donate-ff69b4.svg + :target: https://caspersci.uk.to/donate.html +.. |PyPI-Status| image:: https://img.shields.io/pypi/v/tqdm.svg + :target: https://pypi.org/project/tqdm +.. |PyPI-Downloads| image:: https://img.shields.io/pypi/dm/tqdm.svg?label=pypi%20downloads&logo=python&logoColor=white + :target: https://pypi.org/project/tqdm +.. |PyPI-Versions| image:: https://img.shields.io/pypi/pyversions/tqdm.svg?logo=python&logoColor=white + :target: https://pypi.org/project/tqdm +.. |Conda-Forge-Status| image:: https://img.shields.io/conda/v/conda-forge/tqdm.svg?label=conda-forge + :target: https://anaconda.org/conda-forge/tqdm +.. |Libraries-Rank| image:: https://img.shields.io/librariesio/sourcerank/pypi/tqdm.svg?logo=koding&logoColor=white + :target: https://libraries.io/pypi/tqdm +.. |Libraries-Dependents| image:: https://img.shields.io/librariesio/dependent-repos/pypi/tqdm.svg?logo=koding&logoColor=white + :target: https://github.com/tqdm/tqdm/network/dependents +.. |OpenHub-Status| image:: https://www.openhub.net/p/tqdm/widgets/project_thin_badge?format=gif + :target: https://www.openhub.net/p/tqdm?ref=Thin+badge +.. |LICENCE| image:: https://img.shields.io/pypi/l/tqdm.svg + :target: https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE +.. |DOI-URI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.595120-blue.svg + :target: https://doi.org/10.5281/zenodo.595120 +.. |interactive-demo| image:: https://img.shields.io/badge/demo-interactive-orange.svg?logo=jupyter + :target: https://notebooks.rmotr.com/demo/gh/tqdm/tqdm +.. |Screenshot-Jupyter1| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-1.gif +.. |Screenshot-Jupyter2| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-2.gif +.. |Screenshot-Jupyter3| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-3.gif +.. |README-Hits| image:: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&style=social&r=https://github.com/tqdm/tqdm&l=https://caspersci.uk.to/images/tqdm.png&f=https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif + :target: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot&r=https://github.com/tqdm/tqdm&l=https://caspersci.uk.to/images/tqdm.png&f=https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif&style=social diff --git a/.travis.yml b/.travis.yml index 6afe47609..b786f08a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: env: TOXENV=pypy3 - python: 3.6 env: TOXENV=flake8 - - python: 2.7 + - python: 3.6 env: TOXENV=perf # use cache for big builds like pandas (to minimise build time). # If issues, clear cache diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab2cccdbe..6d190c205 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,8 +28,8 @@ typical steps would be: 3. make a local clone: `git clone https://github.com/your_account/tqdm.git` 4. make changes on the local copy 5. test (see below) and commit changes `git commit -a -m "my message"` -6. `push` to your github account: `git push origin` -7. create a Pull Request (PR) from your github fork +6. `push` to your GitHub account: `git push origin` +7. create a Pull Request (PR) from your GitHub fork (go to your fork's webpage and click on "Pull Request." You can then add a message to describe your proposal.) @@ -103,15 +103,15 @@ Note: tools can be used to automate this process, such as ## Checking setup.py -To check that the `setup.py` file is compliant with PyPi requirements (e.g. -version number; reStructuredText in README.rst) use: +To check that the `setup.py` file is compliant with PyPI requirements (e.g. +version number; reStructuredText in `README.rst`) use: ``` [python setup.py] make testsetup ``` To upload just metadata (including overwriting mistakenly uploaded metadata) -to PyPi, use: +to PyPI, use: ``` [python setup.py] make pypimeta @@ -190,7 +190,7 @@ Formally publishing requires additional steps: testing and tagging. - ensure that all online CI tests have passed - check `setup.py` and `MANIFEST.in` - which define the packaging -process and info that will be uploaded to [pypi](https://pypi.org) - +process and info that will be uploaded to [PyPI](https://pypi.org) - using `[python setup.py] make installdev` ### Tag @@ -219,34 +219,35 @@ Finally, upload everything to pypi. This can be done easily using the [python setup.py] make pypi ``` -Also, the new release can (should) be added to `github` by creating a new +Also, the new release can (should) be added to GitHub by creating a new release from the web interface; uploading packages from the `dist/` folder created by `[python setup.py] make build`. -The [wiki] can be automatically updated with github release notes by +The [wiki] can be automatically updated with GitHub release notes by running `make` within the wiki repository. [wiki]: https://github.com/tqdm/tqdm/wiki ### Notes -- you can also test on the pypi test servers `test.pypi.org` +- you can also test on the PyPI test servers `test.pypi.org` before the real deployment -- in case of a mistake, you can delete an uploaded release on pypi, but you +- in case of a mistake, you can delete an uploaded release on PyPI, but you cannot re-upload another with the same version number -- in case of a mistake in the metadata on pypi (e.g. bad README), +- in case of a mistake in the metadata on PyPI (e.g. bad README), updating just the metadata is possible: `[python setup.py] make pypimeta` ## Updating Websites -The most important file is `README.rst`, which sould always be kept up-to-date +The most important file is `.readme.rst`, which should always be kept up-to-date and in sync with the in-line source documentation. This will affect all of the following: +- `README.rst` (generated by `mkdocs.py` during `make build`) - The [main repository site](https://github.com/tqdm/tqdm) which automatically - serves the latest `README.rst` as well as links to all of github's features. + serves the latest `README.rst` as well as links to all of GitHub's features. This is the preferred online referral link for `tqdm`. -- The [PyPi mirror](https://pypi.org/project/tqdm) which automatically +- The [PyPI mirror](https://pypi.org/project/tqdm) which automatically serves the latest release built from `README.rst` as well as links to past releases. - Many external web crawlers. @@ -277,7 +278,7 @@ For experienced devs, once happy with local master: 9. upload to PyPI using one of the following: a) `[python setup.py] make pypi` b) `twine upload -s -i $(git config user.signingkey) dist/tqdm-*` -10. create new release on https://github.com/tqdm/tqdm/releases +10. create new release on a) add helpful release notes b) attach `dist/tqdm-*` binaries (usually only `*.whl*`) 11. run `make` in the `wiki` submodule to update release notes diff --git a/Makefile b/Makefile index 4e897f4a4..034310cf8 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ testnose: nosetests tqdm -d -v testsetup: + @make README.rst python setup.py check --restructuredtext --strict python setup.py make none @@ -80,6 +81,7 @@ viewasv: asv preview tqdm/tqdm.1: .tqdm.1.md + # TODO: add to mkdocs.py python -m tqdm --help | tail -n+5 |\ sed -r -e 's/\\/\\\\/g' \ -e 's/^ (--.*)=<(.*)> : (.*)$$/\n\\\1=*\2*\n: \3./' \ @@ -87,6 +89,9 @@ tqdm/tqdm.1: .tqdm.1.md cat "$<" - |\ pandoc -o "$@" -s -t man +README.rst: .readme.rst tqdm/_tqdm.py tqdm/_main.py + @python mkdocs.py + distclean: @+make coverclean @+make prebuildclean @@ -122,6 +127,7 @@ install: build: @make prebuildclean + @make testsetup python setup.py sdist bdist_wheel # python setup.py bdist_wininst @@ -129,7 +135,6 @@ pypi: twine upload dist/* buildupload: - @make testsetup @make build @make pypi diff --git a/README.rst b/README.rst index 1f2203fb0..f84343834 100644 --- a/README.rst +++ b/README.rst @@ -301,7 +301,7 @@ Parameters * file : ``io.TextIOWrapper`` or ``io.StringIO``, optional Specifies where to output the progress messages (default: sys.stderr). Uses ``file.write(str)`` and ``file.flush()`` - methods. + methods. For encoding, see ``write_bytes``. * ncols : int, optional The width of the entire output message. If specified, dynamically resizes the progressbar to stay within this bound. @@ -323,9 +323,9 @@ Parameters Tweak this and ``mininterval`` to get very efficient loops. If your progress is erratic with both fast and slow iterations (network, skipping items, etc) you should set miniters=1. -* ascii : bool, optional +* ascii : bool or str, optional If unspecified or False, use unicode (smooth blocks) to fill - the meter. The fallback is to use ASCII characters ``1-9 #``. + the meter. The fallback is to use ASCII characters " 123456789#". * disable : bool, optional Whether to disable the entire progressbar wrapper [default: False]. If set to None, disable on non-TTY. @@ -337,8 +337,8 @@ Parameters automatically and a metric prefix following the International System of Units standard will be added (kilo, mega, etc.) [default: False]. If any other non-zero + number, will scale ``total`` and ``n``. * dynamic_ncols : bool, optional - number, will scale ```total`` and ``n``. If set, constantly alters ``ncols`` to the environment (allowing for window resizes) [default: False]. * smoothing : float, optional @@ -353,7 +353,8 @@ Parameters '{rate_fmt}{postfix}]' Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, percentage, rate, rate_fmt, rate_noinv, rate_noinv_fmt, - rate_inv, rate_inv_fmt, elapsed, remaining, desc, postfix. + rate_inv, rate_inv_fmt, elapsed, elapsed_s, remaining, + remaining_s, desc, postfix, unit. Note that a trailing ": " is automatically removed after {desc} if the latter is empty. * initial : int, optional @@ -368,6 +369,10 @@ Parameters Calls ``set_postfix(**postfix)`` if possible (dict). * unit_divisor : float, optional [default: 1000], ignored unless ``unit_scale`` is True. +* write_bytes : bool, optional + If (default: None) and ``file`` is unspecified, + bytes will be written in Python 2. If ``True`` will also write + bytes. In all other cases will default to unicode. Extra CLI Options ~~~~~~~~~~~~~~~~~ @@ -379,7 +384,8 @@ Extra CLI Options String buffer size in bytes [default: 256] used when ``delim`` is specified. * bytes : bool, optional - If true, will count bytes and ignore ``delim``. + If true, will count bytes, ignore ``delim``, and default + ``unit_scale`` to True, ``unit_divisor`` to 1024, and ``unit`` to 'B'. * manpath : str, optional Directory in which to install tqdm man pages. * log : str, optional @@ -537,7 +543,7 @@ Points to remember when using ``{postfix[...]}`` in the ``bar_format`` string: where the key is not a string. Additional ``bar_format`` parameters may also be defined by overriding -``format_dict``: +``format_dict``, and the bar itself may be modified using ``ascii``: .. code:: python @@ -552,10 +558,14 @@ Additional ``bar_format`` parameters may also be defined by overriding return d for i in TqdmExtraFormat( - range(10), + range(10), ascii=" .oO0", bar_format="{total_time}: {percentage:.0f}%|{bar}{r_bar}"): pass +.. code:: + + 00:01 in total: 40%|000o | 4/10 [00:00<00:00, 9.96it/s] + Nested progress bars ~~~~~~~~~~~~~~~~~~~~ @@ -715,6 +725,20 @@ since it is not meant to be possible to distinguish between ``jupyter notebook`` and ``jupyter console``. Use ``auto`` instead of ``autonotebook`` to suppress this warning. +Custom Integration +~~~~~~~~~~~~~~~~~~ + +``tqdm`` may be inherited from to create custom callbacks (as with the +``TqdmUpTo`` example `above <#hooks-and-callbacks>`__) or for custom frontends +(e.g. GUIs such as notebook or plotting packages). In the latter case: + +1. ``def __init__()`` to call ``super().__init__(..., gui=True)`` to disable + terminal ``status_printer`` creation. +2. Redefine: ``close()``, ``clear()``, ``display()``. + +Consider overloading ``display()`` to use e.g. +``self.frontend(**self.format_dict)`` instead of ``self.sp(repr(self))``. + Writing messages ~~~~~~~~~~~~~~~~ diff --git a/mkdocs.py b/mkdocs.py new file mode 100644 index 000000000..647e76f5c --- /dev/null +++ b/mkdocs.py @@ -0,0 +1,58 @@ +from __future__ import print_function +import tqdm +from textwrap import dedent +from io import open as io_open +from os import path + +HEAD_ARGS = """ +Parameters +---------- +""" +HEAD_RETS = """ +Returns +------- +""" +HEAD_CLI = """ +Extra CLI Options +----------------- +name : type, optional + TODO: find out why this is needed. +""" + + +def doc2rst(doc, arglist=True): + """ + arglist : bool, whether to create argument lists + """ + doc = dedent(doc).replace('`', '``') + if arglist: + doc = '\n'.join([i if not i or i[0] == ' ' else '* ' + i + ' ' + for i in doc.split('\n')]) + return doc + + +src_dir = path.abspath(path.dirname(__file__)) +README_rst = path.join(src_dir, '.readme.rst') +with io_open(README_rst, mode='r', encoding='utf-8') as fd: + README_rst = fd.read() +DOC_tqdm = doc2rst(tqdm.tqdm.__doc__, False).replace('\n', '\n ') +DOC_tqdm_init = doc2rst(tqdm.tqdm.__init__.__doc__) +DOC_tqdm_init_args = DOC_tqdm_init.partition(doc2rst(HEAD_ARGS))[-1]\ + .replace('\n ', '\n ') +DOC_tqdm_init_args, _, DOC_tqdm_init_rets = DOC_tqdm_init_args\ + .partition(doc2rst(HEAD_RETS)) +DOC_cli = doc2rst(tqdm._main.CLI_EXTRA_DOC).partition(doc2rst(HEAD_CLI))[-1] + +# special cases +DOC_tqdm_init_args = DOC_tqdm_init_args.replace(' *,', ' ``*``,') +DOC_tqdm_init_args = DOC_tqdm_init_args.partition('* gui : bool, optional')[0] + +README_rst = README_rst.replace('{DOC_tqdm}', DOC_tqdm)\ + .replace('{DOC_tqdm.tqdm.__init__.Parameters}', DOC_tqdm_init_args)\ + .replace('{DOC_tqdm._main.CLI_EXTRA_DOC}', DOC_cli)\ + .replace('{DOC_tqdm.tqdm.__init__.Returns}', DOC_tqdm_init_rets) + +if __name__ == "__main__": + fndoc = path.join(src_dir, 'README.rst') + with io_open(fndoc, mode='w', encoding='utf-8') as fd: + fd.write(README_rst) diff --git a/tqdm/_main.py b/tqdm/_main.py index 0a6117487..e4cba11c3 100644 --- a/tqdm/_main.py +++ b/tqdm/_main.py @@ -99,7 +99,7 @@ def posix_pipe(fin, fout, delim='\n', buf_size=256, Extra CLI Options ----------------- name : type, optional - TODO: find out why this is needed. + TODO: find out why this is needed. delim : chr, optional Delimiting character [default: '\n']. Use '\0' for null. N.B.: on Windows systems, Python converts '\n' to '\r\n'. diff --git a/tqdm/_tqdm.py b/tqdm/_tqdm.py index 8eac3285d..945f909b7 100755 --- a/tqdm/_tqdm.py +++ b/tqdm/_tqdm.py @@ -13,7 +13,7 @@ # compatibility functions and utilities from ._utils import _supports_unicode, _environ_cols_wrapper, _range, _unich, \ _term_move_up, _unicode, WeakSet, _basestring, _OrderedDict, \ - Comparable, RE_ANSI + Comparable, RE_ANSI, _is_ascii, SimpleTextIOWrapper from ._monitor import TMonitor # native libraries import sys @@ -125,6 +125,9 @@ def create_th_lock(cls): # context and does not allow the user to use 'spawn' or 'forkserver' methods. TqdmDefaultWriteLock.create_th_lock() +ASCII_FMT = " 123456789#" +UTF_FMT = u" " + u''.join(map(_unich, range(0x258F, 0x2587, -1))) + class tqdm(Comparable): """ @@ -274,10 +277,10 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, prefix : str, optional Prefix message (included in total width) [default: '']. Use as {desc} in bar_format string. - ascii : bool, optional + ascii : bool, optional or str, optional If not set, use unicode (smooth blocks) to fill the meter [default: False]. The fallback is to use ASCII characters - (1-9 #). + " 123456789#". unit : str, optional The iteration unit [default: 'it']. unit_scale : bool or int or float, optional @@ -296,7 +299,8 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, '{rate_fmt}{postfix}]' Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, percentage, rate, rate_fmt, rate_noinv, rate_noinv_fmt, - rate_inv, rate_inv_fmt, elapsed, remaining, desc, postfix. + rate_inv, rate_inv_fmt, elapsed, elapsed_s, + remaining, remaining_s, desc, postfix, unit. Note that a trailing ": " is automatically removed after {desc} if the latter is empty. postfix : *, optional @@ -319,14 +323,14 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, # apply custom scale if necessary if unit_scale and unit_scale not in (True, 1): - total *= unit_scale + if total: + total *= unit_scale n *= unit_scale if rate: rate *= unit_scale # by default rate = 1 / self.avg_time unit_scale = False - format_interval = tqdm.format_interval - elapsed_str = format_interval(elapsed) + elapsed_str = tqdm.format_interval(elapsed) # if unspecified, attempt to use rate = average speed # (we allow manual override since predicting time is an arcane art) @@ -345,53 +349,59 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, if unit_scale: n_fmt = format_sizeof(n, divisor=unit_divisor) total_fmt = format_sizeof(total, divisor=unit_divisor) \ - if total else None + if total is not None else '?' else: n_fmt = str(n) - total_fmt = str(total) + total_fmt = str(total) if total is not None else '?' try: postfix = ', ' + postfix if postfix else '' except TypeError: pass + remaining = (total - n) / rate if rate and total else 0 + remaining_str = tqdm.format_interval(remaining) if rate else '?' + + # format the stats displayed to the left and right sides of the bar + if prefix: + # old prefix setup work around + bool_prefix_colon_already = (prefix[-2:] == ": ") + l_bar = prefix if bool_prefix_colon_already else prefix + ": " + else: + l_bar = '' + + r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format( + n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix) + + # Custom bar formatting + # Populate a dict with all available progress indicators + format_dict = dict( + n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt, + rate=inv_rate if inv_rate and inv_rate > 1 else rate, + rate_fmt=rate_fmt, rate_noinv=rate, + rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate, + rate_inv_fmt=rate_inv_fmt, + elapsed=elapsed_str, elapsed_s=elapsed, + remaining=remaining_str, remaining_s=remaining, + l_bar=l_bar, r_bar=r_bar, + desc=prefix or '', postfix=postfix, unit=unit, + # bar=full_bar, # replaced by procedure below + **extra_kwargs) + # total is known: we can predict some stats if total: # fractional and percentage progress frac = n / total percentage = frac * 100 - remaining_str = format_interval((total - n) / rate) \ - if rate else '?' - - # format the stats displayed to the left and right sides of the bar - if prefix: - # old prefix setup work around - bool_prefix_colon_already = (prefix[-2:] == ": ") - l_bar = prefix if bool_prefix_colon_already else prefix + ": " - else: - l_bar = '' l_bar += '{0:3.0f}%|'.format(percentage) - r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format( - n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix) if ncols == 0: return l_bar[:-1] + r_bar[1:] if bar_format: - # Custom bar formatting - # Populate a dict with all available progress indicators - format_dict = dict( - n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt, - percentage=percentage, - rate=inv_rate if inv_rate and inv_rate > 1 else rate, - rate_fmt=rate_fmt, rate_noinv=rate, - rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate, - rate_inv_fmt=rate_inv_fmt, elapsed=elapsed_str, - remaining=remaining_str, l_bar=l_bar, r_bar=r_bar, - desc=prefix or '', postfix=postfix, - # bar=full_bar, # replaced by procedure below - **extra_kwargs) + format_dict.update(l_bar=l_bar, percentage=percentage) + # , bar=full_bar # replaced by procedure below # auto-remove colon for empty `desc` if not prefix: @@ -415,34 +425,33 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, N_BARS = 10 # format bar depending on availability of unicode/ascii chars - if ascii: - bar_length, frac_bar_length = divmod( - int(frac * N_BARS * 10), 10) - - bar = '#' * bar_length - frac_bar = chr(48 + frac_bar_length) if frac_bar_length \ - else ' ' + if ascii is True: + ascii = ASCII_FMT + elif ascii is False: + ascii = UTF_FMT + nsyms = len(ascii) - 1 + bar_length, frac_bar_length = divmod( + int(frac * N_BARS * nsyms), nsyms) - else: - bar_length, frac_bar_length = divmod(int(frac * N_BARS * 8), 8) - - bar = _unich(0x2588) * bar_length - frac_bar = _unich(0x2590 - frac_bar_length) \ - if frac_bar_length else ' ' + bar = ascii[-1] * bar_length + frac_bar = ascii[frac_bar_length] # whitespace padding if bar_length < N_BARS: full_bar = bar + frac_bar + \ - ' ' * max(N_BARS - bar_length - 1, 0) + ascii[0] * (N_BARS - bar_length - 1) else: full_bar = bar + \ - ' ' * max(N_BARS - bar_length, 0) + ascii[0] * (N_BARS - bar_length) # Piece together the bar parts return l_bar + full_bar + r_bar - # no total: no progressbar, ETA, just progress stats + elif bar_format: + # user-specified bar_format but no total + return bar_format.format(bar='?', **format_dict) else: + # no total: no progressbar, ETA, just progress stats return ((prefix + ": ") if prefix else '') + \ '{0}{1} [{2}, {3}{4}]'.format( n_fmt, unit, elapsed_str, rate_fmt, postfix) @@ -701,7 +710,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, miniters=None, ascii=None, disable=False, unit='it', unit_scale=False, dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0, position=None, postfix=None, - unit_divisor=1000, gui=False, **kwargs): + unit_divisor=1000, write_bytes=None, gui=False, **kwargs): """ Parameters ---------- @@ -724,7 +733,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, file : `io.TextIOWrapper` or `io.StringIO`, optional Specifies where to output the progress messages (default: sys.stderr). Uses `file.write(str)` and `file.flush()` - methods. + methods. For encoding, see `write_bytes`. ncols : int, optional The width of the entire output message. If specified, dynamically resizes the progressbar to stay within this bound. @@ -746,9 +755,9 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, Tweak this and `mininterval` to get very efficient loops. If your progress is erratic with both fast and slow iterations (network, skipping items, etc) you should set miniters=1. - ascii : bool, optional + ascii : bool or str, optional If unspecified or False, use unicode (smooth blocks) to fill - the meter. The fallback is to use ASCII characters `1-9 #`. + the meter. The fallback is to use ASCII characters " 123456789#". disable : bool, optional Whether to disable the entire progressbar wrapper [default: False]. If set to None, disable on non-TTY. @@ -776,7 +785,8 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, '{rate_fmt}{postfix}]' Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, percentage, rate, rate_fmt, rate_noinv, rate_noinv_fmt, - rate_inv, rate_inv_fmt, elapsed, remaining, desc, postfix. + rate_inv, rate_inv_fmt, elapsed, elapsed_s, remaining, + remaining_s, desc, postfix, unit. Note that a trailing ": " is automatically removed after {desc} if the latter is empty. initial : int, optional @@ -791,6 +801,10 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, Calls `set_postfix(**postfix)` if possible (dict). unit_divisor : float, optional [default: 1000], ignored unless `unit_scale` is True. + write_bytes : bool, optional + If (default: None) and `file` is unspecified, + bytes will be written in Python 2. If `True` will also write + bytes. In all other cases will default to unicode. gui : bool, optional WARNING: internal parameter - do not use. Use tqdm_gui(...) instead. If set, will attempt to use @@ -800,10 +814,18 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, ------- out : decorated iterator. """ + if write_bytes is None: + write_bytes = file is None and sys.version_info < (3,) if file is None: file = sys.stderr + if write_bytes: + # Despite coercing unicode into bytes, py2 sys.std* streams + # should have bytes written to them. + file = SimpleTextIOWrapper( + file, encoding=getattr(file, 'encoding', 'utf-8')) + if disable is None and hasattr(file, "isatty") and not file.isatty(): disable = True @@ -868,7 +890,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, if ascii is None: ascii = not _supports_unicode(file) - if bar_format and not ascii: + if bar_format and not ((ascii is True) or _is_ascii(ascii)): # Convert bar format into unicode since terminal uses unicode bar_format = _unicode(bar_format) diff --git a/tqdm/_utils.py b/tqdm/_utils.py index 8aebc0ef7..4e9f536c8 100644 --- a/tqdm/_utils.py +++ b/tqdm/_utils.py @@ -142,6 +142,29 @@ def __ge__(self, other): return not self < other +class SimpleTextIOWrapper(object): + """ + Change only `.write()` of the wrapped object by encoding the passed + value and passing the result to the wrapped object's `.write()` method. + """ + # pylint: disable=too-few-public-methods + def __init__(self, wrapped, encoding): + object.__setattr__(self, '_wrapped', wrapped) + object.__setattr__(self, 'encoding', encoding) + + def write(self, s): + """ + Encode `s` and pass to the wrapped object's `.write()` method. + """ + return getattr(self, '_wrapped').write(s.encode(getattr(self, 'encoding'))) + + def __getattr__(self, name): + return getattr(self._wrapped, name) + + def __setattr__(self, name, value): # pragma: no cover + return setattr(self._wrapped, name, value) + + def _is_utf(encoding): try: u'\u2588\u2589'.encode(encoding) @@ -163,6 +186,15 @@ def _supports_unicode(fp): return False +def _is_ascii(s): + if isinstance(s, str): + for c in s: + if ord(c) > 255: + return False + return True + return _supports_unicode(s) + + def _environ_cols_wrapper(): # pragma: no cover """ Return a function which gets width and height of console diff --git a/tqdm/tests/tests_perf.py b/tqdm/tests/tests_perf.py index ac90aedc0..f2ca3a52d 100644 --- a/tqdm/tests/tests_perf.py +++ b/tqdm/tests/tests_perf.py @@ -298,7 +298,7 @@ def test_iter_overhead_simplebar_hard(): # Compute relative overhead of tqdm against native range() try: - assert time_tqdm() < 2.5 * time_bench() + assert time_tqdm() < 3 * time_bench() except AssertionError: raise AssertionError('trange(%g): %f, simple_progress(%g): %f' % (total, time_tqdm(), total, time_bench())) @@ -330,7 +330,7 @@ def test_manual_overhead_simplebar_hard(): # Compute relative overhead of tqdm against native range() try: - assert time_tqdm() < 2.5 * time_bench() + assert time_tqdm() < 3 * time_bench() except AssertionError: raise AssertionError('tqdm(%g): %f, simple_progress(%g): %f' % (total, time_tqdm(), total, time_bench())) diff --git a/tqdm/tests/tests_tqdm.py b/tqdm/tests/tests_tqdm.py index 998b00a08..1a642e435 100644 --- a/tqdm/tests/tests_tqdm.py +++ b/tqdm/tests/tests_tqdm.py @@ -1,8 +1,6 @@ # Advice: use repr(our_file.read()) to print the full output of tqdm # (else '\r' will replace the previous lines and you'll see only the latest. -from __future__ import unicode_literals - import sys import csv import re @@ -21,6 +19,7 @@ except ImportError: from io import StringIO +from io import BytesIO from io import IOBase # to support unicode strings @@ -69,16 +68,16 @@ def pos_line_diff(res_list, expected_list, raise_nonempty=True): Return differences between two bar output lists. To be used with `RE_pos` """ - l = len(res_list) - if l < len(expected_list): - res = [(None, e) for e in expected_list[l:]] - elif l > len(expected_list): - res = [(r, None) for r in res_list[l:]] + ln = len(res_list) + if ln < len(expected_list): + res = [(None, e) for e in expected_list[ln:]] + elif ln > len(expected_list): + res = [(r, None) for r in res_list[ln:]] res = [(r, e) for r, e in zip(res_list, expected_list) for pos in [len(e)-len(e.lstrip('\n'))] # bar position if not r.startswith(e) # start matches or not (r.endswith('\x1b[A' * pos) # move up at end - or r=='\n') # final bar + or r == '\n') # final bar or r[(-1-pos) * len('\x1b[A'):] == '\x1b[A'] # extra move up if res and raise_nonempty: raise AssertionError( @@ -340,6 +339,57 @@ def test_all_defaults(): sys.stderr.write('\rTest default kwargs ... ') +class WriteTypeChecker(BytesIO): + """File-like to assert the expected type is written""" + def __init__(self, expected_type): + super(WriteTypeChecker, self).__init__() + self.expected_type = expected_type + + def write(self, s): + assert isinstance(s, self.expected_type) + + +@with_setup(pretest, posttest) +def test_native_string_io_for_default_file(): + """Native strings written to unspecified files""" + stderr = sys.stderr + try: + sys.stderr = WriteTypeChecker(expected_type=type('')) + + for _ in tqdm(range(3)): + pass + finally: + sys.stderr = stderr + + +@with_setup(pretest, posttest) +def test_unicode_string_io_for_specified_file(): + """Unicode strings written to specified files""" + for _ in tqdm(range(3), file=WriteTypeChecker(expected_type=type(u''))): + pass + + +@with_setup(pretest, posttest) +def test_byte_string_io_for_specified_file_with_forced_bytes(): + """Byte strings written to specified files when forced""" + for _ in tqdm(range(3), file=WriteTypeChecker(expected_type=type(b'')), + write_bytes=True): + pass + + +@with_setup(pretest, posttest) +def test_unicode_string_io_for_unspecified_file_with_forced_unicode(): + """Unicode strings written to unspecified file when forced""" + stderr = sys.stderr + try: + sys.stderr = WriteTypeChecker(expected_type=type(u'')) + + for _ in tqdm(range(3), write_bytes=False): + pass + finally: + sys.stderr = stderr + + @with_setup(pretest, posttest) def test_iterate_over_csv_rows(): """Test csv iterator""" @@ -772,6 +822,21 @@ def test_infinite_total(): pass +@with_setup(pretest, posttest) +def test_nototal(): + """Test unknown total length""" + with closing(StringIO()) as our_file: + for i in tqdm((i for i in range(10)), file=our_file, unit_scale=10): + pass + assert "100it" in our_file.getvalue() + + with closing(StringIO()) as our_file: + for i in tqdm((i for i in range(10)), file=our_file, + bar_format="{l_bar}{bar}{r_bar}"): + pass + assert "10/?" in our_file.getvalue() + + @with_setup(pretest, posttest) def test_unit(): """Test SI unit prefix""" @@ -805,9 +870,19 @@ def test_ascii(): for _ in _range(3): t.update() res = our_file.getvalue().strip("\r").split("\r") - assert "7%|\u258b" in res[1] - assert "13%|\u2588\u258e" in res[2] - assert "20%|\u2588\u2588" in res[3] + assert u"7%|\u258b" in res[1] + assert u"13%|\u2588\u258e" in res[2] + assert u"20%|\u2588\u2588" in res[3] + + # Test custom bar + for ascii in [" .oO0", " #"]: + with closing(StringIO()) as our_file: + for _ in tqdm(_range(len(ascii) - 1), file=our_file, miniters=1, + mininterval=0, ascii=ascii, ncols=1): + pass + res = our_file.getvalue().strip("\r").split("\r") + for bar, line in zip(ascii, res): + assert '|' + bar + '|' in line @with_setup(pretest, posttest) @@ -1152,7 +1227,6 @@ def test_position(): '\n\n\rpos2 bar: 0%'] pos_line_diff(res, exres) - t2.close() t4 = tqdm(total=10, file=our_file, desc='pos3 bar', mininterval=0) t1.update(1)