diff --git a/.meta/mkdocs.py b/.meta/mkdocs.py index a5707f320..0d9ff8ddb 100644 --- a/.meta/mkdocs.py +++ b/.meta/mkdocs.py @@ -1,4 +1,7 @@ from __future__ import print_function +from os import path +import sys +sys.path = [path.dirname(path.dirname(__file__))] + sys.path # NOQA import tqdm import tqdm.cli from textwrap import dedent @@ -29,7 +32,6 @@ def doc2rst(doc, arglist=True, raw=False): doc = doc.replace('`', '``') if raw: doc = doc.replace('\n ', '\n ') - #doc = '\n'.join(i.rstrip() for i in doc.split('\n')) else: doc = dedent(doc) if arglist: diff --git a/.travis.yml b/.travis.yml index a048117eb..737edeef1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,34 +1,71 @@ language: python -matrix: +env: + global: + - PIP_CACHE_DIR="$HOME/.cache/pip" # unify pip cache location for all platforms +# use cache for big builds like pandas (to minimise build time). +# If issues, clear cache +# https://docs.travis-ci.com/user/caching/#Clearing-Caches +cache: + pip: true + directories: + - $HOME/.cache/pip +before_cache: +- rm -f $HOME/.cache/pip/log/debug.log +notifications: + email: false +# branches: # remove travis double-check on pull requests in main repo +# only: +# - master +# - /^\d\.\d+$/ +stages: +- check +- test +- name: deploy + if: repo = tqdm/tqdm +jobs: include: - - python: 2.6 + - name: py2.6 + python: 2.6 env: TOXENV=py26 dist: trusty - - python: 2.7 + - name: py2.7 + python: 2.7 env: TOXENV=py27 - - python: 3.4 + - name: py3.4 + python: 3.4 env: TOXENV=py34 - - python: 3.5 + - name: py3.5 + python: 3.5 env: TOXENV=py35 - - python: 3.6 + - name: py3.6 + python: 3.6 env: TOXENV=py36 - - python: 3.7 + - name: py3.7 + python: 3.7 env: TOXENV=py37 + - name: pypy2.7 + python: pypy2.7-5.10.0 + env: TOXENV=pypy + - name: pypy3.5 + python: pypy3.5-5.10.0 + env: TOXENV=pypy3 + - name: style + stage: check + python: 3.6 + env: TOXENV=flake8 + - name: setup + stage: check + python: 3.6 + env: TOXENV=setup.py + - name: perf + python: 3.6 + env: TOXENV=perf + - name: PyPI and GitHub + stage: deploy + python: 3.7 dist: xenial - sudo: true # required for py37, docker - services: - - docker - after_success: - - echo "$DOCKER_PWD" | docker login -u $DOCKER_USR --password-stdin - - echo "$GITHUB_TOKEN" | docker login docker.pkg.github.com -u $GITHUB_USR --password-stdin - - make -B docker - - | - if [[ -n "$TRAVIS_TAG" ]]; then - docker tag tqdm/tqdm:latest tqdm/tqdm:${TRAVIS_TAG#v} - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:${TRAVIS_TAG#v} ; fi - - docker tag tqdm/tqdm:latest tqdm/tqdm:devel - - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:latest - - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:devel + install: + script: - pip install .[dev] - make build #- make submodules @@ -37,21 +74,43 @@ matrix: -iv $encrypted_a6d6301302b7_iv -in .meta/.tqdm.gpg.enc -out .tqdm.gpg -d - gpg --import .tqdm.gpg - rm .tqdm.gpg + - git log --pretty='format:- %s%n%b---' $(git tag --sort=creatordate | tail -n2 | head -n1)..HEAD > CHANGES.md deploy: - provider: script script: twine upload -s -i tqdm@caspersci.uk.to dist/tqdm-* - skip_cleanup: true + cleanup: false on: tags: true - provider: releases api_key: $GITHUB_TOKEN file_glob: true file: dist/tqdm-*.whl* - skip_cleanup: true + cleanup: false draft: true name: tqdm $TRAVIS_TAG stable + edge: true + release_notes_file: CHANGES.md on: tags: true + - name: docker + stage: deploy + python: 3.7 + dist: xenial + services: + - docker + install: + script: + - echo "$DOCKER_PWD" | docker login -u $DOCKER_USR --password-stdin + - echo "$GITHUB_TOKEN" | docker login docker.pkg.github.com -u $GITHUB_USR --password-stdin + - make -B docker + - | + if [[ -n "$TRAVIS_TAG" ]]; then + docker tag tqdm/tqdm:latest tqdm/tqdm:${TRAVIS_TAG#v} + docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:${TRAVIS_TAG#v} ; fi + - docker tag tqdm/tqdm:latest tqdm/tqdm:devel + - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:latest + - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:devel + deploy: - provider: script script: docker push tqdm/tqdm:${TRAVIS_TAG#v} on: @@ -72,31 +131,6 @@ matrix: script: 'docker push docker.pkg.github.com/tqdm/tqdm/tqdm:devel || :' on: branch: devel - - python: pypy2.7-5.10.0 - env: TOXENV=pypy - - python: pypy3.5-5.10.0 - env: TOXENV=pypy3 - - python: 3.6 - env: TOXENV=flake8 - - python: 3.6 - env: TOXENV=setup.py - - python: 3.6 - env: TOXENV=perf -# use cache for big builds like pandas (to minimise build time). -# If issues, clear cache -# https://docs.travis-ci.com/user/caching/#Clearing-Caches -cache: - pip: true - directories: - - $HOME/.cache/pip -before_cache: -- rm -f $HOME/.cache/pip/log/debug.log -notifications: - email: false -# branches: # remove travis double-check on pull requests in main repo -# only: -# - master -# - /^\d\.\d+$/ before_install: # fix a crash with multiprocessing on Travis # - sudo rm -rf /dev/shm diff --git a/README.rst b/README.rst index 22fbcb2df..b5f23f9a3 100644 --- a/README.rst +++ b/README.rst @@ -406,6 +406,9 @@ Parameters If (default: None) and ``file`` is unspecified, bytes will be written in Python 2. If ``True`` will also write bytes. In all other cases will default to unicode. +* lock_args : tuple, optional + Passed to ``refresh`` for intermediate output + (initialisation, iterating, and updating). Extra CLI Options ~~~~~~~~~~~~~~~~~ @@ -460,7 +463,18 @@ Returns """Clear current bar display.""" def refresh(self): - """Force refresh the display of this bar.""" + """ + Force refresh the display of this bar. + + Parameters + ---------- + nolock : bool, optional + If ``True``, does not lock. + If [default: ``False``]: calls ``acquire()`` on internal lock. + lock_args : tuple, optional + Passed to internal lock's ``acquire()``. + If specified, will only ``display()`` if ``acquire()`` returns ``True``. + """ def unpause(self): """Restart tqdm timer from last print time.""" diff --git a/examples/parallel_bars.py b/examples/parallel_bars.py index e49a2775f..7ac5b4012 100644 --- a/examples/parallel_bars.py +++ b/examples/parallel_bars.py @@ -4,6 +4,7 @@ from random import random from multiprocessing import Pool, freeze_support from concurrent.futures import ThreadPoolExecutor +from threading import RLock from functools import partial import sys @@ -11,11 +12,13 @@ PY2 = sys.version_info[:1] <= (2,) -def progresser(n, auto_position=True, write_safe=False): +def progresser(n, auto_position=True, write_safe=False, blocking=True): interval = random() * 0.002 / (NUM_SUBITERS - n + 2) total = 5000 text = "#{}, est. {:<04.2}s".format(n, interval * total) - for _ in tqdm(range(total), desc=text, position=None if auto_position else n): + for _ in trange(total, desc=text, + lock_args=None if blocking else (False,), + position=None if auto_position else n): sleep(interval) # NB: may not clear instances with higher `position` upon completion # since this worker may not know about other bars #796 @@ -44,6 +47,9 @@ def progresser(n, auto_position=True, write_safe=False): ncols = t.ncols or 80 print(("{msg:<{ncols}}").format(msg="Multi-threading", ncols=ncols)) + # explicitly set just threading lock for nonblocking progress + tqdm.set_lock(RLock()) with ThreadPoolExecutor() as p: - progresser_thread = partial(progresser, write_safe=not PY2) + progresser_thread = partial( + progresser, write_safe=not PY2, blocking=False) p.map(progresser_thread, L) diff --git a/setup.cfg b/setup.cfg index 67afc79db..cd065ddd2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,6 @@ universal = 1 [flake8] -ignore = W503,W504 +ignore = W503,W504,E722 max_line_length = 80 exclude = .asv,.tox,.ipynb_checkpoints,build,dist,.git,__pycache__ diff --git a/tox.ini b/tox.ini index d60a15b42..9c228b57a 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ commands = [testenv:flake8] deps = flake8 commands = - flake8 -j 8 --count --statistics --exit-zero . + flake8 -j 8 --count --statistics . [testenv:setup.py] deps = diff --git a/tqdm/_version.py b/tqdm/_version.py index 9d0030a48..ef204a347 100644 --- a/tqdm/_version.py +++ b/tqdm/_version.py @@ -5,7 +5,7 @@ __all__ = ["__version__"] # major, minor, patch, -extra -version_info = 4, 37, 0 +version_info = 4, 38, 0 # Nice string for the version __version__ = '.'.join(map(str, version_info)) diff --git a/tqdm/std.py b/tqdm/std.py index bc49b02d2..b5ff064ed 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -85,9 +85,9 @@ def __init__(self): cls = type(self) self.locks = [lk for lk in [cls.mp_lock, cls.th_lock] if lk is not None] - def acquire(self): + def acquire(self, *a, **k): for lock in self.locks: - lock.acquire() + lock.acquire(*a, **k) def release(self): for lock in self.locks[::-1]: # Release in inverse order of acquisition @@ -774,7 +774,8 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, miniters=None, ascii=None, disable=False, unit='it', unit_scale=False, dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0, position=None, postfix=None, - unit_divisor=1000, write_bytes=None, gui=False, **kwargs): + unit_divisor=1000, write_bytes=None, lock_args=None, + gui=False, **kwargs): """ Parameters ---------- @@ -871,6 +872,9 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, If (default: None) and `file` is unspecified, bytes will be written in Python 2. If `True` will also write bytes. In all other cases will default to unicode. + lock_args : tuple, optional + Passed to `refresh` for intermediate output + (initialisation, iterating, and updating). gui : bool, optional WARNING: internal parameter - do not use. Use tqdm.gui.tqdm(...) instead. If set, will attempt to use @@ -977,6 +981,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, self.unit = unit self.unit_scale = unit_scale self.unit_divisor = unit_divisor + self.lock_args = lock_args self.gui = gui self.dynamic_ncols = dynamic_ncols self.smoothing = smoothing @@ -1005,8 +1010,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, if not gui: # Initialize the screen printer self.sp = self.status_printer(self.fp) - with self._lock: - self.display() + self.refresh(lock_args=self.lock_args) # Init the time counter self.last_print_t = self._time() @@ -1103,8 +1107,7 @@ def __iter__(self): self.avg_time = avg_time self.n = n - with self._lock: - self.display() + self.refresh(lock_args=self.lock_args) # If no `miniters` was specified, adjust automatically # to the max iteration rate seen so far between 2 prints @@ -1187,8 +1190,7 @@ def update(self, n=1): " instead of `tqdm(..., gui=True)`\n", fp_write=getattr(self.fp, 'write', sys.stderr.write)) - with self._lock: - self.display() + self.refresh(lock_args=self.lock_args) # If no `miniters` was specified, adjust automatically to the # maximum iteration rate seen so far between two prints. @@ -1270,16 +1272,32 @@ def clear(self, nolock=False): if not nolock: self._lock.release() - def refresh(self, nolock=False): - """Force refresh the display of this bar.""" + def refresh(self, nolock=False, lock_args=None): + """ + Force refresh the display of this bar. + + Parameters + ---------- + nolock : bool, optional + If `True`, does not lock. + If [default: `False`]: calls `acquire()` on internal lock. + lock_args : tuple, optional + Passed to internal lock's `acquire()`. + If specified, will only `display()` if `acquire()` returns `True`. + """ if self.disable: return if not nolock: - self._lock.acquire() + if lock_args: + if not self._lock.acquire(*lock_args): + return False + else: + self._lock.acquire() self.display() if not nolock: self._lock.release() + return True def unpause(self): """Restart tqdm timer from last print time.""" diff --git a/tqdm/tests/tests_main.py b/tqdm/tests/tests_main.py index ac0298b24..573948dde 100644 --- a/tqdm/tests/tests_main.py +++ b/tqdm/tests/tests_main.py @@ -48,7 +48,7 @@ def test_main(): sys.argv = ['', '--desc', 'Test CLI --delim', '--ascii', 'True', '--delim', r'\0', '--buf_size', '64'] sys.stdin.write('\0'.join(map(str, _range(int(123))))) - #sys.stdin.write(b'\xff') # TODO + # sys.stdin.write(b'\xff') # TODO sys.stdin.seek(0) main() sys.stdin = IN_DATA_LIST diff --git a/tqdm/tests/tests_perf.py b/tqdm/tests/tests_perf.py index 072d4edc1..6cb7a6ee5 100644 --- a/tqdm/tests/tests_perf.py +++ b/tqdm/tests/tests_perf.py @@ -164,7 +164,7 @@ def assert_performance(thresh, name_left, time_left, name_right, time_right): if time_left > thresh * time_right: raise ValueError( ('{name[0]}: {time[0]:f}, ' - '{name[1]}: {time[0]:f}, ' + '{name[1]}: {time[1]:f}, ' 'ratio {ratio:f} > {thresh:f}').format( name=(name_left, name_right), time=(time_left, time_right), @@ -219,6 +219,48 @@ def test_manual_overhead(): assert_performance(6, 'tqdm', time_tqdm(), 'range', time_bench()) +def worker(total, blocking=True): + def incr_bar(x): + with closing(StringIO()) as our_file: + for _ in trange( + total, file=our_file, + lock_args=None if blocking else (False,), + miniters=1, mininterval=0, maxinterval=0): + pass + return x + 1 + return incr_bar + + +@with_setup(pretest, posttest) +@retry_on_except() +def test_lock_args(): + """Test overhead of nonblocking threads""" + try: + from concurrent.futures import ThreadPoolExecutor + from threading import RLock + except ImportError: + raise SkipTest + import sys + + total = 8 + subtotal = 1000 + + tqdm.set_lock(RLock()) + with ThreadPoolExecutor(total) as pool: + sys.stderr.write('block ... ') + sys.stderr.flush() + with relative_timer() as time_tqdm: + res = list(pool.map(worker(subtotal, True), range(total))) + assert sum(res) == sum(range(total)) + total + sys.stderr.write('noblock ... ') + sys.stderr.flush() + with relative_timer() as time_noblock: + res = list(pool.map(worker(subtotal, False), range(total))) + assert sum(res) == sum(range(total)) + total + + assert_performance(0.2, 'noblock', time_noblock(), 'tqdm', time_tqdm()) + + @with_setup(pretest, posttest) @retry_on_except() def test_iter_overhead_hard(): diff --git a/tqdm/tests/tests_synchronisation.py b/tqdm/tests/tests_synchronisation.py index c0924c63d..2fefafaf9 100644 --- a/tqdm/tests/tests_synchronisation.py +++ b/tqdm/tests/tests_synchronisation.py @@ -1,5 +1,5 @@ from __future__ import division -from tqdm import tqdm, TMonitor +from tqdm import tqdm, trange, TMonitor from tests_tqdm import with_setup, pretest, posttest, SkipTest, \ StringIO, closing from tests_tqdm import DiscreteTimer, cpu_timify @@ -41,6 +41,13 @@ def incr(x): return x + 1 +def incr_bar(x): + with closing(StringIO()) as our_file: + for _ in trange(x, lock_args=(False,), file=our_file): + pass + return incr(x) + + @with_setup(pretest, posttest) def test_monitor_thread(): """Test dummy monitoring thread""" @@ -179,3 +186,18 @@ def test_imap(): pool = Pool() res = list(tqdm(pool.imap(incr, range(100)), disable=True)) assert res[-1] == 100 + + +@with_setup(pretest, posttest) +def test_threadpool(): + """Test concurrent.futures.ThreadPoolExecutor""" + try: + from concurrent.futures import ThreadPoolExecutor + from threading import RLock + except ImportError: + raise SkipTest + + tqdm.set_lock(RLock()) + with ThreadPoolExecutor(8) as pool: + res = list(tqdm(pool.map(incr_bar, range(100)), disable=True)) + assert sum(res) == sum(range(1, 101)) diff --git a/tqdm/tqdm.1 b/tqdm/tqdm.1 index f2ee878e6..17adb32eb 100644 --- a/tqdm/tqdm.1 +++ b/tqdm/tqdm.1 @@ -209,6 +209,13 @@ In all other cases will default to unicode. .RS .RE .TP +.B \-\-lock_args=\f[I]lock_args\f[] +tuple, optional. +Passed to \f[C]refresh\f[] for intermediate output (initialisation, +iterating, and updating). +.RS +.RE +.TP .B \-\-delim=\f[I]delim\f[] chr, optional. Delimiting character [default: \[aq]\\n\[aq]]. diff --git a/tqdm/utils.py b/tqdm/utils.py index b51fb196d..3c653ddef 100644 --- a/tqdm/utils.py +++ b/tqdm/utils.py @@ -31,11 +31,15 @@ try: if IS_WIN: import colorama - colorama.init() else: - colorama = None + raise ImportError except ImportError: colorama = None + else: + try: + colorama.init(strip=False) + except TypeError: + colorama.init() try: from weakref import WeakSet