From c9413dfbe9860871533820c1ca33cb45d1aeb8a8 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 31 Oct 2019 19:16:39 +0000 Subject: [PATCH 01/22] attempt to move to Travis stages --- .travis.yml | 90 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index a048117eb..37b4f9f60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,24 +1,73 @@ language: python -matrix: +env: + global: + - PIP_CACHE_DIR="$HOME/.cache/pip" # unify pip cache location for all platforms +# use cache for big builds like pandas (to minimise build time). +# If issues, clear cache +# https://docs.travis-ci.com/user/caching/#Clearing-Caches +cache: + pip: true + directories: + - $HOME/.cache/pip +before_cache: +- rm -f $HOME/.cache/pip/log/debug.log +notifications: + email: false +# branches: # remove travis double-check on pull requests in main repo +# only: +# - master +# - /^\d\.\d+$/ +stages: +- check +- test +- deploy +jobs: include: - - python: 2.6 + - name: py2.6 + python: 2.6 env: TOXENV=py26 dist: trusty - - python: 2.7 + - name: py2.7 + python: 2.7 env: TOXENV=py27 - - python: 3.4 + - name: py3.4 + python: 3.4 env: TOXENV=py34 - - python: 3.5 + - name: py3.5 + python: 3.5 env: TOXENV=py35 - - python: 3.6 + - name: py3.6 + python: 3.6 env: TOXENV=py36 - - python: 3.7 + - name: py3.7 + python: 3.7 env: TOXENV=py37 + - name: pypy2.7 + python: pypy2.7-5.10.0 + env: TOXENV=pypy + - name: pypy3.5 + python: pypy3.5-5.10.0 + env: TOXENV=pypy3 + - name: style + stage: check + python: 3.6 + env: TOXENV=flake8 + - name: setup + stage: check + python: 3.6 + env: TOXENV=setup.py + - name: perf + python: 3.6 + env: TOXENV=perf + - name: deploy + stage: deploy + python: 3.7 dist: xenial sudo: true # required for py37, docker services: - docker - after_success: + install: + script: - echo "$DOCKER_PWD" | docker login -u $DOCKER_USR --password-stdin - echo "$GITHUB_TOKEN" | docker login docker.pkg.github.com -u $GITHUB_USR --password-stdin - make -B docker @@ -72,31 +121,6 @@ matrix: script: 'docker push docker.pkg.github.com/tqdm/tqdm/tqdm:devel || :' on: branch: devel - - python: pypy2.7-5.10.0 - env: TOXENV=pypy - - python: pypy3.5-5.10.0 - env: TOXENV=pypy3 - - python: 3.6 - env: TOXENV=flake8 - - python: 3.6 - env: TOXENV=setup.py - - python: 3.6 - env: TOXENV=perf -# use cache for big builds like pandas (to minimise build time). -# If issues, clear cache -# https://docs.travis-ci.com/user/caching/#Clearing-Caches -cache: - pip: true - directories: - - $HOME/.cache/pip -before_cache: -- rm -f $HOME/.cache/pip/log/debug.log -notifications: - email: false -# branches: # remove travis double-check on pull requests in main repo -# only: -# - master -# - /^\d\.\d+$/ before_install: # fix a crash with multiprocessing on Travis # - sudo rm -rf /dev/shm From a46caa4ba4a447e423244a56ac06b2e5da164428 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 31 Oct 2019 19:31:53 +0000 Subject: [PATCH 02/22] linting/fix flake8 --- .meta/mkdocs.py | 1 - examples/parallel_bars.py | 3 ++- setup.cfg | 2 +- tox.ini | 2 +- tqdm/tests/tests_main.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.meta/mkdocs.py b/.meta/mkdocs.py index a5707f320..d7b858ea4 100644 --- a/.meta/mkdocs.py +++ b/.meta/mkdocs.py @@ -29,7 +29,6 @@ def doc2rst(doc, arglist=True, raw=False): doc = doc.replace('`', '``') if raw: doc = doc.replace('\n ', '\n ') - #doc = '\n'.join(i.rstrip() for i in doc.split('\n')) else: doc = dedent(doc) if arglist: diff --git a/examples/parallel_bars.py b/examples/parallel_bars.py index e49a2775f..86a49196e 100644 --- a/examples/parallel_bars.py +++ b/examples/parallel_bars.py @@ -15,7 +15,8 @@ def progresser(n, auto_position=True, write_safe=False): interval = random() * 0.002 / (NUM_SUBITERS - n + 2) total = 5000 text = "#{}, est. {:<04.2}s".format(n, interval * total) - for _ in tqdm(range(total), desc=text, position=None if auto_position else n): + for _ in trange(total, desc=text, + position=None if auto_position else n): sleep(interval) # NB: may not clear instances with higher `position` upon completion # since this worker may not know about other bars #796 diff --git a/setup.cfg b/setup.cfg index 67afc79db..cd065ddd2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,6 @@ universal = 1 [flake8] -ignore = W503,W504 +ignore = W503,W504,E722 max_line_length = 80 exclude = .asv,.tox,.ipynb_checkpoints,build,dist,.git,__pycache__ diff --git a/tox.ini b/tox.ini index d60a15b42..9c228b57a 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ commands = [testenv:flake8] deps = flake8 commands = - flake8 -j 8 --count --statistics --exit-zero . + flake8 -j 8 --count --statistics . [testenv:setup.py] deps = diff --git a/tqdm/tests/tests_main.py b/tqdm/tests/tests_main.py index ac0298b24..573948dde 100644 --- a/tqdm/tests/tests_main.py +++ b/tqdm/tests/tests_main.py @@ -48,7 +48,7 @@ def test_main(): sys.argv = ['', '--desc', 'Test CLI --delim', '--ascii', 'True', '--delim', r'\0', '--buf_size', '64'] sys.stdin.write('\0'.join(map(str, _range(int(123))))) - #sys.stdin.write(b'\xff') # TODO + # sys.stdin.write(b'\xff') # TODO sys.stdin.seek(0) main() sys.stdin = IN_DATA_LIST From e77d716fb7f673c3594e663fd77060c1d01dc623 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 31 Oct 2019 20:06:22 +0000 Subject: [PATCH 03/22] separate docker deploy --- .travis.yml | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37b4f9f60..a06c37e02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,25 +59,13 @@ jobs: - name: perf python: 3.6 env: TOXENV=perf - - name: deploy + - name: PyPI and GitHub stage: deploy python: 3.7 dist: xenial - sudo: true # required for py37, docker - services: - - docker + sudo: true # required for py37 install: script: - - echo "$DOCKER_PWD" | docker login -u $DOCKER_USR --password-stdin - - echo "$GITHUB_TOKEN" | docker login docker.pkg.github.com -u $GITHUB_USR --password-stdin - - make -B docker - - | - if [[ -n "$TRAVIS_TAG" ]]; then - docker tag tqdm/tqdm:latest tqdm/tqdm:${TRAVIS_TAG#v} - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:${TRAVIS_TAG#v} ; fi - - docker tag tqdm/tqdm:latest tqdm/tqdm:devel - - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:latest - - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:devel - pip install .[dev] - make build #- make submodules @@ -101,6 +89,26 @@ jobs: name: tqdm $TRAVIS_TAG stable on: tags: true + - name: docker + stage: deploy + python: 3.7 + dist: xenial + sudo: true # required for py37, docker + services: + - docker + install: + script: + - echo "$DOCKER_PWD" | docker login -u $DOCKER_USR --password-stdin + - echo "$GITHUB_TOKEN" | docker login docker.pkg.github.com -u $GITHUB_USR --password-stdin + - make -B docker + - | + if [[ -n "$TRAVIS_TAG" ]]; then + docker tag tqdm/tqdm:latest tqdm/tqdm:${TRAVIS_TAG#v} + docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:${TRAVIS_TAG#v} ; fi + - docker tag tqdm/tqdm:latest tqdm/tqdm:devel + - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:latest + - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:devel + deploy: - provider: script script: docker push tqdm/tqdm:${TRAVIS_TAG#v} on: From ae6de71c3460123944aa7bb80d7cb0b5bdcafb7b Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 1 Nov 2019 11:01:40 +0000 Subject: [PATCH 04/22] fix colorama on win - fixes #678 - fixes #764 --- tqdm/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tqdm/utils.py b/tqdm/utils.py index b51fb196d..3c653ddef 100644 --- a/tqdm/utils.py +++ b/tqdm/utils.py @@ -31,11 +31,15 @@ try: if IS_WIN: import colorama - colorama.init() else: - colorama = None + raise ImportError except ImportError: colorama = None + else: + try: + colorama.init(strip=False) + except TypeError: + colorama.init() try: from weakref import WeakSet From b989bc47b7678b5b82c9c7078702584beda95930 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 7 Nov 2019 22:39:25 +0000 Subject: [PATCH 05/22] support args for `TqdmDefaultWriteLock.acquire()` - related to #838 --- tqdm/std.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index bc49b02d2..801e2a071 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -85,9 +85,9 @@ def __init__(self): cls = type(self) self.locks = [lk for lk in [cls.mp_lock, cls.th_lock] if lk is not None] - def acquire(self): + def acquire(self, *a, **k): for lock in self.locks: - lock.acquire() + lock.acquire(*a, **k) def release(self): for lock in self.locks[::-1]: # Release in inverse order of acquisition From 23e7fb1fe144be5c76a3eb565c59f62083b9665e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 7 Nov 2019 22:56:20 +0000 Subject: [PATCH 06/22] use `self.refresh()` in more places - working towards #838 --- tqdm/std.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index 801e2a071..3f2c60ba3 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -1005,8 +1005,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, if not gui: # Initialize the screen printer self.sp = self.status_printer(self.fp) - with self._lock: - self.display() + self.refresh() # Init the time counter self.last_print_t = self._time() @@ -1103,8 +1102,7 @@ def __iter__(self): self.avg_time = avg_time self.n = n - with self._lock: - self.display() + self.refresh() # If no `miniters` was specified, adjust automatically # to the max iteration rate seen so far between 2 prints @@ -1187,8 +1185,7 @@ def update(self, n=1): " instead of `tqdm(..., gui=True)`\n", fp_write=getattr(self.fp, 'write', sys.stderr.write)) - with self._lock: - self.display() + self.refresh() # If no `miniters` was specified, adjust automatically to the # maximum iteration rate seen so far between two prints. From 4d95227b7eb454e032661564120e925a09338e7e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 7 Nov 2019 23:33:03 +0000 Subject: [PATCH 07/22] add refresh(lock_args=None) --- tqdm/std.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index 3f2c60ba3..2b00dc9f8 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -1267,13 +1267,25 @@ def clear(self, nolock=False): if not nolock: self._lock.release() - def refresh(self, nolock=False): - """Force refresh the display of this bar.""" + def refresh(self, nolock=False, lock_args=None): + """ + Force refresh the display of this bar. + + Parameters + ---------- + nolock : bool, optional + If `True`, does not lock. + If [default: `False`]: calls `acquire()` on internal lock; + `display()`ing only if `acquire()` returns `True`. + lock_args : tuple, optional + Passed to internal lock's `acquire()`. + """ if self.disable: return if not nolock: - self._lock.acquire() + if not self._lock.acquire(*(lock_args or ())): + return self.display() if not nolock: self._lock.release() From dba8a28f13af4fc522b9fe26014b618fda739c8e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 7 Nov 2019 23:34:21 +0000 Subject: [PATCH 08/22] default `lock_args=(False,)` for intermediate progress - fixes #838 --- tqdm/std.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index 2b00dc9f8..eb6a20716 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -1005,7 +1005,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, if not gui: # Initialize the screen printer self.sp = self.status_printer(self.fp) - self.refresh() + self.refresh(lock_args=(False,)) # Init the time counter self.last_print_t = self._time() @@ -1102,7 +1102,7 @@ def __iter__(self): self.avg_time = avg_time self.n = n - self.refresh() + self.refresh(lock_args=(False,)) # If no `miniters` was specified, adjust automatically # to the max iteration rate seen so far between 2 prints @@ -1185,7 +1185,7 @@ def update(self, n=1): " instead of `tqdm(..., gui=True)`\n", fp_write=getattr(self.fp, 'write', sys.stderr.write)) - self.refresh() + self.refresh(lock_args=(False,)) # If no `miniters` was specified, adjust automatically to the # maximum iteration rate seen so far between two prints. From 66037effa468dc82bbe7115170b42bfb4ba3c9ef Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 7 Nov 2019 23:47:13 +0000 Subject: [PATCH 09/22] default lock_args=None and expose - avoids errors with custom locks not supporting acquire(False) --- tqdm/std.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index eb6a20716..b173443ed 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -774,7 +774,8 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, miniters=None, ascii=None, disable=False, unit='it', unit_scale=False, dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0, position=None, postfix=None, - unit_divisor=1000, write_bytes=None, gui=False, **kwargs): + unit_divisor=1000, write_bytes=None, lock_args=None, + gui=False, **kwargs): """ Parameters ---------- @@ -871,6 +872,9 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, If (default: None) and `file` is unspecified, bytes will be written in Python 2. If `True` will also write bytes. In all other cases will default to unicode. + lock_args : tuple, optional + Passed to `refresh` for intermediate output + (creating, iterating, and updating). gui : bool, optional WARNING: internal parameter - do not use. Use tqdm.gui.tqdm(...) instead. If set, will attempt to use @@ -977,6 +981,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, self.unit = unit self.unit_scale = unit_scale self.unit_divisor = unit_divisor + self.lock_args = lock_args self.gui = gui self.dynamic_ncols = dynamic_ncols self.smoothing = smoothing @@ -1005,7 +1010,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, if not gui: # Initialize the screen printer self.sp = self.status_printer(self.fp) - self.refresh(lock_args=(False,)) + self.refresh(lock_args=self.lock_args) # Init the time counter self.last_print_t = self._time() @@ -1102,7 +1107,7 @@ def __iter__(self): self.avg_time = avg_time self.n = n - self.refresh(lock_args=(False,)) + self.refresh(lock_args=self.lock_args) # If no `miniters` was specified, adjust automatically # to the max iteration rate seen so far between 2 prints @@ -1185,7 +1190,7 @@ def update(self, n=1): " instead of `tqdm(..., gui=True)`\n", fp_write=getattr(self.fp, 'write', sys.stderr.write)) - self.refresh(lock_args=(False,)) + self.refresh(lock_args=self.lock_args) # If no `miniters` was specified, adjust automatically to the # maximum iteration rate seen so far between two prints. From 37038f005dfe767ba423748c4356148115f4e2dc Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 01:33:14 +0000 Subject: [PATCH 10/22] fix acquisition and perf debugging --- tqdm/std.py | 10 ++++++---- tqdm/tests/tests_perf.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index b173443ed..6d6510d6c 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -1280,17 +1280,19 @@ def refresh(self, nolock=False, lock_args=None): ---------- nolock : bool, optional If `True`, does not lock. - If [default: `False`]: calls `acquire()` on internal lock; - `display()`ing only if `acquire()` returns `True`. + If [default: `False`]: calls `acquire()` on internal lock. lock_args : tuple, optional Passed to internal lock's `acquire()`. + If specified, will only `display()` if `acquire()` returns `True`. """ if self.disable: return if not nolock: - if not self._lock.acquire(*(lock_args or ())): - return + if lock_args: + if not self._lock.acquire(*lock_args): + return + self._lock.acquire() self.display() if not nolock: self._lock.release() diff --git a/tqdm/tests/tests_perf.py b/tqdm/tests/tests_perf.py index 072d4edc1..44e7fb694 100644 --- a/tqdm/tests/tests_perf.py +++ b/tqdm/tests/tests_perf.py @@ -164,7 +164,7 @@ def assert_performance(thresh, name_left, time_left, name_right, time_right): if time_left > thresh * time_right: raise ValueError( ('{name[0]}: {time[0]:f}, ' - '{name[1]}: {time[0]:f}, ' + '{name[1]}: {time[1]:f}, ' 'ratio {ratio:f} > {thresh:f}').format( name=(name_left, name_right), time=(time_left, time_right), From 52b985e16c6ae9028586956a8556a56a2429a946 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 01:34:36 +0000 Subject: [PATCH 11/22] fix lock_args on creation, add tests --- tqdm/std.py | 4 ++-- tqdm/tests/tests_synchronisation.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index 6d6510d6c..9a66521ff 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -874,7 +874,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, bytes. In all other cases will default to unicode. lock_args : tuple, optional Passed to `refresh` for intermediate output - (creating, iterating, and updating). + (iterating/updating). gui : bool, optional WARNING: internal parameter - do not use. Use tqdm.gui.tqdm(...) instead. If set, will attempt to use @@ -1010,7 +1010,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, if not gui: # Initialize the screen printer self.sp = self.status_printer(self.fp) - self.refresh(lock_args=self.lock_args) + self.refresh() # Init the time counter self.last_print_t = self._time() diff --git a/tqdm/tests/tests_synchronisation.py b/tqdm/tests/tests_synchronisation.py index c0924c63d..149e4636d 100644 --- a/tqdm/tests/tests_synchronisation.py +++ b/tqdm/tests/tests_synchronisation.py @@ -41,6 +41,14 @@ def incr(x): return x + 1 +def incr_bar(x): + with closing(StringIO()) as our_file: + with tqdm(total=x, lock_args=(False,), file=our_file) as t: + for i in range(x): + t.update() + return incr(x) + + @with_setup(pretest, posttest) def test_monitor_thread(): """Test dummy monitoring thread""" @@ -179,3 +187,18 @@ def test_imap(): pool = Pool() res = list(tqdm(pool.imap(incr, range(100)), disable=True)) assert res[-1] == 100 + + +@with_setup(pretest, posttest) +def test_threadpool(): + """Test concurrent.futures.ThreadPoolExecutor""" + try: + from concurrent.futures import ThreadPoolExecutor + from threading import RLock + except ImportError: + raise SkipTest + + tqdm.set_lock(RLock()) + with ThreadPoolExecutor() as pool: + res = list(tqdm(pool.map(incr_bar, range(100)), disable=True)) + assert sum(res) == sum(range(1, 101)) From 99d4fd20d31d22ee561ea6770c7583d7b39c82c7 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 01:54:15 +0000 Subject: [PATCH 12/22] perf test non-blocking --- tqdm/tests/tests_perf.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tqdm/tests/tests_perf.py b/tqdm/tests/tests_perf.py index 44e7fb694..bc1146cd9 100644 --- a/tqdm/tests/tests_perf.py +++ b/tqdm/tests/tests_perf.py @@ -219,6 +219,40 @@ def test_manual_overhead(): assert_performance(6, 'tqdm', time_tqdm(), 'range', time_bench()) +def worker(total, blocking=True): + def incr_bar(x): + with closing(StringIO()) as our_file: + with tqdm(total=x, lock_args=None if blocking else (False,), + file=our_file) as t: + for i in range(total): + t.update() + return x + 1 + return incr_bar + + +@with_setup(pretest, posttest) +@retry_on_except() +def test_lock_args(): + """Test overhead of nonblocking threads""" + try: + from concurrent.futures import ThreadPoolExecutor + from threading import RLock + except ImportError: + raise SkipTest + + total = 8 + subtotal = int(1e4) + + tqdm.set_lock(RLock()) + with ThreadPoolExecutor(total) as pool: + with relative_timer() as time_tqdm: + res = list(pool.map(worker(subtotal, True), range(total))) + with relative_timer() as time_noblock: + res = list(pool.map(worker(subtotal, False), range(total))) + + assert_performance(0.99, 'noblock', time_noblock(), 'tqdm', time_tqdm()) + + @with_setup(pretest, posttest) @retry_on_except() def test_iter_overhead_hard(): From 7c38d0b58ecfe11c71c35539aaaa1823a5ab43a7 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 14:16:27 +0000 Subject: [PATCH 13/22] fix flake8 and add test logic --- tqdm/tests/tests_perf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tqdm/tests/tests_perf.py b/tqdm/tests/tests_perf.py index bc1146cd9..eda6c62af 100644 --- a/tqdm/tests/tests_perf.py +++ b/tqdm/tests/tests_perf.py @@ -247,8 +247,10 @@ def test_lock_args(): with ThreadPoolExecutor(total) as pool: with relative_timer() as time_tqdm: res = list(pool.map(worker(subtotal, True), range(total))) + assert sum(res) == sum(range(total)) + total with relative_timer() as time_noblock: res = list(pool.map(worker(subtotal, False), range(total))) + assert sum(res) == sum(range(total)) + total assert_performance(0.99, 'noblock', time_noblock(), 'tqdm', time_tqdm()) From d301a1237cceda3cd8aedccac9a6914141ebf4f9 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 14:53:37 +0000 Subject: [PATCH 14/22] debugging thread races --- tqdm/std.py | 3 ++- tqdm/tests/tests_perf.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index 9a66521ff..58e0da1f6 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -1291,11 +1291,12 @@ def refresh(self, nolock=False, lock_args=None): if not nolock: if lock_args: if not self._lock.acquire(*lock_args): - return + return False self._lock.acquire() self.display() if not nolock: self._lock.release() + return True def unpause(self): """Restart tqdm timer from last print time.""" diff --git a/tqdm/tests/tests_perf.py b/tqdm/tests/tests_perf.py index eda6c62af..d19069e34 100644 --- a/tqdm/tests/tests_perf.py +++ b/tqdm/tests/tests_perf.py @@ -223,7 +223,7 @@ def worker(total, blocking=True): def incr_bar(x): with closing(StringIO()) as our_file: with tqdm(total=x, lock_args=None if blocking else (False,), - file=our_file) as t: + file=our_file, miniters=1, mininterval=0) as t: for i in range(total): t.update() return x + 1 @@ -239,18 +239,24 @@ def test_lock_args(): from threading import RLock except ImportError: raise SkipTest + import sys total = 8 - subtotal = int(1e4) + subtotal = 10 tqdm.set_lock(RLock()) with ThreadPoolExecutor(total) as pool: + sys.stderr.write('block ... ') + sys.stderr.flush() with relative_timer() as time_tqdm: res = list(pool.map(worker(subtotal, True), range(total))) assert sum(res) == sum(range(total)) + total + sys.stderr.write('noblock ... ') + sys.stderr.flush() with relative_timer() as time_noblock: res = list(pool.map(worker(subtotal, False), range(total))) assert sum(res) == sum(range(total)) + total + sys.stderr.write('done ... ') assert_performance(0.99, 'noblock', time_noblock(), 'tqdm', time_tqdm()) From 213f30288c3b30e6841f06838a5d669e3f19dd4a Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 23:05:44 +0000 Subject: [PATCH 15/22] fix silly lock bug, tighten perf tests --- tqdm/std.py | 5 +++-- tqdm/tests/tests_perf.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index 58e0da1f6..8dacf3c1f 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -1010,7 +1010,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, if not gui: # Initialize the screen printer self.sp = self.status_printer(self.fp) - self.refresh() + self.refresh(lock_args=self.lock_args) # Init the time counter self.last_print_t = self._time() @@ -1292,7 +1292,8 @@ def refresh(self, nolock=False, lock_args=None): if lock_args: if not self._lock.acquire(*lock_args): return False - self._lock.acquire() + else: + self._lock.acquire() self.display() if not nolock: self._lock.release() diff --git a/tqdm/tests/tests_perf.py b/tqdm/tests/tests_perf.py index d19069e34..ffc6d90ad 100644 --- a/tqdm/tests/tests_perf.py +++ b/tqdm/tests/tests_perf.py @@ -222,8 +222,9 @@ def test_manual_overhead(): def worker(total, blocking=True): def incr_bar(x): with closing(StringIO()) as our_file: - with tqdm(total=x, lock_args=None if blocking else (False,), - file=our_file, miniters=1, mininterval=0) as t: + with tqdm(total=x, file=our_file, + lock_args=None if blocking else (False,), + miniters=1, mininterval=0, maxinterval=0) as t: for i in range(total): t.update() return x + 1 @@ -242,7 +243,7 @@ def test_lock_args(): import sys total = 8 - subtotal = 10 + subtotal = 100 tqdm.set_lock(RLock()) with ThreadPoolExecutor(total) as pool: @@ -256,9 +257,8 @@ def test_lock_args(): with relative_timer() as time_noblock: res = list(pool.map(worker(subtotal, False), range(total))) assert sum(res) == sum(range(total)) + total - sys.stderr.write('done ... ') - assert_performance(0.99, 'noblock', time_noblock(), 'tqdm', time_tqdm()) + assert_performance(0.2, 'noblock', time_noblock(), 'tqdm', time_tqdm()) @with_setup(pretest, posttest) From 166e1bc5dcae974ff39b99e3d51d2b67a18dde71 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 23:20:43 +0000 Subject: [PATCH 16/22] liting and minor tidy --- tqdm/tests/tests_perf.py | 12 ++++++------ tqdm/tests/tests_synchronisation.py | 7 +++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tqdm/tests/tests_perf.py b/tqdm/tests/tests_perf.py index ffc6d90ad..6cb7a6ee5 100644 --- a/tqdm/tests/tests_perf.py +++ b/tqdm/tests/tests_perf.py @@ -222,11 +222,11 @@ def test_manual_overhead(): def worker(total, blocking=True): def incr_bar(x): with closing(StringIO()) as our_file: - with tqdm(total=x, file=our_file, - lock_args=None if blocking else (False,), - miniters=1, mininterval=0, maxinterval=0) as t: - for i in range(total): - t.update() + for _ in trange( + total, file=our_file, + lock_args=None if blocking else (False,), + miniters=1, mininterval=0, maxinterval=0): + pass return x + 1 return incr_bar @@ -243,7 +243,7 @@ def test_lock_args(): import sys total = 8 - subtotal = 100 + subtotal = 1000 tqdm.set_lock(RLock()) with ThreadPoolExecutor(total) as pool: diff --git a/tqdm/tests/tests_synchronisation.py b/tqdm/tests/tests_synchronisation.py index 149e4636d..f399f1569 100644 --- a/tqdm/tests/tests_synchronisation.py +++ b/tqdm/tests/tests_synchronisation.py @@ -1,5 +1,5 @@ from __future__ import division -from tqdm import tqdm, TMonitor +from tqdm import tqdm, trange, TMonitor from tests_tqdm import with_setup, pretest, posttest, SkipTest, \ StringIO, closing from tests_tqdm import DiscreteTimer, cpu_timify @@ -43,9 +43,8 @@ def incr(x): def incr_bar(x): with closing(StringIO()) as our_file: - with tqdm(total=x, lock_args=(False,), file=our_file) as t: - for i in range(x): - t.update() + for _ in trange(x, lock_args=(False,), file=our_file): + pass return incr(x) From 464165c3cb7b6f1e1917e62bbaa5d349cdc436ee Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 23:25:15 +0000 Subject: [PATCH 17/22] fix py3.4 tests --- tqdm/tests/tests_synchronisation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tqdm/tests/tests_synchronisation.py b/tqdm/tests/tests_synchronisation.py index f399f1569..2fefafaf9 100644 --- a/tqdm/tests/tests_synchronisation.py +++ b/tqdm/tests/tests_synchronisation.py @@ -198,6 +198,6 @@ def test_threadpool(): raise SkipTest tqdm.set_lock(RLock()) - with ThreadPoolExecutor() as pool: + with ThreadPoolExecutor(8) as pool: res = list(tqdm(pool.map(incr_bar, range(100)), disable=True)) assert sum(res) == sum(range(1, 101)) From 1599ada42dedce16ff2f1988ddffd1641cd9f5fd Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 23:41:19 +0000 Subject: [PATCH 18/22] update docs --- .meta/mkdocs.py | 3 +++ README.rst | 16 +++++++++++++++- tqdm/std.py | 2 +- tqdm/tqdm.1 | 7 +++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.meta/mkdocs.py b/.meta/mkdocs.py index d7b858ea4..0d9ff8ddb 100644 --- a/.meta/mkdocs.py +++ b/.meta/mkdocs.py @@ -1,4 +1,7 @@ from __future__ import print_function +from os import path +import sys +sys.path = [path.dirname(path.dirname(__file__))] + sys.path # NOQA import tqdm import tqdm.cli from textwrap import dedent diff --git a/README.rst b/README.rst index 22fbcb2df..b5f23f9a3 100644 --- a/README.rst +++ b/README.rst @@ -406,6 +406,9 @@ Parameters If (default: None) and ``file`` is unspecified, bytes will be written in Python 2. If ``True`` will also write bytes. In all other cases will default to unicode. +* lock_args : tuple, optional + Passed to ``refresh`` for intermediate output + (initialisation, iterating, and updating). Extra CLI Options ~~~~~~~~~~~~~~~~~ @@ -460,7 +463,18 @@ Returns """Clear current bar display.""" def refresh(self): - """Force refresh the display of this bar.""" + """ + Force refresh the display of this bar. + + Parameters + ---------- + nolock : bool, optional + If ``True``, does not lock. + If [default: ``False``]: calls ``acquire()`` on internal lock. + lock_args : tuple, optional + Passed to internal lock's ``acquire()``. + If specified, will only ``display()`` if ``acquire()`` returns ``True``. + """ def unpause(self): """Restart tqdm timer from last print time.""" diff --git a/tqdm/std.py b/tqdm/std.py index 8dacf3c1f..b5ff064ed 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -874,7 +874,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, bytes. In all other cases will default to unicode. lock_args : tuple, optional Passed to `refresh` for intermediate output - (iterating/updating). + (initialisation, iterating, and updating). gui : bool, optional WARNING: internal parameter - do not use. Use tqdm.gui.tqdm(...) instead. If set, will attempt to use diff --git a/tqdm/tqdm.1 b/tqdm/tqdm.1 index f2ee878e6..17adb32eb 100644 --- a/tqdm/tqdm.1 +++ b/tqdm/tqdm.1 @@ -209,6 +209,13 @@ In all other cases will default to unicode. .RS .RE .TP +.B \-\-lock_args=\f[I]lock_args\f[] +tuple, optional. +Passed to \f[C]refresh\f[] for intermediate output (initialisation, +iterating, and updating). +.RS +.RE +.TP .B \-\-delim=\f[I]delim\f[] chr, optional. Delimiting character [default: \[aq]\\n\[aq]]. From b1fdd0c4787c2e7189ca3a277c16ef0034cf084b Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 8 Nov 2019 23:53:58 +0000 Subject: [PATCH 19/22] update example --- examples/parallel_bars.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/parallel_bars.py b/examples/parallel_bars.py index 86a49196e..7ac5b4012 100644 --- a/examples/parallel_bars.py +++ b/examples/parallel_bars.py @@ -4,6 +4,7 @@ from random import random from multiprocessing import Pool, freeze_support from concurrent.futures import ThreadPoolExecutor +from threading import RLock from functools import partial import sys @@ -11,11 +12,12 @@ PY2 = sys.version_info[:1] <= (2,) -def progresser(n, auto_position=True, write_safe=False): +def progresser(n, auto_position=True, write_safe=False, blocking=True): interval = random() * 0.002 / (NUM_SUBITERS - n + 2) total = 5000 text = "#{}, est. {:<04.2}s".format(n, interval * total) for _ in trange(total, desc=text, + lock_args=None if blocking else (False,), position=None if auto_position else n): sleep(interval) # NB: may not clear instances with higher `position` upon completion @@ -45,6 +47,9 @@ def progresser(n, auto_position=True, write_safe=False): ncols = t.ncols or 80 print(("{msg:<{ncols}}").format(msg="Multi-threading", ncols=ncols)) + # explicitly set just threading lock for nonblocking progress + tqdm.set_lock(RLock()) with ThreadPoolExecutor() as p: - progresser_thread = partial(progresser, write_safe=not PY2) + progresser_thread = partial( + progresser, write_safe=not PY2, blocking=False) p.map(progresser_thread, L) From 265d46a434aa00e71ea71cea8550525ffd9360de Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 9 Nov 2019 00:45:54 +0000 Subject: [PATCH 20/22] deploy notes and only parent repo --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a06c37e02..de1c53267 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,8 @@ notifications: stages: - check - test -- deploy +- name: deploy + if: repo = tqdm/tqdm jobs: include: - name: py2.6 @@ -74,6 +75,7 @@ jobs: -iv $encrypted_a6d6301302b7_iv -in .meta/.tqdm.gpg.enc -out .tqdm.gpg -d - gpg --import .tqdm.gpg - rm .tqdm.gpg + - git log --pretty='format:- %s%n%b---' $(git tag --sort=creatordate | tail -n2 | head -n1)..HEAD > CHANGES.md deploy: - provider: script script: twine upload -s -i tqdm@caspersci.uk.to dist/tqdm-* @@ -87,6 +89,8 @@ jobs: skip_cleanup: true draft: true name: tqdm $TRAVIS_TAG stable + edge: true + release_notes_file: CHANGES.md on: tags: true - name: docker From f44a3f681ab298ed80c89b75398059e71e47eaa0 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 9 Nov 2019 00:47:48 +0000 Subject: [PATCH 21/22] remove unneeded sudo --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index de1c53267..6bc996c83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,6 @@ jobs: stage: deploy python: 3.7 dist: xenial - sudo: true # required for py37 install: script: - pip install .[dev] @@ -97,7 +96,6 @@ jobs: stage: deploy python: 3.7 dist: xenial - sudo: true # required for py37, docker services: - docker install: From 0743062a8c88982e051731835825952ebb46b245 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 9 Nov 2019 00:48:07 +0000 Subject: [PATCH 22/22] use new cleanup --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6bc996c83..737edeef1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,14 +78,14 @@ jobs: deploy: - provider: script script: twine upload -s -i tqdm@caspersci.uk.to dist/tqdm-* - skip_cleanup: true + cleanup: false on: tags: true - provider: releases api_key: $GITHUB_TOKEN file_glob: true file: dist/tqdm-*.whl* - skip_cleanup: true + cleanup: false draft: true name: tqdm $TRAVIS_TAG stable edge: true