Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix iterable length handling in contrib.concurrent #1473

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fix iterable length handling in contrib.concurrent #1473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4997c59
Fix length detection in contrib.concurrent
svank May 12, 2023
dd37e6c
Handle no-length iterables in contrib.concurrent
svank May 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 3 additions & 1 deletion tests/tests_concurrent.py
View file
Open in desktop
Expand Up @@ -38,7 +38,9 @@ def test_process_map():

@mark.parametrize("iterables,should_warn", [([], False), (['x'], False), ([()], False),
(['x', ()], False), (['x' * 1001], True),
(['x' * 100, ('x',) * 1001], True)])
(['x' * 100, ('x',) * 1001], False),
(['x' * 1001, ('x',) * 100], False),
(['x' * 1001, ('x',) * 1001], True)])
def test_chunksize_warning(iterables, should_warn):
"""Test contrib.concurrent.process_map chunksize warnings"""
patch = importorskip('unittest.mock').patch
Expand Down
18 changes: 14 additions & 4 deletions tqdm/contrib/concurrent.py
View file
Open in desktop
Expand Up @@ -26,6 +26,16 @@ def ensure_lock(tqdm_class, lock_name=""):
tqdm_class.set_lock(old_lock)


def _shortest_iterable_length(iterables):
# Negative default for iterables that have no length (e.g. itertools.repeat)
iterable_lengths = [length_hint(iterable, -1) for iterable in iterables]
# Remove the negative values
iterable_lengths = filter(lambda length: length >= 0, iterable_lengths)
# Take the shortest of the finite iterables
shortest_iterable_len = min(iterable_lengths)
return shortest_iterable_len


def _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs):
"""
Implementation of `thread_map` and `process_map`.
Expand All @@ -39,7 +49,7 @@ def _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs):
"""
kwargs = tqdm_kwargs.copy()
if "total" not in kwargs:
kwargs["total"] = length_hint(iterables[0])
kwargs["total"] = _shortest_iterable_length(iterables)
tqdm_class = kwargs.pop("tqdm_class", tqdm_auto)
max_workers = kwargs.pop("max_workers", min(32, cpu_count() + 4))
chunksize = kwargs.pop("chunksize", 1)
Expand Down Expand Up @@ -92,12 +102,12 @@ def process_map(fn, *iterables, **tqdm_kwargs):
if iterables and "chunksize" not in tqdm_kwargs:
# default `chunksize=1` has poor performance for large iterables
# (most time spent dispatching items to workers).
longest_iterable_len = max(map(length_hint, iterables))
if longest_iterable_len > 1000:
shortest_iterable_len = _shortest_iterable_length(iterables)
if shortest_iterable_len > 1000:
from warnings import warn
warn("Iterable length %d > 1000 but `chunksize` is not set."
" This may seriously degrade multiprocess performance."
" Set `chunksize=1` or more." % longest_iterable_len,
" Set `chunksize=1` or more." % shortest_iterable_len,
TqdmWarning, stacklevel=2)
if "lock_name" not in tqdm_kwargs:
tqdm_kwargs = tqdm_kwargs.copy()
Expand Down