Skip to content

Commit

Permalink
fixed with progress
Browse files Browse the repository at this point in the history
  • Loading branch information
casperdcl committed Aug 25, 2019
1 parent 5747e37 commit c934275
Show file tree
Hide file tree
Showing 6 changed files with 24 additions and 11 deletions.
15 changes: 12 additions & 3 deletions dvc/progress.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ def __init__(
bytes=False, # pylint: disable=W0622
desc_truncate=None,
leave=None,
bar_format=(
"{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}"
" [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
),
**kwargs
):
"""
Expand All @@ -49,6 +53,7 @@ def __init__(
kwargs : anything accepted by `tqdm.tqdm()`
"""
kwargs = deepcopy(kwargs)
kwargs.setdefault("unit_scale", True)
if bytes:
for k, v in dict(
unit="B", unit_scale=True, unit_divisor=1024, miniters=1
Expand All @@ -62,14 +67,18 @@ def __init__(
>= logging.CRITICAL
)
super(Tqdm, self).__init__(
iterable=iterable, disable=disable, leave=leave, **kwargs
iterable=iterable,
disable=disable,
leave=leave,
bar_format=bar_format,
**kwargs
)

def update_desc(self, desc, n=1, truncate=True):
"""
Calls `set_description(truncate(desc))` and `update(n)`
Calls `set_description_str(truncate(desc))` and `update(n)`
"""
self.set_description(
self.set_description_str(
self.truncate(desc) if truncate else desc, refresh=False
)
self.update(n)
Expand Down
4 changes: 3 additions & 1 deletion dvc/remote/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,9 @@ def cache_exists(self, checksums, jobs=None):
if not self.no_traverse:
return list(set(checksums) & set(self.all()))

with Tqdm(total=len(checksums), unit="md5") as pbar:
with Tqdm(
desc="Querying remote cache", total=len(checksums), unit="md5"
) as pbar:

def exists_with_progress(path_info):
ret = self.exists(path_info)
Expand Down
6 changes: 3 additions & 3 deletions dvc/remote/local/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,9 @@ def move(self, from_info, to_info):
def cache_exists(self, checksums, jobs=None):
return [
checksum
for checksum in Tqdm(checksums, unit="md5")
for checksum in Tqdm(
checksums, unit="md5", desc="Querying local cache"
)
if not self.changed_cache_file(checksum)
]

Expand Down Expand Up @@ -319,7 +321,6 @@ def status(
ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
md5s = list(ret)

logger.info("Collecting information from local cache...")
local_exists = self.cache_exists(md5s, jobs=jobs)

# This is a performance optimization. We can safely assume that,
Expand All @@ -329,7 +330,6 @@ def status(
if download and sorted(local_exists) == sorted(md5s):
remote_exists = local_exists
else:
logger.info("Collecting information from remote cache...")
remote_exists = list(remote.cache_exists(md5s, jobs=jobs))

self._fill_statuses(ret, local_exists, remote_exists)
Expand Down
6 changes: 4 additions & 2 deletions dvc/remote/ssh/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,9 @@ def cache_exists(self, checksums, jobs=None):
if not self.no_traverse:
return list(set(checksums) & set(self.all()))

with Tqdm(total=len(checksums), unit="md5") as pbar:
with Tqdm(
desc="Querying remote cache", total=len(checksums), unit="md5"
) as pbar:

def exists_with_progress(chunks):
return self.batch_exists(chunks, callback=pbar.update_desc)
Expand All @@ -278,5 +280,5 @@ def exists_with_progress(chunks):
results = executor.map(exists_with_progress, chunks)
in_remote = itertools.chain.from_iterable(results)
ret = list(itertools.compress(checksums, in_remote))
pbar.update_desc("", 0) # clear path name description
pbar.set_description_str("Querying remote cache")
return ret
2 changes: 1 addition & 1 deletion dvc/repo/checkout.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,4 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False):
)

stage.checkout(force=force, progress_callback=pbar.update_desc)
pbar.update_desc("Checkout", 0) # clear path name description
pbar.set_description_str("Checkout")
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def run(self):
"funcy>=1.12",
"pathspec>=0.5.9",
"shortuuid>=0.5.0",
"tqdm>=4.34.0",
"tqdm>=4.35.0",
"win-unicode-console>=0.5; sys_platform == 'win32'",
]

Expand Down

0 comments on commit c934275

Please sign in to comment.