From cf91bd4d424b05ec2f20bd023843e78799510394 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 15 Dec 2021 16:55:28 +0100 Subject: [PATCH 01/25] Add `Progress.read` and `rich.progress.read` to track progress reading a file --- rich/progress.py | 190 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) diff --git a/rich/progress.py b/rich/progress.py index e4abbdb66..6b6c39996 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -1,13 +1,17 @@ from abc import ABC, abstractmethod from collections import deque from collections.abc import Sized +from contextlib import contextmanager from dataclasses import dataclass, field from datetime import timedelta +from io import RawIOBase from math import ceil +from os import PathLike, stat from threading import Event, RLock, Thread from types import TracebackType from typing import ( Any, + BinaryIO, Callable, Deque, Dict, @@ -149,6 +153,149 @@ def track( ) +class _Reader(RawIOBase): + """A reader that tracks progress while it's being read from.""" + + def __init__(self, handle: BinaryIO, progress: "Progress", task: TaskID): + self.handle = handle + self.progress = progress + self.task = task + + def __enter__(self): + self.handle.__enter__() + return self + + def __exit__(self, exc_val, exc_ty, tb): + return self.handle.__exit__(exc_val, exc_ty, tb) + + def __iter__(self): + return self + + def __next__(self): + line = next(self.handle) + self.progress.advance(self.task, advance=len(line)) + return line + + @property + def closed(self): + return self.closed + + def isatty(self): + return self.handle.isatty() + + def readable(self): + return self.handle.readable() + + def seekable(self): + return self.handle.seekable() + + def writable(self): + return False + + def read(self, size=-1): + block = self.handle.read(size) + self.progress.advance(self.task, advance=len(block)) + return block + + def readall(self): + block = self.handle.readall() + self.progress.advance(self.task, advance=len(block)) + return block + + def readinto(self, b): + n = self.handle.readinto(b) + self.progress.advance(self.task, advance=n) + return n + + def readline(self, size=-1): + line = self.handle.readline(size) + self.progress.advance(self.task, advance=len(line)) + return line + + def readlines(self, hint=-1): + lines = self.handle.readlines(hint) + self.progress.advance(self.task, advance=sum(map(len, lines))) + return lines + + def close(self): + self.handle.close() + + def seek(self, offset, whence=0): + pos = self.handle.seek(offset, whence) + self.progress.update(self.task, completed=pos) + return pos + + def tell(self): + return self.handle.tell() + + +@contextmanager +def read( + file: Union[str, PathLike, BinaryIO], + description: str = "Read...", + total: Optional[int] = None, + auto_refresh: bool = True, + console: Optional[Console] = None, + transient: bool = False, + get_time: Optional[Callable[[], float]] = None, + refresh_per_second: float = 10, + style: StyleType = "bar.back", + complete_style: StyleType = "bar.complete", + finished_style: StyleType = "bar.finished", + pulse_style: StyleType = "bar.pulse", + update_period: float = 0.1, + disable: bool = False, +) -> ContextManager[BinaryIO]: + """Read bytes from a file while tracking progress. + + Args: + sequence (Iterable[ProgressType]): A sequence (must support "len") you wish to iterate over. + description (str, optional): Description of task show next to progress bar. Defaults to "Working". + total: (int, optional): Total number of steps. Default is len(sequence). + auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True. + transient: (bool, optional): Clear the progress on exit. Defaults to False. + console (Console, optional): Console to write to. Default creates internal Console instance. + refresh_per_second (float): Number of times per second to refresh the progress information. Defaults to 10. + style (StyleType, optional): Style for the bar background. Defaults to "bar.back". + complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete". + finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.done". + pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse". + update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1. + disable (bool, optional): Disable display of progress. + Returns: + ContextManager[BinaryIO]: An iterable of the values in the sequence. + + """ + + columns: List["ProgressColumn"] = ( + [TextColumn("[progress.description]{task.description}")] if description else [] + ) + columns.extend( + ( + BarColumn( + style=style, + complete_style=complete_style, + finished_style=finished_style, + pulse_style=pulse_style, + ), + DownloadColumn(), + TimeRemainingColumn(), + ) + ) + progress = Progress( + *columns, + auto_refresh=auto_refresh, + console=console, + transient=transient, + get_time=get_time, + refresh_per_second=refresh_per_second or 10, + disable=disable, + ) + + with progress: + yield progress.read(file, total=total, description=description) + + class ProgressColumn(ABC): """Base class for a widget to use in progress display.""" @@ -794,6 +941,49 @@ def track( advance(task_id, 1) refresh() + def read( + self, + file: Union[str, PathLike, BinaryIO], + total: Optional[int] = None, + task_id: Optional[TaskID] = None, + description: str = "Reading...", + ) -> BinaryIO: + """Track progress by iterating over a sequence. + + Args: + file (Sequence[ProgressType]): A sequence of values you want to iterate over and track progress. + total: (int, optional): Total number of bytes to read. Must be provided if reading from a file handle. Default for a path is os.stat(file).st_size. + task_id: (TaskID): Task to track. Default is new task. + description: (str, optional): Description of task, if new task is created. + + Returns: + BinaryIO: A readable file-like object in binary mode. + """ + + if total is None: + if isinstance(file, (str, PathLike)): + task_total = stat(file).st_size + else: + raise ValueError( + f"unable to get size of {file!r}, please specify 'total'" + ) + else: + task_total = total + + if task_id is None: + task_id = self.add_task(description, total=task_total) + else: + self.update(task_id, total=task_total) + + if isinstance(file, (str, PathLike)): + handle = open(file, "rb") + else: + if not isinstance(file.read(0), bytes): + raise ValueError("expected file open in binary mode") + handle = file + + return _Reader(handle, self, task_id) + def start_task(self, task_id: TaskID) -> None: """Start a task. From 5b2a4a77173f7326d5e62de1d92432f0d2d468a2 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 13:56:31 +0100 Subject: [PATCH 02/25] Update documentation of `rich.progress.Progress.read` --- rich/progress.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index 6b6c39996..d0993fa26 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -232,7 +232,7 @@ def tell(self): @contextmanager def read( file: Union[str, PathLike, BinaryIO], - description: str = "Read...", + description: str = "Reading...", total: Optional[int] = None, auto_refresh: bool = True, console: Optional[Console] = None, @@ -249,9 +249,9 @@ def read( """Read bytes from a file while tracking progress. Args: - sequence (Iterable[ProgressType]): A sequence (must support "len") you wish to iterate over. - description (str, optional): Description of task show next to progress bar. Defaults to "Working". - total: (int, optional): Total number of steps. Default is len(sequence). + file (Union[str, PathLike, BinaryIO]): The path to the file to read, or a file-like object in binary mode. + description (str, optional): Description of task show next to progress bar. Defaults to "Reading". + total: (int, optional): Total number of bytes to read. Must be provided if reading from a file handle. Default for a path is os.stat(file).st_size. auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True. transient: (bool, optional): Clear the progress on exit. Defaults to False. console (Console, optional): Console to write to. Default creates internal Console instance. @@ -948,10 +948,10 @@ def read( task_id: Optional[TaskID] = None, description: str = "Reading...", ) -> BinaryIO: - """Track progress by iterating over a sequence. + """Track progress while reading from a binary file. Args: - file (Sequence[ProgressType]): A sequence of values you want to iterate over and track progress. + file (Union[str, PathLike, BinaryIO]): The path to the file to read, or a file-like object in binary mode. total: (int, optional): Total number of bytes to read. Must be provided if reading from a file handle. Default for a path is os.stat(file).st_size. task_id: (TaskID): Task to track. Default is new task. description: (str, optional): Description of task, if new task is created. From a5fa23134894412f2b9c9d94a5793d034cde4799 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 14:27:34 +0100 Subject: [PATCH 03/25] Update `Progress.read` to close inner file only if given a path --- rich/progress.py | 52 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index d0993fa26..073200c6d 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -13,6 +13,7 @@ Any, BinaryIO, Callable, + ContextManager, Deque, Dict, Iterable, @@ -156,17 +157,24 @@ def track( class _Reader(RawIOBase): """A reader that tracks progress while it's being read from.""" - def __init__(self, handle: BinaryIO, progress: "Progress", task: TaskID): + def __init__(self, handle: BinaryIO, progress: "Progress", task: TaskID, close_handle: bool = True): self.handle = handle self.progress = progress self.task = task + self.close_handle = close_handle + self._closed = False def __enter__(self): self.handle.__enter__() return self - def __exit__(self, exc_val, exc_ty, tb): - return self.handle.__exit__(exc_val, exc_ty, tb) + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self.close() def __iter__(self): return self @@ -178,7 +186,7 @@ def __next__(self): @property def closed(self): - return self.closed + return self._closed def isatty(self): return self.handle.isatty() @@ -218,7 +226,9 @@ def readlines(self, hint=-1): return lines def close(self): - self.handle.close() + if self.close_handle: + self.handle.close() + self._closed = True def seek(self, offset, whence=0): pos = self.handle.seek(offset, whence) @@ -229,7 +239,27 @@ def tell(self): return self.handle.tell() -@contextmanager +class _ReadContext(ContextManager[_Reader]): + """A utility class to handle a context for both a reader and a progress.""" + + def __init__(self, progress: "Progress", reader: _Reader) -> None: + self.progress = progress + self.reader = reader + + def __enter__(self) -> _Reader: + self.progress.start() + return self.reader.__enter__() + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> bool: + self.progress.stop() + return self.reader.__exit__(exc_type, exc_val, exc_tb) + + def read( file: Union[str, PathLike, BinaryIO], description: str = "Reading...", @@ -263,7 +293,7 @@ def read( update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1. disable (bool, optional): Disable display of progress. Returns: - ContextManager[BinaryIO]: An iterable of the values in the sequence. + ContextManager[BinaryIO]: A context manager yielding a progress reader. """ @@ -292,8 +322,8 @@ def read( disable=disable, ) - with progress: - yield progress.read(file, total=total, description=description) + reader = progress.read(file, total=total, description=description) + return _ReadContext(progress, reader) class ProgressColumn(ABC): @@ -977,12 +1007,14 @@ def read( if isinstance(file, (str, PathLike)): handle = open(file, "rb") + close_handle = True else: if not isinstance(file.read(0), bytes): raise ValueError("expected file open in binary mode") handle = file + close_handle = False - return _Reader(handle, self, task_id) + return _Reader(handle, self, task_id, close_handle=close_handle) def start_task(self, task_id: TaskID) -> None: """Start a task. From d6f61d83dea5ecac150e1b2d8e0d87421d0fb569 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 14:28:43 +0100 Subject: [PATCH 04/25] Add tests to `tests.test_progress` to make sure files are closed properly --- tests/test_progress.py | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_progress.py b/tests/test_progress.py index db2e825d3..c7e7e0bed 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -1,6 +1,8 @@ # encoding=utf-8 import io +import os +import tempfile from time import sleep from types import SimpleNamespace @@ -15,6 +17,7 @@ TotalFileSizeColumn, DownloadColumn, TransferSpeedColumn, + read, RenderableColumn, SpinnerColumn, MofNCompleteColumn, @@ -549,6 +552,58 @@ def test_no_output_if_progress_is_disabled() -> None: assert result == expected +def test_read_file_closed() -> None: + console = Console( + file=io.StringIO(), + force_terminal=True, + width=60, + color_system="truecolor", + legacy_windows=False, + _environ={}, + ) + progress = Progress( + console=console, + ) + + fd, filename = tempfile.mkstemp() + with os.fdopen(fd, "wb") as f: + f.write(b"Hello, World!") + try: + with read(filename) as f: + assert f.read() == b"Hello, World!" + assert f.closed + assert f.handle.closed + finally: + os.remove(filename) + +def test_read_filehandle_not_closed() -> None: + console = Console( + file=io.StringIO(), + force_terminal=True, + width=60, + color_system="truecolor", + legacy_windows=False, + _environ={}, + ) + progress = Progress( + console=console, + ) + + fd, filename = tempfile.mkstemp() + with os.fdopen(fd, "wb") as f: + total = f.write(b"Hello, World!") + try: + with open(filename, "rb") as file: + with read(file, total=total) as f: + assert f.read() == b"Hello, World!" + assert f.closed + assert not f.handle.closed + assert not file.closed + assert file.closed + finally: + os.remove(filename) + + if __name__ == "__main__": _render = render_progress() print(_render) From 5cfa2039c1111d2083c76448458862706d7e139c Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 14:51:57 +0100 Subject: [PATCH 05/25] Add missing type annotations to `rich.progress._Reader` --- rich/progress.py | 55 +++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index 073200c6d..d0fbfa86b 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -4,8 +4,9 @@ from contextlib import contextmanager from dataclasses import dataclass, field from datetime import timedelta -from io import RawIOBase +from io import RawIOBase, UnsupportedOperation from math import ceil +from mmap import mmap from os import PathLike, stat from threading import Event, RLock, Thread from types import TracebackType @@ -154,10 +155,10 @@ def track( ) -class _Reader(RawIOBase): +class _Reader(RawIOBase, BinaryIO): """A reader that tracks progress while it's being read from.""" - def __init__(self, handle: BinaryIO, progress: "Progress", task: TaskID, close_handle: bool = True): + def __init__(self, handle: BinaryIO, progress: "Progress", task: TaskID, close_handle: bool = True) -> None: self.handle = handle self.progress = progress self.task = task @@ -176,77 +177,83 @@ def __exit__( ) -> None: self.close() - def __iter__(self): + def __iter__(self) -> BinaryIO: return self - def __next__(self): + def __next__(self) -> bytes: line = next(self.handle) self.progress.advance(self.task, advance=len(line)) return line @property - def closed(self): + def closed(self) -> bool: return self._closed - def isatty(self): + def fileno(self) -> int: + return self.handle.fileno() + + def isatty(self) -> bool: return self.handle.isatty() - def readable(self): + def readable(self) -> bool: return self.handle.readable() - def seekable(self): + def seekable(self) -> bool: return self.handle.seekable() - def writable(self): + def writable(self) -> bool: return False - def read(self, size=-1): + def read(self, size: int = -1) -> bytes: block = self.handle.read(size) self.progress.advance(self.task, advance=len(block)) return block - def readall(self): + def readall(self) -> bytes: block = self.handle.readall() self.progress.advance(self.task, advance=len(block)) return block - def readinto(self, b): - n = self.handle.readinto(b) + def readinto(self, b: Union[bytearray, memoryview, mmap]): # type: ignore + n = self.handle.readinto(b) # type: ignore self.progress.advance(self.task, advance=n) return n - def readline(self, size=-1): + def readline(self, size: int = -1) -> bytes: # type: ignore line = self.handle.readline(size) self.progress.advance(self.task, advance=len(line)) return line - def readlines(self, hint=-1): + def readlines(self, hint: int = -1) -> List[bytes]: lines = self.handle.readlines(hint) self.progress.advance(self.task, advance=sum(map(len, lines))) return lines - def close(self): + def close(self) -> None: if self.close_handle: self.handle.close() self._closed = True - def seek(self, offset, whence=0): + def seek(self, offset: int, whence: int = 0): pos = self.handle.seek(offset, whence) self.progress.update(self.task, completed=pos) return pos - def tell(self): + def tell(self) -> int: return self.handle.tell() + def write(self, s: Any) -> int: + raise UnsupportedOperation("write") + -class _ReadContext(ContextManager[_Reader]): +class _ReadContext(ContextManager[BinaryIO]): """A utility class to handle a context for both a reader and a progress.""" - def __init__(self, progress: "Progress", reader: _Reader) -> None: + def __init__(self, progress: "Progress", reader: BinaryIO) -> None: self.progress = progress self.reader = reader - def __enter__(self) -> _Reader: + def __enter__(self) -> BinaryIO: self.progress.start() return self.reader.__enter__() @@ -255,9 +262,9 @@ def __exit__( exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], - ) -> bool: + ) -> None: self.progress.stop() - return self.reader.__exit__(exc_type, exc_val, exc_tb) + self.reader.__exit__(exc_type, exc_val, exc_tb) def read( From 1432457bf1017b28be5391d5a2082a39061296cb Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 14:52:38 +0100 Subject: [PATCH 06/25] Add an example showing how to copy a file with a progress bar --- examples/cp_progress.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 examples/cp_progress.py diff --git a/examples/cp_progress.py b/examples/cp_progress.py new file mode 100644 index 000000000..fc2291ec8 --- /dev/null +++ b/examples/cp_progress.py @@ -0,0 +1,39 @@ +""" +A very minimal `cp` clone that displays a progress bar. +""" +import os +import shutil +import sys + +from rich.progress import ( + BarColumn, + DownloadColumn, + Progress, + TaskID, + TextColumn, + TimeRemainingColumn, + TransferSpeedColumn, +) + +progress = Progress( + TextColumn("[bold blue]{task.description}", justify="right"), + BarColumn(bar_width=None), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + DownloadColumn(), + "•", + TransferSpeedColumn(), + "•", + TimeRemainingColumn(), +) + +if __name__ == "__main__": + if len(sys.argv) == 3: + + with progress: + desc=os.path.basename(sys.argv[1]) + with progress.read(sys.argv[1], description=desc) as src: + with open(sys.argv[2], "wb") as dst: + shutil.copyfileobj(src, dst) + else: + print("Usage:\n\tpython cp_progress.py SRC DST") From 46576a93bc6df96b5eb81443bb5f4d8ebc2ee5f9 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 14:56:55 +0100 Subject: [PATCH 07/25] Ignore type hint in `rich.progress._Reader.readall` --- rich/progress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rich/progress.py b/rich/progress.py index d0fbfa86b..4bd26cf74 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -210,7 +210,7 @@ def read(self, size: int = -1) -> bytes: return block def readall(self) -> bytes: - block = self.handle.readall() + block = self.handle.readall() # type: ignore self.progress.advance(self.task, advance=len(block)) return block From 97da885c04b8b9f88144a2a73f1ee161174bdc51 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 14:57:46 +0100 Subject: [PATCH 08/25] Reformat new `progress` code with `black` --- examples/cp_progress.py | 2 +- rich/progress.py | 8 +++++++- tests/test_progress.py | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/cp_progress.py b/examples/cp_progress.py index fc2291ec8..0f4059d02 100644 --- a/examples/cp_progress.py +++ b/examples/cp_progress.py @@ -31,7 +31,7 @@ if len(sys.argv) == 3: with progress: - desc=os.path.basename(sys.argv[1]) + desc = os.path.basename(sys.argv[1]) with progress.read(sys.argv[1], description=desc) as src: with open(sys.argv[2], "wb") as dst: shutil.copyfileobj(src, dst) diff --git a/rich/progress.py b/rich/progress.py index 4bd26cf74..1bda66ca4 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -158,7 +158,13 @@ def track( class _Reader(RawIOBase, BinaryIO): """A reader that tracks progress while it's being read from.""" - def __init__(self, handle: BinaryIO, progress: "Progress", task: TaskID, close_handle: bool = True) -> None: + def __init__( + self, + handle: BinaryIO, + progress: "Progress", + task: TaskID, + close_handle: bool = True, + ) -> None: self.handle = handle self.progress = progress self.task = task diff --git a/tests/test_progress.py b/tests/test_progress.py index c7e7e0bed..da3879b46 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -576,6 +576,7 @@ def test_read_file_closed() -> None: finally: os.remove(filename) + def test_read_filehandle_not_closed() -> None: console = Console( file=io.StringIO(), From 5e8da24c4eb13721e541da093688c9610f541e7f Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 15:02:08 +0100 Subject: [PATCH 09/25] Fix additional typing issues in `rich.progress` --- CHANGELOG.md | 7 +++++++ CONTRIBUTORS.md | 1 + rich/progress.py | 12 ++++++------ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24ffa527d..79b01a84c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- Progress.read method to track the progress while reading from a file https://github.com/willmcgugan/rich/pull/1759 + + ## [12.0.0] - 2022-03-10 ### Added diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 140f77f43..f3c0237a4 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -18,6 +18,7 @@ The following people have contributed to the development of Rich: - [Finn Hughes](https://github.com/finnhughes) - [Josh Karpel](https://github.com/JoshKarpel) - [Andrew Kettmann](https://github.com/akettmann) +- [Martin Larralde](https://github.com/althonos) - [Hedy Li](https://github.com/hedythedev) - [Alexander Mancevice](https://github.com/amancevice) - [Will McGugan](https://github.com/willmcgugan) diff --git a/rich/progress.py b/rich/progress.py index 1bda66ca4..b22568e6c 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -171,7 +171,7 @@ def __init__( self.close_handle = close_handle self._closed = False - def __enter__(self): + def __enter__(self) -> "_Reader": self.handle.__enter__() return self @@ -218,7 +218,7 @@ def read(self, size: int = -1) -> bytes: def readall(self) -> bytes: block = self.handle.readall() # type: ignore self.progress.advance(self.task, advance=len(block)) - return block + return block # type: ignore def readinto(self, b: Union[bytearray, memoryview, mmap]): # type: ignore n = self.handle.readinto(b) # type: ignore @@ -240,7 +240,7 @@ def close(self) -> None: self.handle.close() self._closed = True - def seek(self, offset: int, whence: int = 0): + def seek(self, offset: int, whence: int = 0) -> int: pos = self.handle.seek(offset, whence) self.progress.update(self.task, completed=pos) return pos @@ -274,7 +274,7 @@ def __exit__( def read( - file: Union[str, PathLike, BinaryIO], + file: Union[str, PathLike[str], BinaryIO], description: str = "Reading...", total: Optional[int] = None, auto_refresh: bool = True, @@ -986,7 +986,7 @@ def track( def read( self, - file: Union[str, PathLike, BinaryIO], + file: Union[str, PathLike[str], BinaryIO], total: Optional[int] = None, task_id: Optional[TaskID] = None, description: str = "Reading...", @@ -994,7 +994,7 @@ def read( """Track progress while reading from a binary file. Args: - file (Union[str, PathLike, BinaryIO]): The path to the file to read, or a file-like object in binary mode. + file (Union[str, PathLike[str], BinaryIO]): The path to the file to read, or a file-like object in binary mode. total: (int, optional): Total number of bytes to read. Must be provided if reading from a file handle. Default for a path is os.stat(file).st_size. task_id: (TaskID): Task to track. Default is new task. description: (str, optional): Description of task, if new task is created. From f3f1bca230cf8e1bdfe183674186af2d396539f2 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 16:54:33 +0100 Subject: [PATCH 10/25] Reformat `rich.progress` and document `Progress.read` in `progress.rst` --- docs/source/progress.rst | 46 ++++++++++++++++++++++++++++++++++++---- rich/progress.py | 2 +- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/docs/source/progress.rst b/docs/source/progress.rst index 4ce358836..71e7a7c5d 100644 --- a/docs/source/progress.rst +++ b/docs/source/progress.rst @@ -26,6 +26,16 @@ For basic usage call the :func:`~rich.progress.track` function, which accepts a for n in track(range(n), description="Processing..."): do_work(n) + +To get a progress bar while reading from a file, you may consider using the :func:`~rich.progress.read` function, which accepts a path, or a *file-like* object. It will return a *file-like* object in *binary mode* that will update the progress information as it's being read from. Here's an example, tracking the progresses made by :func:`json.load` to load a file:: + + import json + from rich.progress import read + + with read("data.json", description="Loading data...") as f: + data = json.load(f) + + Advanced usage -------------- @@ -34,9 +44,9 @@ If you require multiple tasks in the display, or wish to configure the columns i The Progress class is designed to be used as a *context manager* which will start and stop the progress display automatically. Here's a simple example:: - + import time - + from rich.progress import Progress with Progress() as progress: @@ -179,7 +189,7 @@ If you have another Console object you want to use, pass it in to the :class:`~r with Progress(console=my_console) as progress: my_console.print("[bold blue]Starting work!") do_work(progress) - + Redirecting stdout / stderr ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -199,6 +209,35 @@ If the :class:`~rich.progress.Progress` class doesn't offer exactly what you nee def get_renderables(self): yield Panel(self.make_tasks_table(self.tasks)) +Reading from a file +~~~~~~~~~~~~~~~~~~~ + +You can obtain a progress-tracking reader using the :meth:`~rich.progress.Progress.read` method by giving either a path or a *file-like* object. When a path is given, :meth:`~rich.progress.Progress.read` will query the size of the file with :func:`os.stat`, and take care of opening the file for you, but you are still responsible for closing it. For this, you should consider using a *context*:: + + from rich.progress import Progress + + with Progress() as progress: + with progress.read("file.bin") as file: + do_work(file) + +If a file-like object is provided, it must be in *binary mode*, and the total size must be provided to :meth:`~rich.progress.Progress.read` with the ``total`` argument. In that case :meth:`~rich.progress.Progress.read` will not close the file-like object, so you need to take care of that yourself:: + + from rich.progress import Progress + + with Progress() as progress: + with open("file.bin", "rb") as file: + do_work(progress.read(file, total=2048)) + +If the API consuming the file is expecting an object in *text mode* (for instance, :func:`csv.reader`), you can always wrap the object returned by :meth:`~rich.progress.Progress.read` in an :class:`io.TextIOWrapper`:: + + import io + from rich.progress import Progress + + with Progress() as progress: + with progress.read("file.bin") as file: + do_work_txt(io.TextIOWrapper(file)) + + Multiple Progress ----------------- @@ -208,4 +247,3 @@ Example ------- See `downloader.py `_ for a realistic application of a progress display. This script can download multiple concurrent files with a progress bar, transfer speed and file size. - diff --git a/rich/progress.py b/rich/progress.py index b22568e6c..b3c79cf8c 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -218,7 +218,7 @@ def read(self, size: int = -1) -> bytes: def readall(self) -> bytes: block = self.handle.readall() # type: ignore self.progress.advance(self.task, advance=len(block)) - return block # type: ignore + return block # type: ignore def readinto(self, b: Union[bytearray, memoryview, mmap]): # type: ignore n = self.handle.readinto(b) # type: ignore From a270e60158f158d62679bf04d8c18d7b19ff0d67 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 16 Dec 2021 17:58:15 +0100 Subject: [PATCH 11/25] Fix `os.PathLike` type annotations in `rich.progress` --- rich/progress.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index b3c79cf8c..bd8947506 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -274,7 +274,7 @@ def __exit__( def read( - file: Union[str, PathLike[str], BinaryIO], + file: Union[str, "PathLike[str]", BinaryIO], description: str = "Reading...", total: Optional[int] = None, auto_refresh: bool = True, @@ -292,7 +292,7 @@ def read( """Read bytes from a file while tracking progress. Args: - file (Union[str, PathLike, BinaryIO]): The path to the file to read, or a file-like object in binary mode. + file (Union[str, PathLike[str], BinaryIO]): The path to the file to read, or a file-like object in binary mode. description (str, optional): Description of task show next to progress bar. Defaults to "Reading". total: (int, optional): Total number of bytes to read. Must be provided if reading from a file handle. Default for a path is os.stat(file).st_size. auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True. @@ -986,7 +986,7 @@ def track( def read( self, - file: Union[str, PathLike[str], BinaryIO], + file: Union[str, "PathLike[str]", BinaryIO], total: Optional[int] = None, task_id: Optional[TaskID] = None, description: str = "Reading...", From 3008780fc663a4eb689fab19a3b2823e126950ee Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 18 Dec 2021 12:30:52 +0100 Subject: [PATCH 12/25] Remove unused import from `rich.progress` --- rich/progress.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rich/progress.py b/rich/progress.py index bd8947506..ab55da489 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -1,7 +1,6 @@ from abc import ABC, abstractmethod from collections import deque from collections.abc import Sized -from contextlib import contextmanager from dataclasses import dataclass, field from datetime import timedelta from io import RawIOBase, UnsupportedOperation From 1320046537e1a386e60324bb2f9fb2eb8f23569d Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 13 Mar 2022 18:36:47 +0100 Subject: [PATCH 13/25] Split `Progress.read` into `Progress.open` and `Progress.wrap_file` methods --- docs/source/progress.rst | 26 ++++-- rich/progress.py | 181 ++++++++++++++++++++++++++++++++------- tests/test_progress.py | 34 ++++++-- 3 files changed, 197 insertions(+), 44 deletions(-) diff --git a/docs/source/progress.rst b/docs/source/progress.rst index 71e7a7c5d..c0e565e6f 100644 --- a/docs/source/progress.rst +++ b/docs/source/progress.rst @@ -212,30 +212,38 @@ If the :class:`~rich.progress.Progress` class doesn't offer exactly what you nee Reading from a file ~~~~~~~~~~~~~~~~~~~ -You can obtain a progress-tracking reader using the :meth:`~rich.progress.Progress.read` method by giving either a path or a *file-like* object. When a path is given, :meth:`~rich.progress.Progress.read` will query the size of the file with :func:`os.stat`, and take care of opening the file for you, but you are still responsible for closing it. For this, you should consider using a *context*:: +You can obtain a progress-tracking reader using the :meth:`~rich.progress.Progress.open` method by giving it a path. You can specify the number of bytes to be read, but by default :meth:`~rich.progress.Progress.open` will query the size of the file with :func:`os.stat`. You are responsible for closing the file, and you should consider using a *context* to make sure it is closed :: + import json from rich.progress import Progress with Progress() as progress: - with progress.read("file.bin") as file: - do_work(file) + with progress.open("file.json", "rb") as file: + json.load(file) + -If a file-like object is provided, it must be in *binary mode*, and the total size must be provided to :meth:`~rich.progress.Progress.read` with the ``total`` argument. In that case :meth:`~rich.progress.Progress.read` will not close the file-like object, so you need to take care of that yourself:: +Note that in the above snippet we use the `"rb"` mode, because we needed the file to be opened in binary mode to pass it to :func:`json.load`. If the API consuming the file is expecting an object in *text mode* (for instance, :func:`csv.reader`), you can open the file with the `"r"` mode, which happens to be the default :: from rich.progress import Progress with Progress() as progress: - with open("file.bin", "rb") as file: - do_work(progress.read(file, total=2048)) + with progress.open("README.md") as file: + for line in file: + print(line) + -If the API consuming the file is expecting an object in *text mode* (for instance, :func:`csv.reader`), you can always wrap the object returned by :meth:`~rich.progress.Progress.read` in an :class:`io.TextIOWrapper`:: +Reading from a file-like object +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can obtain a progress-tracking reader wrapping a file-like object using the :meth:`~rich.progress.Progress.wrap_file` method. The file-like object must be in *binary mode*, and a total must be provided, unless it was provided to a :class:`~rich.progress.Task` created beforehand. The returned reader may be used in a context, but will not take care of closing the wrapped file :: import io + import json from rich.progress import Progress with Progress() as progress: - with progress.read("file.bin") as file: - do_work_txt(io.TextIOWrapper(file)) + file = io.BytesIO("...") + json.load(progress.read(file, total=2048)) Multiple Progress diff --git a/rich/progress.py b/rich/progress.py index ab55da489..b64a47ca2 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -1,3 +1,4 @@ +import io from abc import ABC, abstractmethod from collections import deque from collections.abc import Sized @@ -272,10 +273,10 @@ def __exit__( self.reader.__exit__(exc_type, exc_val, exc_tb) -def read( - file: Union[str, "PathLike[str]", BinaryIO], +def wrap_file( + file: BinaryIO, + total: int, description: str = "Reading...", - total: Optional[int] = None, auto_refresh: bool = True, console: Optional[Console] = None, transient: bool = False, @@ -292,8 +293,77 @@ def read( Args: file (Union[str, PathLike[str], BinaryIO]): The path to the file to read, or a file-like object in binary mode. + total (int): Total number of bytes to read. description (str, optional): Description of task show next to progress bar. Defaults to "Reading". + auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True. + transient: (bool, optional): Clear the progress on exit. Defaults to False. + console (Console, optional): Console to write to. Default creates internal Console instance. + refresh_per_second (float): Number of times per second to refresh the progress information. Defaults to 10. + style (StyleType, optional): Style for the bar background. Defaults to "bar.back". + complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete". + finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.done". + pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse". + update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1. + disable (bool, optional): Disable display of progress. + Returns: + ContextManager[BinaryIO]: A context manager yielding a progress reader. + + """ + + columns: List["ProgressColumn"] = ( + [TextColumn("[progress.description]{task.description}")] if description else [] + ) + columns.extend( + ( + BarColumn( + style=style, + complete_style=complete_style, + finished_style=finished_style, + pulse_style=pulse_style, + ), + DownloadColumn(), + TimeRemainingColumn(), + ) + ) + progress = Progress( + *columns, + auto_refresh=auto_refresh, + console=console, + transient=transient, + get_time=get_time, + refresh_per_second=refresh_per_second or 10, + disable=disable, + ) + + reader = progress.wrap_file(file, total=total, description=description) + return _ReadContext(progress, reader) + + +def open( + file: Union[str, "PathLike[str]", bytes], + mode: str = "rb", + total: Optional[int] = None, + description: str = "Reading...", + auto_refresh: bool = True, + console: Optional[Console] = None, + transient: bool = False, + get_time: Optional[Callable[[], float]] = None, + refresh_per_second: float = 10, + style: StyleType = "bar.back", + complete_style: StyleType = "bar.complete", + finished_style: StyleType = "bar.finished", + pulse_style: StyleType = "bar.pulse", + update_period: float = 0.1, + disable: bool = False, + encoding: Optional[str] = None, +) -> ContextManager[BinaryIO]: + """Read bytes from a file while tracking progress. + + Args: + path (Union[str, PathLike[str], BinaryIO]): The path to the file to read, or a file-like object in binary mode. + mode (str): The mode to use to open the file. Only supports "r", "rb" or "rt". total: (int, optional): Total number of bytes to read. Must be provided if reading from a file handle. Default for a path is os.stat(file).st_size. + description (str, optional): Description of task show next to progress bar. Defaults to "Reading". auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True. transient: (bool, optional): Clear the progress on exit. Defaults to False. console (Console, optional): Console to write to. Default creates internal Console instance. @@ -304,6 +374,8 @@ def read( pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse". update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1. disable (bool, optional): Disable display of progress. + encoding (str, optional): The encoding to use when reading in text mode. + Returns: ContextManager[BinaryIO]: A context manager yielding a progress reader. @@ -334,7 +406,13 @@ def read( disable=disable, ) - reader = progress.read(file, total=total, description=description) + reader = progress.open( + file, + mode=mode, + total=total, + description=description, + encoding=encoding, + ) return _ReadContext(progress, reader) @@ -983,50 +1061,93 @@ def track( advance(task_id, 1) refresh() - def read( + def wrap_file( self, - file: Union[str, "PathLike[str]", BinaryIO], + file: BinaryIO, total: Optional[int] = None, task_id: Optional[TaskID] = None, description: str = "Reading...", ) -> BinaryIO: - """Track progress while reading from a binary file. + """Track progress file reading from a binary file. Args: - file (Union[str, PathLike[str], BinaryIO]): The path to the file to read, or a file-like object in binary mode. - total: (int, optional): Total number of bytes to read. Must be provided if reading from a file handle. Default for a path is os.stat(file).st_size. - task_id: (TaskID): Task to track. Default is new task. - description: (str, optional): Description of task, if new task is created. + file (BinaryIO): A file-like object opened in binary mode. + total (int, optional): Total number of bytes to read. This must be provided unless a task with a total is also given. + task_id (TaskID): Task to track. Default is new task. + description (str, optional): Description of task, if new task is created. Returns: BinaryIO: A readable file-like object in binary mode. - """ + Raises: + ValueError: When no total value can be extracted from the arguments or the task. + """ + # attempt to recover the total from the task + if total is None and task_id is not None: + with self._lock: + task = self._tasks[task_id].total if total is None: - if isinstance(file, (str, PathLike)): - task_total = stat(file).st_size - else: - raise ValueError( - f"unable to get size of {file!r}, please specify 'total'" - ) - else: - task_total = total + raise ValueError( + f"unable to get the total number of bytes, please specify 'total'" + ) + # update total of task or create new task if task_id is None: - task_id = self.add_task(description, total=task_total) + task_id = self.add_task(description, total=total) else: - self.update(task_id, total=task_total) + self.update(task_id, total=total) + + # return a reader + return _Reader(file, self, task_id, close_handle=False) + + def open( + self, + file: Union[str, "PathLike[str]", bytes], + mode: str = "r", + total: Optional[int] = None, + task_id: Optional[TaskID] = None, + description: str = "Reading...", + encoding: Optional[str] = None, + ) -> BinaryIO: + """Track progress while reading from a binary file. - if isinstance(file, (str, PathLike)): - handle = open(file, "rb") - close_handle = True + Args: + path (Union[str, PathLike[str]]): The path to the file to read. + mode (str): The mode to use to open the file. Only supports "r", "rb" or "rt". + total (int, optional): Total number of bytes to read. If none given, os.stat(path).st_size is used. + task_id (TaskID): Task to track. Default is new task. + description (str, optional): Description of task, if new task is created. + encoding (str, optional): The encoding to use when reading in text mode. + + Returns: + BinaryIO: A readable file-like object in binary mode. + + Raises: + ValueError: When an invalid mode is given. + """ + # attempt to get the total with `os.stat` + if total is None: + total = stat(file).st_size + + # update total of task or create new task + if task_id is None: + task_id = self.add_task(description, total=total) else: - if not isinstance(file.read(0), bytes): - raise ValueError("expected file open in binary mode") - handle = file - close_handle = False + self.update(task_id, total=total) + + # normalize the mode (always rb, rt) + _mode = "".join(sorted(mode, reverse=False)) + if _mode not in ("br", "rt", "r"): + raise ValueError("invalid mode {!r}".format(mode)) + + # open the file in binary mode, + reader = _Reader(io.open(file, "rb"), self, task_id, close_handle=True) + + # wrap the reader in a `TextIOWrapper` if text mode + if mode == "r" or mode == "rt": + reader = io.TextIOWrapper(reader, encoding=encoding) - return _Reader(handle, self, task_id, close_handle=close_handle) + return reader def start_task(self, task_id: TaskID) -> None: """Start a task. diff --git a/tests/test_progress.py b/tests/test_progress.py index da3879b46..57f98965a 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -8,6 +8,7 @@ import pytest +import rich.progress from rich.progress_bar import ProgressBar from rich.console import Console from rich.highlighter import NullHighlighter @@ -17,7 +18,6 @@ TotalFileSizeColumn, DownloadColumn, TransferSpeedColumn, - read, RenderableColumn, SpinnerColumn, MofNCompleteColumn, @@ -552,7 +552,7 @@ def test_no_output_if_progress_is_disabled() -> None: assert result == expected -def test_read_file_closed() -> None: +def test_open() -> None: console = Console( file=io.StringIO(), force_terminal=True, @@ -569,7 +569,7 @@ def test_read_file_closed() -> None: with os.fdopen(fd, "wb") as f: f.write(b"Hello, World!") try: - with read(filename) as f: + with rich.progress.open(filename) as f: assert f.read() == b"Hello, World!" assert f.closed assert f.handle.closed @@ -577,7 +577,31 @@ def test_read_file_closed() -> None: os.remove(filename) -def test_read_filehandle_not_closed() -> None: +def test_open_text_mode() -> None: + console = Console( + file=io.StringIO(), + force_terminal=True, + width=60, + color_system="truecolor", + legacy_windows=False, + _environ={}, + ) + progress = Progress( + console=console, + ) + + fd, filename = tempfile.mkstemp() + with os.fdopen(fd, "wb") as f: + f.write(b"Hello, World!") + try: + with rich.progress.open(filename, "r") as f: + assert f.read() == "Hello, World!" + assert f.closed + finally: + os.remove(filename) + + +def test_wrap_file() -> None: console = Console( file=io.StringIO(), force_terminal=True, @@ -595,7 +619,7 @@ def test_read_filehandle_not_closed() -> None: total = f.write(b"Hello, World!") try: with open(filename, "rb") as file: - with read(file, total=total) as f: + with rich.progress.wrap_file(file, total=total) as f: assert f.read() == b"Hello, World!" assert f.closed assert not f.handle.closed From fc341f7f7371e22b4785311a4c3b612da80ff07b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 13 Mar 2022 18:45:29 +0100 Subject: [PATCH 14/25] Add test to ensure `Progress.wrap_file` can extract the total from a task --- rich/progress.py | 2 +- tests/test_progress.py | 40 +++++++++++++++++++++------------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index b64a47ca2..f61bacc58 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -1085,7 +1085,7 @@ def wrap_file( # attempt to recover the total from the task if total is None and task_id is not None: with self._lock: - task = self._tasks[task_id].total + total = self._tasks[task_id].total if total is None: raise ValueError( f"unable to get the total number of bytes, please specify 'total'" diff --git a/tests/test_progress.py b/tests/test_progress.py index 57f98965a..b462f47e7 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -578,18 +578,6 @@ def test_open() -> None: def test_open_text_mode() -> None: - console = Console( - file=io.StringIO(), - force_terminal=True, - width=60, - color_system="truecolor", - legacy_windows=False, - _environ={}, - ) - progress = Progress( - console=console, - ) - fd, filename = tempfile.mkstemp() with os.fdopen(fd, "wb") as f: f.write(b"Hello, World!") @@ -602,6 +590,22 @@ def test_open_text_mode() -> None: def test_wrap_file() -> None: + fd, filename = tempfile.mkstemp() + with os.fdopen(fd, "wb") as f: + total = f.write(b"Hello, World!") + try: + with open(filename, "rb") as file: + with rich.progress.wrap_file(file, total=total) as f: + assert f.read() == b"Hello, World!" + assert f.closed + assert not f.handle.closed + assert not file.closed + assert file.closed + finally: + os.remove(filename) + + +def test_wrap_file_task_total() -> None: console = Console( file=io.StringIO(), force_terminal=True, @@ -618,13 +622,11 @@ def test_wrap_file() -> None: with os.fdopen(fd, "wb") as f: total = f.write(b"Hello, World!") try: - with open(filename, "rb") as file: - with rich.progress.wrap_file(file, total=total) as f: - assert f.read() == b"Hello, World!" - assert f.closed - assert not f.handle.closed - assert not file.closed - assert file.closed + with progress: + with open(filename, "rb") as file: + task_id = progress.add_task("Reading", total=total) + with progress.wrap_file(file, task_id=task_id) as f: + assert f.read() == b"Hello, World!" finally: os.remove(filename) From 866307358f21a727dced87aed3ec2aa34230fefd Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 13 Mar 2022 19:21:18 +0100 Subject: [PATCH 15/25] Make `Progress.open` have a signature compatible with `open` --- rich/progress.py | 60 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index f61bacc58..6dbde7655 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -1,4 +1,5 @@ import io +import warnings from abc import ABC, abstractmethod from collections import deque from collections.abc import Sized @@ -342,6 +343,10 @@ def wrap_file( def open( file: Union[str, "PathLike[str]", bytes], mode: str = "rb", + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, total: Optional[int] = None, description: str = "Reading...", auto_refresh: bool = True, @@ -355,13 +360,16 @@ def open( pulse_style: StyleType = "bar.pulse", update_period: float = 0.1, disable: bool = False, - encoding: Optional[str] = None, ) -> ContextManager[BinaryIO]: """Read bytes from a file while tracking progress. Args: path (Union[str, PathLike[str], BinaryIO]): The path to the file to read, or a file-like object in binary mode. mode (str): The mode to use to open the file. Only supports "r", "rb" or "rt". + buffering (int): The buffering strategy to use, see :func:`io.open`. + encoding (str, optional): The encoding to use when reading in text mode, see :func:`io.open`. + errors (str, optional): The error handling strategy for decoding errors, see :func:`io.open`. + newline (str, optional): The strategy for handling newlines in text mode, see :func:`io.open` total: (int, optional): Total number of bytes to read. Must be provided if reading from a file handle. Default for a path is os.stat(file).st_size. description (str, optional): Description of task show next to progress bar. Defaults to "Reading". auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True. @@ -409,9 +417,12 @@ def open( reader = progress.open( file, mode=mode, + buffering=buffering, + encoding=encoding, + errors=errors, + newline=newline, total=total, description=description, - encoding=encoding, ) return _ReadContext(progress, reader) @@ -1104,20 +1115,26 @@ def open( self, file: Union[str, "PathLike[str]", bytes], mode: str = "r", + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, total: Optional[int] = None, task_id: Optional[TaskID] = None, description: str = "Reading...", - encoding: Optional[str] = None, ) -> BinaryIO: """Track progress while reading from a binary file. Args: path (Union[str, PathLike[str]]): The path to the file to read. mode (str): The mode to use to open the file. Only supports "r", "rb" or "rt". + buffering (int): The buffering strategy to use, see :func:`io.open`. + encoding (str, optional): The encoding to use when reading in text mode, see :func:`io.open`. + errors (str, optional): The error handling strategy for decoding errors, see :func:`io.open`. + newline (str, optional): The strategy for handling newlines in text mode, see :func:`io.open`. total (int, optional): Total number of bytes to read. If none given, os.stat(path).st_size is used. task_id (TaskID): Task to track. Default is new task. description (str, optional): Description of task, if new task is created. - encoding (str, optional): The encoding to use when reading in text mode. Returns: BinaryIO: A readable file-like object in binary mode. @@ -1125,6 +1142,25 @@ def open( Raises: ValueError: When an invalid mode is given. """ + # normalize the mode (always rb, rt) + _mode = "".join(sorted(mode, reverse=False)) + if _mode not in ("br", "rt", "r"): + raise ValueError("invalid mode {!r}".format(mode)) + + # patch buffering to provide the same behaviour as the builtin `open` + line_buffering = buffering == 1 + if _mode == "br" and buffering == 1: + warnings.warn( + "line buffering (buffering=1) isn't supported in binary mode, the default buffer size will be used", + RuntimeWarning, + ) + buffering = -1 + elif _mode == "rt" or _mode == "r": + if buffering == 0: + raise ValueError("can't have unbuffered text I/O") + elif buffering == 1: + buffering = -1 + # attempt to get the total with `os.stat` if total is None: total = stat(file).st_size @@ -1135,17 +1171,19 @@ def open( else: self.update(task_id, total=total) - # normalize the mode (always rb, rt) - _mode = "".join(sorted(mode, reverse=False)) - if _mode not in ("br", "rt", "r"): - raise ValueError("invalid mode {!r}".format(mode)) - # open the file in binary mode, - reader = _Reader(io.open(file, "rb"), self, task_id, close_handle=True) + handle = io.open(file, "rb", buffering=buffering) + reader = _Reader(handle, self, task_id, close_handle=True) # wrap the reader in a `TextIOWrapper` if text mode if mode == "r" or mode == "rt": - reader = io.TextIOWrapper(reader, encoding=encoding) + reader = io.TextIOWrapper( + reader, + encoding=encoding, + errors=errors, + newline=newline, + line_buffering=line_buffering, + ) return reader From 2b8ec3baa88d9ab56b6705d32c2c70177176dad9 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 13 Mar 2022 19:44:43 +0100 Subject: [PATCH 16/25] Add overloaded annotations for `Progress.open` and `rich.progress.open` --- rich/progress.py | 108 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 99 insertions(+), 9 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index 6dbde7655..15f308b13 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -1,4 +1,6 @@ import io +import sys +import typing import warnings from abc import ABC, abstractmethod from collections import deque @@ -18,18 +20,25 @@ ContextManager, Deque, Dict, + Generic, Iterable, List, NamedTuple, NewType, Optional, Sequence, + TextIO, Tuple, Type, TypeVar, Union, ) +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal # pragma: no cover + from . import filesize, get_console from .console import Console, JustifyMethod, RenderableType, Group from .highlighter import Highlighter @@ -48,6 +57,9 @@ GetTimeCallable = Callable[[], float] +_I = typing.TypeVar("_I", TextIO, BinaryIO) + + class _TrackThread(Thread): """A thread to periodically update progress.""" @@ -253,14 +265,14 @@ def write(self, s: Any) -> int: raise UnsupportedOperation("write") -class _ReadContext(ContextManager[BinaryIO]): +class _ReadContext(ContextManager[_I], Generic[_I]): """A utility class to handle a context for both a reader and a progress.""" - def __init__(self, progress: "Progress", reader: BinaryIO) -> None: + def __init__(self, progress: "Progress", reader: _I) -> None: self.progress = progress - self.reader = reader + self.reader: _I = reader - def __enter__(self) -> BinaryIO: + def __enter__(self) -> _I: self.progress.start() return self.reader.__enter__() @@ -340,9 +352,34 @@ def wrap_file( return _ReadContext(progress, reader) +@typing.overload def open( file: Union[str, "PathLike[str]", bytes], - mode: str = "rb", + mode: Union[Literal["rt"], Literal["r"]], + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + total: Optional[int] = None, + description: str = "Reading...", + auto_refresh: bool = True, + console: Optional[Console] = None, + transient: bool = False, + get_time: Optional[Callable[[], float]] = None, + refresh_per_second: float = 10, + style: StyleType = "bar.back", + complete_style: StyleType = "bar.complete", + finished_style: StyleType = "bar.finished", + pulse_style: StyleType = "bar.pulse", + update_period: float = 0.1, + disable: bool = False, +) -> ContextManager[TextIO]: + pass + +@typing.overload +def open( + file: Union[str, "PathLike[str]", bytes], + mode: Literal["rb"], buffering: int = -1, encoding: Optional[str] = None, errors: Optional[str] = None, @@ -361,6 +398,29 @@ def open( update_period: float = 0.1, disable: bool = False, ) -> ContextManager[BinaryIO]: + pass + +def open( + file: Union[str, "PathLike[str]", bytes], + mode: Union[Literal["rb"], Literal["rt"], Literal["r"]] = "r", + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + total: Optional[int] = None, + description: str = "Reading...", + auto_refresh: bool = True, + console: Optional[Console] = None, + transient: bool = False, + get_time: Optional[Callable[[], float]] = None, + refresh_per_second: float = 10, + style: StyleType = "bar.back", + complete_style: StyleType = "bar.complete", + finished_style: StyleType = "bar.finished", + pulse_style: StyleType = "bar.pulse", + update_period: float = 0.1, + disable: bool = False, +) -> Union[ContextManager[BinaryIO], ContextManager[TextIO]]: """Read bytes from a file while tracking progress. Args: @@ -424,7 +484,7 @@ def open( total=total, description=description, ) - return _ReadContext(progress, reader) + return _ReadContext(progress, reader) # type: ignore class ProgressColumn(ABC): @@ -1096,7 +1156,7 @@ def wrap_file( # attempt to recover the total from the task if total is None and task_id is not None: with self._lock: - total = self._tasks[task_id].total + total = self._tasks[task_id].total # type: ignore if total is None: raise ValueError( f"unable to get the total number of bytes, please specify 'total'" @@ -1111,10 +1171,11 @@ def wrap_file( # return a reader return _Reader(file, self, task_id, close_handle=False) + @typing.overload def open( self, file: Union[str, "PathLike[str]", bytes], - mode: str = "r", + mode: Literal["rb"], buffering: int = -1, encoding: Optional[str] = None, errors: Optional[str] = None, @@ -1123,6 +1184,35 @@ def open( task_id: Optional[TaskID] = None, description: str = "Reading...", ) -> BinaryIO: + pass + + @typing.overload + def open( + self, + file: Union[str, "PathLike[str]", bytes], + mode: Union[Literal["r"], Literal["rt"]], + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + total: Optional[int] = None, + task_id: Optional[TaskID] = None, + description: str = "Reading...", + ) -> TextIO: + pass + + def open( + self, + file: Union[str, "PathLike[str]", bytes], + mode: Union[Literal["rb"], Literal["rt"], Literal["r"]] = "r", + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + total: Optional[int] = None, + task_id: Optional[TaskID] = None, + description: str = "Reading...", + ) -> Union[BinaryIO, TextIO]: """Track progress while reading from a binary file. Args: @@ -1177,7 +1267,7 @@ def open( # wrap the reader in a `TextIOWrapper` if text mode if mode == "r" or mode == "rt": - reader = io.TextIOWrapper( + return io.TextIOWrapper( reader, encoding=encoding, errors=errors, From 0406578dc77f193ee70cfeafec929d4420410600 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 13 Mar 2022 19:48:58 +0100 Subject: [PATCH 17/25] Update `CHANGELOG.md` to show the `Progress.open` and `Progress.wrap_file` --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79b01a84c..239ea37b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Progress.read method to track the progress while reading from a file https://github.com/willmcgugan/rich/pull/1759 +- Progress.open and Progress.wrap_file method to track the progress while reading from a file or file-like object https://github.com/willmcgugan/rich/pull/1759 ## [12.0.0] - 2022-03-10 From 2e77302738c217fd9abf3284ca3f049536fdc367 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 21 Mar 2022 13:28:39 +0100 Subject: [PATCH 18/25] Remove unused `update_period` argument from `rich.progress` --- rich/progress.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index 15f308b13..816d199cd 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -299,7 +299,6 @@ def wrap_file( complete_style: StyleType = "bar.complete", finished_style: StyleType = "bar.finished", pulse_style: StyleType = "bar.pulse", - update_period: float = 0.1, disable: bool = False, ) -> ContextManager[BinaryIO]: """Read bytes from a file while tracking progress. @@ -316,7 +315,6 @@ def wrap_file( complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete". finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.done". pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse". - update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1. disable (bool, optional): Disable display of progress. Returns: ContextManager[BinaryIO]: A context manager yielding a progress reader. @@ -371,7 +369,6 @@ def open( complete_style: StyleType = "bar.complete", finished_style: StyleType = "bar.finished", pulse_style: StyleType = "bar.pulse", - update_period: float = 0.1, disable: bool = False, ) -> ContextManager[TextIO]: pass @@ -395,7 +392,6 @@ def open( complete_style: StyleType = "bar.complete", finished_style: StyleType = "bar.finished", pulse_style: StyleType = "bar.pulse", - update_period: float = 0.1, disable: bool = False, ) -> ContextManager[BinaryIO]: pass @@ -418,7 +414,6 @@ def open( complete_style: StyleType = "bar.complete", finished_style: StyleType = "bar.finished", pulse_style: StyleType = "bar.pulse", - update_period: float = 0.1, disable: bool = False, ) -> Union[ContextManager[BinaryIO], ContextManager[TextIO]]: """Read bytes from a file while tracking progress. @@ -440,7 +435,6 @@ def open( complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete". finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.done". pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse". - update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1. disable (bool, optional): Disable display of progress. encoding (str, optional): The encoding to use when reading in text mode. @@ -1154,21 +1148,23 @@ def wrap_file( ValueError: When no total value can be extracted from the arguments or the task. """ # attempt to recover the total from the task - if total is None and task_id is not None: + total_bytes: Optional[float] = None + if total is not None: + total_bytes = total + elif task_id is not None: with self._lock: - total = self._tasks[task_id].total # type: ignore - if total is None: + total_bytes = self._tasks[task_id].total + if total_bytes is None: raise ValueError( f"unable to get the total number of bytes, please specify 'total'" ) # update total of task or create new task if task_id is None: - task_id = self.add_task(description, total=total) + task_id = self.add_task(description, total=total_bytes) else: - self.update(task_id, total=total) + self.update(task_id, total=total_bytes) - # return a reader return _Reader(file, self, task_id, close_handle=False) @typing.overload From aa5ca607eb07766c2c9de9ed9ebb2d899d98030f Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 21 Mar 2022 13:32:11 +0100 Subject: [PATCH 19/25] Fix failing tests in `tests.test_progress` --- tests/test_progress.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_progress.py b/tests/test_progress.py index b462f47e7..d3c6171c9 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -570,9 +570,8 @@ def test_open() -> None: f.write(b"Hello, World!") try: with rich.progress.open(filename) as f: - assert f.read() == b"Hello, World!" + assert f.read() == "Hello, World!" assert f.closed - assert f.handle.closed finally: os.remove(filename) From 130f5b4e1d06891a6fb0ef3e1ca6c49a886c329a Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 21 Mar 2022 14:21:13 +0100 Subject: [PATCH 20/25] Reformat code in `rich.progress` --- rich/progress.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rich/progress.py b/rich/progress.py index 816d199cd..0ffda83ee 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -373,6 +373,7 @@ def open( ) -> ContextManager[TextIO]: pass + @typing.overload def open( file: Union[str, "PathLike[str]", bytes], @@ -396,6 +397,7 @@ def open( ) -> ContextManager[BinaryIO]: pass + def open( file: Union[str, "PathLike[str]", bytes], mode: Union[Literal["rb"], Literal["rt"], Literal["r"]] = "r", @@ -478,7 +480,7 @@ def open( total=total, description=description, ) - return _ReadContext(progress, reader) # type: ignore + return _ReadContext(progress, reader) # type: ignore class ProgressColumn(ABC): From 5ea30326fe282e09c20e28e55257f932863f9fbe Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 30 Mar 2022 13:22:29 +0200 Subject: [PATCH 21/25] Force keyword arguments for `Progress.open` options not in builtin `open` --- rich/progress.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rich/progress.py b/rich/progress.py index 0ffda83ee..36789e935 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -358,6 +358,7 @@ def open( encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, + *, total: Optional[int] = None, description: str = "Reading...", auto_refresh: bool = True, @@ -382,6 +383,7 @@ def open( encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, + *, total: Optional[int] = None, description: str = "Reading...", auto_refresh: bool = True, @@ -405,6 +407,7 @@ def open( encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, + *, total: Optional[int] = None, description: str = "Reading...", auto_refresh: bool = True, @@ -1178,6 +1181,7 @@ def open( encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, + *, total: Optional[int] = None, task_id: Optional[TaskID] = None, description: str = "Reading...", @@ -1193,6 +1197,7 @@ def open( encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, + *, total: Optional[int] = None, task_id: Optional[TaskID] = None, description: str = "Reading...", @@ -1207,6 +1212,7 @@ def open( encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, + *, total: Optional[int] = None, task_id: Optional[TaskID] = None, description: str = "Reading...", From 54d660def173cc2082c89b19bab7dfa449e19e20 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 30 Mar 2022 13:26:37 +0200 Subject: [PATCH 22/25] Update example for `wrap_file` in `rich.progress` API documentation --- docs/source/progress.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/progress.rst b/docs/source/progress.rst index c0e565e6f..f905b255e 100644 --- a/docs/source/progress.rst +++ b/docs/source/progress.rst @@ -218,7 +218,7 @@ You can obtain a progress-tracking reader using the :meth:`~rich.progress.Progre from rich.progress import Progress with Progress() as progress: - with progress.open("file.json", "rb") as file: + with progress.open("data.json", "rb") as file: json.load(file) @@ -237,13 +237,12 @@ Reading from a file-like object You can obtain a progress-tracking reader wrapping a file-like object using the :meth:`~rich.progress.Progress.wrap_file` method. The file-like object must be in *binary mode*, and a total must be provided, unless it was provided to a :class:`~rich.progress.Task` created beforehand. The returned reader may be used in a context, but will not take care of closing the wrapped file :: - import io import json from rich.progress import Progress with Progress() as progress: - file = io.BytesIO("...") - json.load(progress.read(file, total=2048)) + with open("data.json", "rb") as file: + json.load(progress.wrap_file(file, total=2048)) Multiple Progress From bb9790eb8f5a1cbb9f78fedeebb9ca33859e9c40 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 30 Mar 2022 14:22:28 +0200 Subject: [PATCH 23/25] Force keyword arguments for `Progress.wrap_file` arguments after `total` --- rich/progress.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rich/progress.py b/rich/progress.py index 36789e935..ecdb9accf 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -289,6 +289,7 @@ def __exit__( def wrap_file( file: BinaryIO, total: int, + *, description: str = "Reading...", auto_refresh: bool = True, console: Optional[Console] = None, @@ -1135,6 +1136,7 @@ def wrap_file( self, file: BinaryIO, total: Optional[int] = None, + *, task_id: Optional[TaskID] = None, description: str = "Reading...", ) -> BinaryIO: From 76953e2a595c19e5487b8c166a422b7de0400ddf Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 30 Mar 2022 14:39:16 +0200 Subject: [PATCH 24/25] Remove `_Reader.readall` method since wrapped object may not all have one --- rich/progress.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index ecdb9accf..f3eb60fee 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -228,11 +228,6 @@ def read(self, size: int = -1) -> bytes: self.progress.advance(self.task, advance=len(block)) return block - def readall(self) -> bytes: - block = self.handle.readall() # type: ignore - self.progress.advance(self.task, advance=len(block)) - return block # type: ignore - def readinto(self, b: Union[bytearray, memoryview, mmap]): # type: ignore n = self.handle.readinto(b) # type: ignore self.progress.advance(self.task, advance=n) From 5717d8d13ca73c9d9524eb038af4c6a6e2fc723d Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 30 Mar 2022 15:13:14 +0200 Subject: [PATCH 25/25] Add ignore codes for `mypy` in `rich.progress` --- rich/progress.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rich/progress.py b/rich/progress.py index f3eb60fee..a3b30a85b 100644 --- a/rich/progress.py +++ b/rich/progress.py @@ -228,12 +228,12 @@ def read(self, size: int = -1) -> bytes: self.progress.advance(self.task, advance=len(block)) return block - def readinto(self, b: Union[bytearray, memoryview, mmap]): # type: ignore - n = self.handle.readinto(b) # type: ignore + def readinto(self, b: Union[bytearray, memoryview, mmap]): # type: ignore[no-untyped-def, override] + n = self.handle.readinto(b) # type: ignore[attr-defined] self.progress.advance(self.task, advance=n) return n - def readline(self, size: int = -1) -> bytes: # type: ignore + def readline(self, size: int = -1) -> bytes: # type: ignore[override] line = self.handle.readline(size) self.progress.advance(self.task, advance=len(line)) return line @@ -479,7 +479,7 @@ def open( total=total, description=description, ) - return _ReadContext(progress, reader) # type: ignore + return _ReadContext(progress, reader) # type: ignore[return-value, type-var] class ProgressColumn(ABC):