From ce3d85a19ebe2a24d8da71858ea05ca80fdf4998 Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Sun, 20 Feb 2022 13:19:58 -0500 Subject: [PATCH 01/12] Throttle upload for testing --- twine/repository.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/twine/repository.py b/twine/repository.py index 087a2f31..53856281 100644 --- a/twine/repository.py +++ b/twine/repository.py @@ -37,16 +37,6 @@ logger = logging.getLogger(__name__) -class ProgressBar(tqdm.tqdm): - def update_to(self, n: int) -> None: - """Update the bar in the way compatible with requests-toolbelt. - - This is identical to tqdm.update, except ``n`` will be the current - value - not the delta as tqdm expects. - """ - self.update(n - self.n) # will also do self.n = n - - class Repository: def __init__( self, @@ -159,7 +149,8 @@ def _upload(self, package: package_file.PackageFile) -> requests.Response: ("content", (package.basefilename, fp, "application/octet-stream")) ) encoder = requests_toolbelt.MultipartEncoder(data_to_send) - with ProgressBar( + + with tqdm.tqdm( total=encoder.len, unit="B", unit_scale=True, @@ -168,9 +159,12 @@ def _upload(self, package: package_file.PackageFile) -> requests.Response: file=sys.stdout, disable=self.disable_progress_bar, ) as bar: - monitor = requests_toolbelt.MultipartEncoderMonitor( - encoder, lambda monitor: bar.update_to(monitor.bytes_read) - ) + + def update_bar(monitor): # type: ignore + bar.update(monitor.bytes_read - bar.n) + import time; time.sleep(0.2) # fmt: skip + + monitor = requests_toolbelt.MultipartEncoderMonitor(encoder, update_bar) resp = self.session.post( self.url, From 52c745dfcc105bfe24c4caa4120da3c6df6a3e8d Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Sun, 20 Feb 2022 13:24:37 -0500 Subject: [PATCH 02/12] Add initial rich progress bar --- twine/repository.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/twine/repository.py b/twine/repository.py index 53856281..c2371b0f 100644 --- a/twine/repository.py +++ b/twine/repository.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import sys from typing import Any, Dict, List, Optional, Set, Tuple, cast import requests import requests_toolbelt -import tqdm +import rich.progress import urllib3 from requests import adapters from requests_toolbelt.utils import user_agent @@ -149,22 +148,29 @@ def _upload(self, package: package_file.PackageFile) -> requests.Response: ("content", (package.basefilename, fp, "application/octet-stream")) ) encoder = requests_toolbelt.MultipartEncoder(data_to_send) - - with tqdm.tqdm( - total=encoder.len, - unit="B", - unit_scale=True, - unit_divisor=1024, - miniters=1, - file=sys.stdout, + # with tqdm.tqdm( + # total=encoder.len, + # unit="B", + # unit_scale=True, + # unit_divisor=1024, + # miniters=1, + # file=sys.stdout, + # disable=self.disable_progress_bar, + # ) as bar: + with rich.progress.Progress( + # TODO: % BAR uploaded/total elapsed/remaining speed disable=self.disable_progress_bar, - ) as bar: + ) as progress: + task_id = progress.add_task("Progress:", total=encoder.len) - def update_bar(monitor): # type: ignore - bar.update(monitor.bytes_read - bar.n) + def update_progress(monitor): # type: ignore + progress.update(task_id, completed=monitor.bytes_read) import time; time.sleep(0.2) # fmt: skip - monitor = requests_toolbelt.MultipartEncoderMonitor(encoder, update_bar) + monitor = requests_toolbelt.MultipartEncoderMonitor( + encoder, + update_progress, + ) resp = self.session.post( self.url, From dfdb9ccd6ebd5a285a6f20e5e188904c807c884a Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Sun, 20 Feb 2022 14:51:47 -0500 Subject: [PATCH 03/12] Define progress bar columns --- twine/repository.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/twine/repository.py b/twine/repository.py index c2371b0f..83e0466e 100644 --- a/twine/repository.py +++ b/twine/repository.py @@ -148,17 +148,16 @@ def _upload(self, package: package_file.PackageFile) -> requests.Response: ("content", (package.basefilename, fp, "application/octet-stream")) ) encoder = requests_toolbelt.MultipartEncoder(data_to_send) - # with tqdm.tqdm( - # total=encoder.len, - # unit="B", - # unit_scale=True, - # unit_divisor=1024, - # miniters=1, - # file=sys.stdout, - # disable=self.disable_progress_bar, - # ) as bar: + with rich.progress.Progress( - # TODO: % BAR uploaded/total elapsed/remaining speed + "[progress.percentage]{task.percentage:>3.0f}%", + rich.progress.BarColumn(), + rich.progress.DownloadColumn(), + "•", + rich.progress.TimeElapsedColumn(), + rich.progress.TimeRemainingColumn(), + "•", + rich.progress.TransferSpeedColumn(), disable=self.disable_progress_bar, ) as progress: task_id = progress.add_task("Progress:", total=encoder.len) From 78074dbb6eaa4919216c04062a55d39d4667f1c5 Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Sun, 20 Feb 2022 14:53:23 -0500 Subject: [PATCH 04/12] Remove throttling code --- twine/repository.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/twine/repository.py b/twine/repository.py index 83e0466e..4e80f2b6 100644 --- a/twine/repository.py +++ b/twine/repository.py @@ -160,15 +160,14 @@ def _upload(self, package: package_file.PackageFile) -> requests.Response: rich.progress.TransferSpeedColumn(), disable=self.disable_progress_bar, ) as progress: - task_id = progress.add_task("Progress:", total=encoder.len) - - def update_progress(monitor): # type: ignore - progress.update(task_id, completed=monitor.bytes_read) - import time; time.sleep(0.2) # fmt: skip + task_id = progress.add_task("", total=encoder.len) monitor = requests_toolbelt.MultipartEncoderMonitor( encoder, - update_progress, + lambda monitor: progress.update( + task_id, + completed=monitor.bytes_read, + ), ) resp = self.session.post( From ad1259aeeeedd6be3af03fc2a2ed33f949fec516 Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Sun, 20 Feb 2022 15:13:22 -0500 Subject: [PATCH 05/12] Remove tqdm --- mypy.ini | 6 +----- setup.cfg | 1 - tests/test_repository.py | 11 +++++++---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/mypy.ini b/mypy.ini index 5f2d5780..87242265 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,7 +5,7 @@ show_traceback = True warn_redundant_casts = True warn_unused_configs = True warn_unused_ignores = True -; Enabling this will fail on subclasses of untyped imports, e.g. tqdm +; Enabling this will fail on subclasses of untyped imports, e.g. pkginfo ; disallow_subclassing_any = True disallow_any_generics = True disallow_untyped_calls = True @@ -33,10 +33,6 @@ ignore_missing_imports = True [mypy-rfc3986] ignore_missing_imports = True -[mypy-tqdm] -; https://github.com/tqdm/tqdm/issues/260 -ignore_missing_imports = True - [mypy-urllib3] ; https://github.com/urllib3/urllib3/issues/867 ignore_missing_imports = True diff --git a/setup.cfg b/setup.cfg index 3073d0dc..6f9604e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,6 @@ install_requires= requests >= 2.20 requests-toolbelt >= 0.8.0, != 0.9.0 urllib3 >= 1.26.0 - tqdm >= 4.14 importlib_metadata >= 3.6 keyring >= 15.1 rfc3986 >= 1.4.0 diff --git a/tests/test_repository.py b/tests/test_repository.py index 0c0098c7..5526d823 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -195,18 +195,21 @@ def test_package_is_registered(default_repo): @pytest.mark.parametrize("disable_progress_bar", [True, False]) -def test_disable_progress_bar_is_forwarded_to_tqdm( +def test_disable_progress_bar_is_forwarded_to_rich( monkeypatch, tmpdir, disable_progress_bar, default_repo ): """Toggle display of upload progress bar.""" @contextmanager - def progressbarstub(*args, **kwargs): + def ProgressStub(*args, **kwargs): assert "disable" in kwargs assert kwargs["disable"] == disable_progress_bar - yield + yield pretend.stub( + add_task=lambda description, total: None, + update=lambda task_id, completed: None, + ) - monkeypatch.setattr(repository, "ProgressBar", progressbarstub) + monkeypatch.setattr(repository.rich.progress, "Progress", ProgressStub) default_repo.disable_progress_bar = disable_progress_bar default_repo.session = pretend.stub( From 530ec1294fcb394de3d96f236f70196996679fe0 Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Sun, 20 Feb 2022 15:32:52 -0500 Subject: [PATCH 06/12] Rework retries test --- tests/test_repository.py | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/tests/test_repository.py b/tests/test_repository.py index 5526d823..e44dc966 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -254,35 +254,25 @@ def test_upload_retry(tmpdir, default_repo, capsys): metadata_dictionary=lambda: {"name": "fake"}, ) + def assert_retries(output, total): + retries = [line for line in output.splitlines() if line.startswith("Received")] + assert retries == [ + ( + 'Received "500: Internal server error" ' + f"Package upload appears to have failed. Retry {i} of {total}" + ) + for i in range(1, total + 1) + ] + # Upload with default max_redirects of 5 default_repo.upload(package) - msg = [ - ( - "Uploading fake.whl\n" - 'Received "500: Internal server error" ' - f"Package upload appears to have failed. Retry {i} of 5" - ) - for i in range(1, 6) - ] - - captured = capsys.readouterr() - assert captured.out == "\n".join(msg) + "\n" + assert_retries(capsys.readouterr().out, 5) # Upload with custom max_redirects of 3 default_repo.upload(package, 3) - msg = [ - ( - "Uploading fake.whl\n" - 'Received "500: Internal server error" ' - f"Package upload appears to have failed. Retry {i} of 3" - ) - for i in range(1, 4) - ] - - captured = capsys.readouterr() - assert captured.out == "\n".join(msg) + "\n" + assert_retries(capsys.readouterr().out, 3) @pytest.mark.parametrize( From 7fed68d86da0169492340e1133f050ccffa31a0d Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Tue, 22 Feb 2022 06:05:00 -0500 Subject: [PATCH 07/12] Add custom time column --- tests/test_repository.py | 21 +++++++++++++++++++++ twine/repository.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/tests/test_repository.py b/tests/test_repository.py index e44dc966..9b47c4be 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -233,6 +233,27 @@ def dictfunc(): default_repo.upload(package) +@pytest.mark.parametrize("finished", [False, True]) +@pytest.mark.parametrize( + "task_time, formatted", + [ + (None, "--:--"), + (0, "00:00"), + (59, "00:59"), + (71, "01:11"), + (4210, "1:10:10"), + ], +) +def test_time_column_renders_condensed_time(finished, task_time, formatted): + if finished: + task = pretend.stub(finished=finished, finished_time=task_time) + else: + task = pretend.stub(finished=finished, time_remaining=task_time) + + column = repository.CondensedTimeColumn() + assert str(column.render(task)) == formatted + + def test_upload_retry(tmpdir, default_repo, capsys): """Print retry messages when the upload response indicates a server error.""" default_repo.disable_progress_bar = True diff --git a/twine/repository.py b/twine/repository.py index 4e80f2b6..e8b9e387 100644 --- a/twine/repository.py +++ b/twine/repository.py @@ -17,6 +17,7 @@ import requests import requests_toolbelt import rich.progress +import rich.text import urllib3 from requests import adapters from requests_toolbelt.utils import user_agent @@ -36,6 +37,30 @@ logger = logging.getLogger(__name__) +class CondensedTimeColumn(rich.progress.ProgressColumn): + """Renders estimated time remaining, or elapsed time when the task is finished.""" + + # Only refresh twice a second to prevent jitter + max_refresh = 0.5 + + def render(self, task: rich.progress.Task) -> rich.text.Text: + """Show time.""" + style = "progress.elapsed" if task.finished else "progress.remaining" + task_time = task.finished_time if task.finished else task.time_remaining + if task_time is None: + return rich.text.Text("--:--", style=style) + + # Based on https://github.com/tqdm/tqdm/blob/master/tqdm/std.py + minutes, seconds = divmod(int(task_time), 60) + hours, minutes = divmod(minutes, 60) + if hours: + formatted = f"{hours:d}:{minutes:02d}:{seconds:02d}" + else: + formatted = f"{minutes:02d}:{seconds:02d}" + + return rich.text.Text(formatted, style=style) + + class Repository: def __init__( self, @@ -154,8 +179,7 @@ def _upload(self, package: package_file.PackageFile) -> requests.Response: rich.progress.BarColumn(), rich.progress.DownloadColumn(), "•", - rich.progress.TimeElapsedColumn(), - rich.progress.TimeRemainingColumn(), + CondensedTimeColumn(), "•", rich.progress.TransferSpeedColumn(), disable=self.disable_progress_bar, From baa16c1a034fb3e06863ccb5341ff69090eafbb1 Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Tue, 22 Feb 2022 07:07:37 -0500 Subject: [PATCH 08/12] Add rich to intersphinx --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 1f00d233..c9ba1910 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -279,6 +279,7 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "requests": ("https://docs.python-requests.org/en/latest/", None), + "rich": ("https://rich.readthedocs.io/en/stable/", None), } # Be strict about the invalid references: From 4f8c1d42e02a9fc2c085bc5809a52bf60c273156 Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Thu, 10 Mar 2022 19:44:53 -0500 Subject: [PATCH 09/12] Revert "Add rich to intersphinx" This reverts commit baa16c1a034fb3e06863ccb5341ff69090eafbb1. --- docs/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index c9ba1910..1f00d233 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -279,7 +279,6 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "requests": ("https://docs.python-requests.org/en/latest/", None), - "rich": ("https://rich.readthedocs.io/en/stable/", None), } # Be strict about the invalid references: From eaa7a63cdd4547bc60aa2493476101f61ea4469e Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Thu, 10 Mar 2022 19:45:04 -0500 Subject: [PATCH 10/12] Revert "Add custom time column" This reverts commit 7fed68d86da0169492340e1133f050ccffa31a0d. --- tests/test_repository.py | 21 --------------------- twine/repository.py | 28 ++-------------------------- 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/tests/test_repository.py b/tests/test_repository.py index 9b47c4be..e44dc966 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -233,27 +233,6 @@ def dictfunc(): default_repo.upload(package) -@pytest.mark.parametrize("finished", [False, True]) -@pytest.mark.parametrize( - "task_time, formatted", - [ - (None, "--:--"), - (0, "00:00"), - (59, "00:59"), - (71, "01:11"), - (4210, "1:10:10"), - ], -) -def test_time_column_renders_condensed_time(finished, task_time, formatted): - if finished: - task = pretend.stub(finished=finished, finished_time=task_time) - else: - task = pretend.stub(finished=finished, time_remaining=task_time) - - column = repository.CondensedTimeColumn() - assert str(column.render(task)) == formatted - - def test_upload_retry(tmpdir, default_repo, capsys): """Print retry messages when the upload response indicates a server error.""" default_repo.disable_progress_bar = True diff --git a/twine/repository.py b/twine/repository.py index e8b9e387..4e80f2b6 100644 --- a/twine/repository.py +++ b/twine/repository.py @@ -17,7 +17,6 @@ import requests import requests_toolbelt import rich.progress -import rich.text import urllib3 from requests import adapters from requests_toolbelt.utils import user_agent @@ -37,30 +36,6 @@ logger = logging.getLogger(__name__) -class CondensedTimeColumn(rich.progress.ProgressColumn): - """Renders estimated time remaining, or elapsed time when the task is finished.""" - - # Only refresh twice a second to prevent jitter - max_refresh = 0.5 - - def render(self, task: rich.progress.Task) -> rich.text.Text: - """Show time.""" - style = "progress.elapsed" if task.finished else "progress.remaining" - task_time = task.finished_time if task.finished else task.time_remaining - if task_time is None: - return rich.text.Text("--:--", style=style) - - # Based on https://github.com/tqdm/tqdm/blob/master/tqdm/std.py - minutes, seconds = divmod(int(task_time), 60) - hours, minutes = divmod(minutes, 60) - if hours: - formatted = f"{hours:d}:{minutes:02d}:{seconds:02d}" - else: - formatted = f"{minutes:02d}:{seconds:02d}" - - return rich.text.Text(formatted, style=style) - - class Repository: def __init__( self, @@ -179,7 +154,8 @@ def _upload(self, package: package_file.PackageFile) -> requests.Response: rich.progress.BarColumn(), rich.progress.DownloadColumn(), "•", - CondensedTimeColumn(), + rich.progress.TimeElapsedColumn(), + rich.progress.TimeRemainingColumn(), "•", rich.progress.TransferSpeedColumn(), disable=self.disable_progress_bar, From 6c0d11b45da5532eb9d9ec776e0bfa218b735fe3 Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Thu, 10 Mar 2022 19:54:15 -0500 Subject: [PATCH 11/12] Use single compact time column --- setup.cfg | 2 +- twine/repository.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6f9604e5..8a23b48c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ install_requires= importlib_metadata >= 3.6 keyring >= 15.1 rfc3986 >= 1.4.0 - rich + rich >= 12.0.0 include_package_data = True [options.entry_points] diff --git a/twine/repository.py b/twine/repository.py index 4e80f2b6..b7a943f7 100644 --- a/twine/repository.py +++ b/twine/repository.py @@ -154,8 +154,10 @@ def _upload(self, package: package_file.PackageFile) -> requests.Response: rich.progress.BarColumn(), rich.progress.DownloadColumn(), "•", - rich.progress.TimeElapsedColumn(), - rich.progress.TimeRemainingColumn(), + rich.progress.TimeRemainingColumn( + compact=True, + elapsed_when_finished=True, + ), "•", rich.progress.TransferSpeedColumn(), disable=self.disable_progress_bar, From 464f52579d692682a53672c02e131dab3e8a5ef5 Mon Sep 17 00:00:00 2001 From: Brian Rutledge Date: Thu, 10 Mar 2022 20:01:16 -0500 Subject: [PATCH 12/12] Add changelog entry --- changelog/877.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/877.feature.rst diff --git a/changelog/877.feature.rst b/changelog/877.feature.rst new file mode 100644 index 00000000..c2fb4462 --- /dev/null +++ b/changelog/877.feature.rst @@ -0,0 +1 @@ +Use Rich instead of tqdm for upload progress bar.