Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Don't issue redundant stdin detection warning when is in place. #1303

Merged
merged 4 commits into from Mar 7, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Expand Up @@ -7,6 +7,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).

- Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285))
- Fixed displaying of status code without a status message on non-`auto` themes. ([#1300](https://github.com/httpie/httpie/issues/1300))
- Fixed redundant issuance of stdin detection warnings on some rare cases due to underlying implementation. ([#1303](https://github.com/httpie/httpie/pull/1303))
- Improved regulation of top-level arrays. ([#1292](https://github.com/httpie/httpie/commit/225dccb2186f14f871695b6c4e0bfbcdb2e3aa28))
- Double `--quiet` flags will now suppress all python level warnings. ([#1271](https://github.com/httpie/httpie/issues/1271))

Expand Down
61 changes: 48 additions & 13 deletions httpie/uploads.py
Expand Up @@ -2,6 +2,8 @@
import os
import zlib
import functools
import time
import threading
from typing import Any, Callable, IO, Iterable, Optional, Tuple, Union, TYPE_CHECKING
from urllib.parse import urlencode

Expand All @@ -22,25 +24,40 @@ def __iter__(self) -> Iterable[Union[str, bytes]]:


class ChunkedUploadStream(ChunkedStream):
def __init__(self, stream: Iterable, callback: Callable):
def __init__(
self,
stream: Iterable,
callback: Callable,
event: Optional[threading.Event] = None
) -> None:
self.callback = callback
self.stream = stream
self.event = event

def __iter__(self) -> Iterable[Union[str, bytes]]:
for chunk in self.stream:
if self.event:
self.event.set()
self.callback(chunk)
yield chunk


class ChunkedMultipartUploadStream(ChunkedStream):
chunk_size = 100 * 1024

def __init__(self, encoder: 'MultipartEncoder'):
def __init__(
self,
encoder: 'MultipartEncoder',
event: Optional[threading.Event] = None
) -> None:
self.encoder = encoder
self.event = event

def __iter__(self) -> Iterable[Union[str, bytes]]:
while True:
chunk = self.encoder.read(self.chunk_size)
if self.event:
self.event.set()
if not chunk:
break
yield chunk
Expand Down Expand Up @@ -77,10 +94,10 @@ def is_stdin(file: IO) -> bool:
return file_no == sys.stdin.fileno()


READ_THRESHOLD = float(os.getenv("HTTPIE_STDIN_READ_WARN_THRESHOLD", 10.0))
READ_THRESHOLD = float(os.getenv('HTTPIE_STDIN_READ_WARN_THRESHOLD', 10.0))


def observe_stdin_for_data_thread(env: Environment, file: IO) -> None:
def observe_stdin_for_data_thread(env: Environment, file: IO, read_event: threading.Event) -> None:
# Windows unfortunately does not support select() operation
# on regular files, like stdin in our use case.
# https://docs.python.org/3/library/select.html#select.select
Expand All @@ -92,34 +109,50 @@ def observe_stdin_for_data_thread(env: Environment, file: IO) -> None:
if READ_THRESHOLD == 0:
return None

import select
import threading

def worker():
can_read, _, _ = select.select([file], [], [], READ_THRESHOLD)
if not can_read:
def worker(event: threading.Event) -> None:
time.sleep(READ_THRESHOLD)
if not event.is_set():
env.stderr.write(
f'> warning: no stdin data read in {READ_THRESHOLD}s '
f'(perhaps you want to --ignore-stdin)\n'
f'> See: https://httpie.io/docs/cli/best-practices\n'
)

thread = threading.Thread(
target=worker
target=worker,
args=(read_event,)
)
thread.start()


def _read_file_with_selectors(file: IO, read_event: threading.Event) -> bytes:
if is_windows or not is_stdin(file):
return as_bytes(file.read())

import select

# Try checking whether there is any incoming data for READ_THRESHOLD
# seconds. If there isn't anything in the given period, issue
# a warning about a misusage.
read_selectors, _, _ = select.select([file], [], [], READ_THRESHOLD)
if read_selectors:
read_event.set()

return as_bytes(file.read())


def _prepare_file_for_upload(
env: Environment,
file: Union[IO, 'MultipartEncoder'],
callback: CallbackT,
chunked: bool = False,
content_length_header_value: Optional[int] = None,
) -> Union[bytes, IO, ChunkedStream]:
read_event = threading.Event()
if not super_len(file):
if is_stdin(file):
observe_stdin_for_data_thread(env, file)
observe_stdin_for_data_thread(env, file, read_event)

# Zero-length -> assume stdin.
if content_length_header_value is None and not chunked:
# Read the whole stdin to determine `Content-Length`.
Expand All @@ -129,7 +162,7 @@ def _prepare_file_for_upload(
# something like --no-chunked.
# This would be backwards-incompatible so wait until v3.0.0.
#
file = as_bytes(file.read())
file = _read_file_with_selectors(file, read_event)
else:
file.read = _wrap_function_with_callback(
file.read,
Expand All @@ -141,11 +174,13 @@ def _prepare_file_for_upload(
if isinstance(file, MultipartEncoder):
return ChunkedMultipartUploadStream(
encoder=file,
event=read_event,
)
else:
return ChunkedUploadStream(
stream=file,
callback=callback,
event=read_event
)
else:
return file
Expand Down