diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 560835dab2..08d24700ea 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -13,8 +13,8 @@ jobs: if: github.event.label.name == 'benchmark' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: python-version: "3.9" diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml index 3d220d72f0..b53b353b04 100644 --- a/.github/workflows/code-style.yml +++ b/.github/workflows/code-style.yml @@ -13,8 +13,8 @@ jobs: code-style: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: python-version: 3.9 - run: make venv diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 5edb47fa46..acd9aeb9da 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,8 +12,8 @@ jobs: coverage: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: python-version: "3.10" - run: make install diff --git a/.github/workflows/docs-check-markdown.yml b/.github/workflows/docs-check-markdown.yml index f019bf7017..a19c25916a 100644 --- a/.github/workflows/docs-check-markdown.yml +++ b/.github/workflows/docs-check-markdown.yml @@ -10,7 +10,7 @@ jobs: doc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Ruby uses: ruby/setup-ruby@v1 with: diff --git a/.github/workflows/docs-update-install.yml b/.github/workflows/docs-update-install.yml index 3a22aaa0bb..2d3fbe09d9 100644 --- a/.github/workflows/docs-update-install.yml +++ b/.github/workflows/docs-update-install.yml @@ -15,8 +15,8 @@ jobs: doc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: python-version: 3.9 - run: make install diff --git a/.github/workflows/release-snap.yml b/.github/workflows/release-snap.yml index f67c0e2577..149adc50e7 100644 --- a/.github/workflows/release-snap.yml +++ b/.github/workflows/release-snap.yml @@ -16,7 +16,7 @@ jobs: snap: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.branch }} - uses: snapcore/action-build@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 12753b49f5..30561369d1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,13 +18,13 @@ jobs: new-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.branch }} - name: PyPI configuration run: | echo "[distutils]\nindex-servers=\n httpie\n\n[httpie]\nrepository = https://upload.pypi.org/legacy/\n" > $HOME/.pypirc - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: 3.9 - run: make publish diff --git a/.github/workflows/test-package-linux-snap.yml b/.github/workflows/test-package-linux-snap.yml index 12a2766a40..ac9640a06d 100644 --- a/.github/workflows/test-package-linux-snap.yml +++ b/.github/workflows/test-package-linux-snap.yml @@ -12,7 +12,7 @@ jobs: snap: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build uses: snapcore/action-build@v1 id: snapcraft diff --git a/.github/workflows/test-package-mac-brew.yml b/.github/workflows/test-package-mac-brew.yml index 6570a2afd9..babdaa5def 100644 --- a/.github/workflows/test-package-mac-brew.yml +++ b/.github/workflows/test-package-mac-brew.yml @@ -11,7 +11,7 @@ jobs: brew: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup brew run: | brew developer on diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3cde99669..17d03d3eef 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,8 +26,8 @@ jobs: pyopenssl: [0, 1] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Windows setup diff --git a/.packit.yaml b/.packit.yaml index 8fc33379d6..0ae2100cbc 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -3,16 +3,15 @@ specfile_path: httpie.spec actions: # get the current Fedora Rawhide specfile: - # post-upstream-clone: "wget https://src.fedoraproject.org/rpms/httpie/raw/rawhide/f/httpie.spec -O httpie.spec" - post-upstream-clone: "cp docs/packaging/linux-fedora/httpie.spec.txt httpie.spec" + post-upstream-clone: "wget https://src.fedoraproject.org/rpms/httpie/raw/rawhide/f/httpie.spec -O httpie.spec" + # Use this when the latest spec is not up-to-date. + # post-upstream-clone: "cp docs/packaging/linux-fedora/httpie.spec.txt httpie.spec" jobs: - job: copr_build trigger: pull_request metadata: targets: - fedora-all - additional_repos: - - "https://kojipkgs.fedoraproject.org/repos/f$releasever-build/latest/$basearch/" - job: propose_downstream trigger: release metadata: diff --git a/CHANGELOG.md b/CHANGELOG.md index 045d406cfe..7ecf797d29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,16 @@ This document records all notable changes to [HTTPie](https://httpie.io). This project adheres to [Semantic Versioning](https://semver.org/). +## [3.0.3.dev0](https://github.com/httpie/httpie/compare/3.0.2...HEAD) (Unreleased) + +- Added support for specifying certificate private key passphrases through `--cert-key-pass` and prompts. ([#946](https://github.com/httpie/httpie/issues/946)) +- Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285)) +- Fixed displaying of status code without a status message on non-`auto` themes. ([#1300](https://github.com/httpie/httpie/issues/1300)) +- Fixed redundant issuance of stdin detection warnings on some rare cases due to underlying implementation. ([#1303](https://github.com/httpie/httpie/pull/1303)) +- Improved regulation of top-level arrays. ([#1292](https://github.com/httpie/httpie/commit/225dccb2186f14f871695b6c4e0bfbcdb2e3aa28)) +- Improved UI layout for standalone invocations. ([#1296](https://github.com/httpie/httpie/pull/1296)) +- Double `--quiet` flags will now suppress all python level warnings. ([#1271](https://github.com/httpie/httpie/issues/1271)) + ## [3.0.2](https://github.com/httpie/httpie/compare/3.0.1...3.0.2) (2022-01-24) [What’s new in HTTPie for Terminal 3.0 →](https://httpie.io/blog/httpie-3.0.0) diff --git a/LICENSE b/LICENSE index a6270bbf89..4a53e9d679 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright © 2012-2021 Jakub Roztocil +Copyright © 2012-2022 Jakub Roztocil Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..542bcd7854 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ +# Security policy + +## Reporting a vulnerability + +When you identify a vulnerability in HTTPie, please report it privately using one of the following channels: + +- Email to [`security@httpie.io`](mailto:security@httpie.io) +- Report on [huntr.dev](https://huntr.dev/) + +In addition to the description of the vulnerability, include the following information: + +- A short reproducer to verify it (it can be a small HTTP server, shell script, docker image, etc.) +- Your deemed severity level of the vulnerability (`LOW`/`MEDIUM`/`HIGH`/`CRITICAL`) +- [CWE](https://cwe.mitre.org/) ID, if available. diff --git a/docs/README.md b/docs/README.md index b82661db1b..1c171bd80d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -205,12 +205,12 @@ Also works for other Arch-derived distributions like ArcoLinux, EndeavourOS, Art ```bash # Install httpie -$ pacman -Sy httpie +$ pacman -Syu httpie ``` ```bash # Upgrade httpie -$ pacman -Syu httpie +$ pacman -Syu ``` ### FreeBSD @@ -260,7 +260,7 @@ Verify that now you have the [current development version identifier](https://gi ```bash $ http --version -# 3.0.0 +# 3.0.3.dev0 ``` ## Usage @@ -854,6 +854,47 @@ $ http PUT pie.dev/put \ #### Advanced usage +##### Top level arrays + +If you want to send an array instead of a regular object, you can simply +do that by omitting the starting key: + +```bash +$ http --offline --print=B pie.dev/post \ + []:=1 \ + []:=2 \ + []:=3 +``` + +```json +[ + 1, + 2, + 3 +] +``` + +You can also apply the nesting to the items by referencing their index: + +```bash +http --offline --print=B pie.dev/post \ + [0][type]=platform [0][name]=terminal \ + [1][type]=platform [1][name]=desktop +``` + +```json +[ + { + "type": "platform", + "name": "terminal" + }, + { + "type": "platform", + "name": "desktop" + } +] +``` + ##### Escaping behavior Nested JSON syntax uses the same [escaping rules](#escaping-rules) as @@ -1336,7 +1377,7 @@ Here are a few picks: - [httpie-jwt-auth](https://github.com/teracyhq/httpie-jwt-auth): JWTAuth (JSON Web Tokens) - [httpie-negotiate](https://github.com/ndzou/httpie-negotiate): SPNEGO (GSS Negotiate) - [httpie-ntlm](https://github.com/httpie/httpie-ntlm): NTLM (NT LAN Manager) -- [httpie-oauth](https://github.com/httpie/httpie-oauth): OAuth +- [httpie-oauth1](https://github.com/qcif/httpie-oauth1): OAuth 1.0a - [requests-hawk](https://github.com/mozilla-services/requests-hawk): Hawk See [plugin manager](#plugin-manager) for more details. @@ -1448,6 +1489,21 @@ path of the key file with `--cert-key`: $ http --cert=client.crt --cert-key=client.key https://example.org ``` +If the given private key requires a passphrase, HTTPie will automatically detect it +and ask it through a prompt: + +```bash +$ http --cert=client.pem --cert-key=client.key https://example.org +http: passphrase for client.key: **** +``` + +If you don't want to see a prompt, you can supply the passphrase with the `--cert-key-pass` +argument: + +```bash +$ http --cert=client.pem --cert-key=client.key --cert-key-pass=my_password https://example.org +``` + ### SSL version Use the `--ssl=` option to specify the desired protocol version to use. @@ -2101,38 +2157,88 @@ $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:orig- $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:new-value ``` -### Cookie Storage Behavior +### Host-based cookie policy -**TL;DR:** Cookie storage priority: Server response > Command line request > Session file +Cookies persisted in sessions files have a `domain` field. This _binds_ them to a specified hostname. For example: -To set a cookie within a Session there are three options: +```json +{ + "cookies": [ + { + "domain": "pie.dev", + "name": "pie", + "value": "apple" + }, + { + "domain": "httpbin.org", + "name": "bin", + "value": "http" + } + ] +} +``` -1. Get a `Set-Cookie` header in a response from a server +Using this session file, we include `Cookie: pie=apple` only in requests against `pie.dev` and subdomains (e.g., `foo.pie.dev` or `foo.bar.pie.dev`): - ```bash - $ http --session=./session.json pie.dev/cookie/set?foo=bar - ``` +```bash +$ http --session=./session.json pie.dev/cookies +``` -2. Set the cookie name and value through the command line as seen in [cookies](#cookies) +```json +{ + "cookies": { + "pie": "apple" + } +} +``` - ```bash - $ http --session=./session.json pie.dev/headers Cookie:foo=bar - ``` +To make a cookie domain _unbound_ (i.e., to make it available to all hosts, including throughout a cross-domain redirect chain), you can set the `domain` field to `null` in the session file: -3. Manually set cookie parameters in the JSON file of the session +```json +{ + "cookies": [ + { + "domain": null, + "name": "unbound-cookie", + "value": "send-me-to-any-host" + } + ] +} +``` + +```bash +$ http --session=./session.json pie.dev/cookies +``` + +```json +{ + "cookies": { + "unbound-cookie": "send-me-to-any-host" + } +} +``` - ```json - { - "__meta__": { - "about": "HTTPie session file", - "help": "https://httpie.org/doc#sessions", - "httpie": "2.2.0-dev" - }, - "auth": { - "password": null, - "type": null, - "username": null - }, + +### Cookie storage behavior + +There are three possible sources of persisted cookies within a session. They have the following storage priority: 1—response; 2—command line; 3—session file. + +1. Receive a response with a `Set-Cookie` header: + + ```bash + $ http --session=./session.json pie.dev/cookie/set?foo=bar + ``` + +2. Send a cookie specified on the command line as seen in [cookies](#cookies): + + ```bash + $ http --session=./session.json pie.dev/headers Cookie:foo=bar + ``` + +3. Manually set cookie parameters in the session file: + + ```json + { "cookies": { "foo": { "expires": null, @@ -2141,16 +2247,53 @@ To set a cookie within a Session there are three options: "value": "bar" } } - } - ``` + } + ``` + +In summary: + +- Cookies set via the CLI overwrite cookies of the same name inside session files. +- Server-sent `Set-Cookie` header cookies overwrite any pre-existing ones with the same name. -Cookies will be set in the session file with the priority specified above. -For example, a cookie set through the command line will overwrite a cookie of the same name stored in the session file. -If the server returns a `Set-Cookie` header with a cookie of the same name, the returned cookie will overwrite the preexisting cookie. +Cookie expiration handling: -Expired cookies are never stored. -If a cookie in a session file expires, it will be removed before sending a new request. -If the server expires an existing cookie, it will also be removed from the session file. +- When the server expires an existing cookie, HTTPie removes it from the session file. +- When a cookie in a session file expires, HTTPie removes it before sending a new request. + +### Upgrading sessions + +HTTPie may introduce changes in the session file format. When HTTPie detects an obsolete format, it shows a warning. You can upgrade your session files using the following commands: + +Upgrade all existing [named sessions](#named-sessions) inside the `sessions` subfolder of your [config directory](https://httpie.io/docs/cli/config-file-directory): + +```bash +$ httpie cli sessions upgrade-all +Upgraded 'api_auth' @ 'pie.dev' to v3.1.0 +Upgraded 'login_cookies' @ 'httpie.io' to v3.1.0 +``` + +Upgrading individual sessions requires you to specify the session's hostname. That allows HTTPie to find the correct file in the case of name sessions. Additionally, it allows it to correctly bind cookies when upgrading with [`--bind-cookies`](#session-upgrade-options). + +Upgrade a single [named session](#named-sessions): + +```bash +$ httpie cli sessions upgrade pie.dev api_auth +Upgraded 'api_auth' @ 'pie.dev' to v3.1.0 +``` + +Upgrade a single [anonymous session](#anonymous-sessions) using a file path: + +```bash +$ httpie cli sessions upgrade pie.dev ./session.json +Upgraded 'session.json' @ 'pie.dev' to v3.1.0 +``` + +#### Session upgrade options + +These flags are available for both `sessions upgrade` and `sessions upgrade-all`: + +------------------|------------------------------------------ +`--bind-cookies` | Bind all previously [unbound cookies](#host-based-cookie-policy) to the session’s host. ## Config @@ -2163,7 +2306,7 @@ To see the exact location for your installation, run `http --debug` and look for The default location of the configuration file on most platforms is `$XDG_CONFIG_HOME/httpie/config.json` (defaulting to `~/.config/httpie/config.json`). -For backwards compatibility, if the directory `~/.httpie` exists, the configuration file there will be used instead. +For backward compatibility, if the directory `~/.httpie` exists, the configuration file there will be used instead. On Windows, the config file is located at `%APPDATA%\httpie\config.json`. @@ -2260,9 +2403,10 @@ For managing these plugins; starting with 3.0, we are offering a new plugin mana This command is currently in beta. ### `httpie cli` -#### `httpie cli export` -`httpie cli export` command was designed to expose parser spec of the `http/`https` commands +#### `httpie cli export-args` + +`httpie cli export-args` command can expose the parser specification of `http`/`https` commands (like an API definition) to outside tools so that they can use this to build better interactions over them (e.g offer auto-complete). @@ -2275,7 +2419,7 @@ Available formats to export in include: You can use any of these formats with `--format` parameter, but the default one is `json`. ```bash -$ httpie cli export | jq '"Program: " + .spec.name + ", Version: " + .version' +$ httpie cli export-args | jq '"Program: " + .spec.name + ", Version: " + .version' "Program: http, Version: 0.0.1a0" ``` @@ -2409,6 +2553,10 @@ Helpers to convert from other client tools: See [CONTRIBUTING](https://github.com/httpie/httpie/blob/master/CONTRIBUTING.md). +### Security policy + +See [github.com/httpie/httpie/security/policy](https://github.com/httpie/httpie/security/policy). + ### Change log See [CHANGELOG](https://github.com/httpie/httpie/blob/master/CHANGELOG.md). diff --git a/docs/installation/methods.yml b/docs/installation/methods.yml index 1583b2364a..0828b0d603 100644 --- a/docs/installation/methods.yml +++ b/docs/installation/methods.yml @@ -106,9 +106,9 @@ tools: package: https://archlinux.org/packages/community/any/httpie/ commands: install: - - pacman -Sy httpie - upgrade: - pacman -Syu httpie + upgrade: + - pacman -Syu pkg: title: FreshPorts diff --git a/docs/packaging/linux-fedora/httpie.spec.txt b/docs/packaging/linux-fedora/httpie.spec.txt index b0ab5b7b3a..f18f487801 100644 --- a/docs/packaging/linux-fedora/httpie.spec.txt +++ b/docs/packaging/linux-fedora/httpie.spec.txt @@ -1,5 +1,5 @@ Name: httpie -Version: 2.6.0 +Version: 3.0.2 Release: 1%{?dist} Summary: A Curl-like tool for humans @@ -78,6 +78,21 @@ help2man %{buildroot}%{_bindir}/httpie > %{buildroot}%{_mandir}/man1/httpie.1 %changelog +* Mon Jan 24 2022 Miro Hrončok - 3.0.2-1 +- Update to 3.0.2 +- Fixes: rhbz#2044572 + +* Mon Jan 24 2022 Miro Hrončok - 3.0.1-1 +- Update to 3.0.1 +- Fixes: rhbz#2044058 + +* Fri Jan 21 2022 Miro Hrončok - 3.0.0-1 +- Update to 3.0.0 +- Fixes: rhbz#2043680 + +* Thu Jan 20 2022 Fedora Release Engineering - 2.6.0-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_36_Mass_Rebuild + * Fri Oct 15 2021 Miro Hrončok - 2.6.0-1 - Update to 2.6.0 - Fixes: rhbz#2014022 diff --git a/docs/packaging/windows-chocolatey/httpie.nuspec b/docs/packaging/windows-chocolatey/httpie.nuspec index ce69afa5a7..0406a1f21e 100644 --- a/docs/packaging/windows-chocolatey/httpie.nuspec +++ b/docs/packaging/windows-chocolatey/httpie.nuspec @@ -2,7 +2,7 @@ httpie - 2.6.0 + 3.0.2 Modern, user-friendly command-line HTTP client for the API era HTTPie *aitch-tee-tee-pie* is a user-friendly command-line HTTP client for the API era. @@ -29,11 +29,11 @@ Main features: HTTPie HTTPie jakubroztocil - 2012-2021 Jakub Roztocil + 2012-2022 Jakub Roztocil https://raw.githubusercontent.com/httpie/httpie/master/LICENSE https://pie-assets.s3.eu-central-1.amazonaws.com/LogoIcons/GB.png false - See the [changelog](https://github.com/httpie/httpie/blob/2.6.0/CHANGELOG.md). + See the [changelog](https://httpie.io/blog/httpie-3.0.0). httpie http https rest api client curl python ssl cli foss oss url https://httpie.io https://github.com/httpie/httpie/tree/master/docs/packaging/windows-chocolatey diff --git a/httpie/__init__.py b/httpie/__init__.py index 7d48b3499c..683e7b98a7 100644 --- a/httpie/__init__.py +++ b/httpie/__init__.py @@ -3,6 +3,6 @@ """ -__version__ = '3.0.2' +__version__ = '3.0.3.dev0' __author__ = 'Jakub Roztocil' __licence__ = 'BSD' diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index 64481096c7..a312b8ba10 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -10,7 +10,8 @@ from requests.utils import get_netrc_auth from .argtypes import ( - AuthCredentials, KeyValueArgType, PARSED_DEFAULT_FORMAT_OPTIONS, + AuthCredentials, SSLCredentials, KeyValueArgType, + PARSED_DEFAULT_FORMAT_OPTIONS, parse_auth, parse_format_options, ) @@ -47,12 +48,39 @@ def _split_lines(self, text, width): text = dedent(text).strip() + '\n\n' return text.splitlines() + def add_usage(self, usage, actions, groups, prefix=None): + # Only display the positional arguments + displayed_actions = [ + action + for action in actions + if not action.option_strings + ] + + _, exception, _ = sys.exc_info() + if ( + isinstance(exception, argparse.ArgumentError) + and len(exception.args) >= 1 + and isinstance(exception.args[0], argparse.Action) + ): + # add_usage path is also taken when you pass an invalid option, + # e.g --style=invalid. If something like that happens, we want + # to include to action that caused to the invalid usage into + # the list of actions we are displaying. + displayed_actions.insert(0, exception.args[0]) + + super().add_usage( + usage, + displayed_actions, + groups, + prefix="usage:\n " + ) + # TODO: refactor and design type-annotated data structures # for raw args + parsed args and keep things immutable. class BaseHTTPieArgumentParser(argparse.ArgumentParser): - def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs): - super().__init__(*args, formatter_class=formatter_class, **kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.env = None self.args = None self.has_stdin_data = False @@ -115,9 +143,9 @@ class HTTPieArgumentParser(BaseHTTPieArgumentParser): """ - def __init__(self, *args, **kwargs): + def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs): kwargs.setdefault('add_help', False) - super().__init__(*args, **kwargs) + super().__init__(*args, formatter_class=formatter_class, **kwargs) # noinspection PyMethodOverriding def parse_args( @@ -148,6 +176,7 @@ def parse_args( self._parse_items() self._process_url() self._process_auth() + self._process_ssl_cert() if self.args.raw is not None: self._body_from_input(self.args.raw) @@ -230,9 +259,24 @@ def _setup_standard_streams(self): self.env.stdout_isatty = False if self.args.quiet: + self.env.quiet = self.args.quiet self.env.stderr = self.env.devnull if not (self.args.output_file_specified and not self.args.download): self.env.stdout = self.env.devnull + self.env.apply_warnings_filter() + + def _process_ssl_cert(self): + from httpie.ssl_ import _is_key_file_encrypted + + if self.args.cert_key_pass is None: + self.args.cert_key_pass = SSLCredentials(None) + + if ( + self.args.cert_key is not None + and self.args.cert_key_pass.value is None + and _is_key_file_encrypted(self.args.cert_key) + ): + self.args.cert_key_pass.prompt_password(self.args.cert_key) def _process_auth(self): # TODO: refactor & simplify this method. @@ -512,3 +556,20 @@ def _process_format_options(self): for options_group in format_options: parsed_options = parse_format_options(options_group, defaults=parsed_options) self.args.format_options = parsed_options + + def error(self, message): + """Prints a usage message incorporating the message to stderr and + exits.""" + self.print_usage(sys.stderr) + self.exit( + 2, + dedent( + f''' + error: + {message} + + for more information: + run '{self.prog} --help' or visit https://httpie.io/docs/cli + ''' + ) + ) diff --git a/httpie/cli/argtypes.py b/httpie/cli/argtypes.py index 7bc487ea81..8f19c3c51e 100644 --- a/httpie/cli/argtypes.py +++ b/httpie/cli/argtypes.py @@ -130,16 +130,11 @@ def tokenize(self, s: str) -> List[Union[str, Escaped]]: return tokens -class AuthCredentials(KeyValueArg): - """Represents parsed credentials.""" - - def has_password(self) -> bool: - return self.value is not None - - def prompt_password(self, host: str): - prompt_text = f'http: password for {self.key}@{host}: ' +class PromptMixin: + def _prompt_password(self, prompt: str) -> str: + prompt_text = f'http: {prompt}: ' try: - self.value = self._getpass(prompt_text) + return self._getpass(prompt_text) except (EOFError, KeyboardInterrupt): sys.stderr.write('\n') sys.exit(0) @@ -150,6 +145,26 @@ def _getpass(prompt): return getpass.getpass(str(prompt)) +class SSLCredentials(PromptMixin): + """Represents the passphrase for the certificate's key.""" + + def __init__(self, value: Optional[str]) -> None: + self.value = value + + def prompt_password(self, key_file: str) -> None: + self.value = self._prompt_password(f'passphrase for {key_file}') + + +class AuthCredentials(KeyValueArg, PromptMixin): + """Represents parsed credentials.""" + + def has_password(self) -> bool: + return self.value is not None + + def prompt_password(self, host: str) -> None: + self.value = self._prompt_password(f'password for {self.key}@{host}:') + + class AuthCredentialsArgType(KeyValueArgType): """A key-value arg type that parses credentials.""" diff --git a/httpie/cli/constants.py b/httpie/cli/constants.py index 59490219eb..e8188938d5 100644 --- a/httpie/cli/constants.py +++ b/httpie/cli/constants.py @@ -133,6 +133,7 @@ class RequestType(enum.Enum): JSON = enum.auto() +EMPTY_STRING = '' OPEN_BRACKET = '[' CLOSE_BRACKET = ']' BACKSLASH = '\\' diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 2935ae3aa2..5db5b390d2 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -5,8 +5,8 @@ from httpie import __doc__, __version__ from httpie.cli.argtypes import (KeyValueArgType, SessionNameValidator, - readable_file_arg, response_charset_type, - response_mime_type) + SSLCredentials, readable_file_arg, + response_charset_type, response_mime_type) from httpie.cli.constants import (BASE_OUTPUT_OPTIONS, DEFAULT_FORMAT_OPTIONS, OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY, OUT_RESP_HEAD, OUT_RESP_META, OUTPUT_OPTIONS, @@ -29,19 +29,17 @@ epilog=""" For every --OPTION there is also a --no-OPTION that reverts OPTION to its default value. - Suggestions and bug reports are greatly appreciated: - https://github.com/httpie/httpie/issues - """, ) + ####################################################################### # Positional arguments. ####################################################################### -positional_arguments = options.new_group( +positional_arguments = options.add_group( 'Positional Arguments', description=""" These arguments come after any flags and in the order they are listed here. @@ -130,7 +128,7 @@ # Content type. ####################################################################### -content_types = options.new_group('Predefined Content Types') +content_types = options.add_group('Predefined Content Types') content_types.add_argument( '--json', @@ -203,7 +201,7 @@ # Content processing. ####################################################################### -processing_options = options.new_group('Content Processing Options') +processing_options = options.add_group('Content Processing Options') processing_options.add_argument( '--compress', @@ -256,7 +254,7 @@ def format_style_help(available_styles): 'dest': 'format_options', } -output_processing = options.new_group('Output Processing') +output_processing = options.add_group('Output Processing') output_processing.add_argument( '--pretty', @@ -363,7 +361,7 @@ def format_style_help(available_styles): # Output options ####################################################################### -output_options = options.new_group('Output Options') +output_options = options.add_group('Output Options') output_options.add_argument( '--print', @@ -540,7 +538,7 @@ def format_style_help(available_styles): 'Session name contains invalid characters.' ) -sessions = options.new_group('Sessions', is_mutually_exclusive=True) +sessions = options.add_group('Sessions', is_mutually_exclusive=True) sessions.add_argument( '--session', @@ -603,7 +601,7 @@ def format_auth_help(auth_plugins_mapping): ) -authentication = options.new_group('Authentication') +authentication = options.add_group('Authentication') authentication.add_argument( '--auth', @@ -641,7 +639,7 @@ def format_auth_help(auth_plugins_mapping): # Network ####################################################################### -network = options.new_group('Network') +network = options.add_group('Network') network.add_argument( '--offline', @@ -751,7 +749,7 @@ def format_auth_help(auth_plugins_mapping): # SSL ####################################################################### -ssl = options.new_group('SSL') +ssl = options.add_group('SSL') ssl.add_argument( '--verify', @@ -809,11 +807,22 @@ def format_auth_help(auth_plugins_mapping): """, ) +ssl.add_argument( + '--cert-key-pass', + default=None, + type=SSLCredentials, + help=''' + The passphrase to be used to with the given private key. Only needed if --cert-key + is given and the key file requires a passphrase. + If not provided, you’ll be prompted interactively. + ''' +) + ####################################################################### # Troubleshooting ####################################################################### -troubleshooting = options.new_group('Troubleshooting') +troubleshooting = options.add_group('Troubleshooting') troubleshooting.add_argument( '--ignore-stdin', '-I', diff --git a/httpie/cli/dicts.py b/httpie/cli/dicts.py index 3d0cab5a45..434a396672 100644 --- a/httpie/cli/dicts.py +++ b/httpie/cli/dicts.py @@ -82,3 +82,7 @@ class MultipartRequestDataDict(MultiValueOrderedDict): class RequestFilesDict(RequestDataDict): pass + + +class NestedJSONArray(list): + """Denotes a top-level JSON array.""" diff --git a/httpie/cli/nested_json.py b/httpie/cli/nested_json.py index 501e3b594f..c75798929d 100644 --- a/httpie/cli/nested_json.py +++ b/httpie/cli/nested_json.py @@ -9,7 +9,8 @@ Type, Union, ) -from httpie.cli.constants import OPEN_BRACKET, CLOSE_BRACKET, BACKSLASH, HIGHLIGHTER +from httpie.cli.dicts import NestedJSONArray +from httpie.cli.constants import EMPTY_STRING, OPEN_BRACKET, CLOSE_BRACKET, BACKSLASH, HIGHLIGHTER class HTTPieSyntaxError(ValueError): @@ -52,6 +53,7 @@ def to_name(self) -> str: OPERATORS = {OPEN_BRACKET: TokenKind.LEFT_BRACKET, CLOSE_BRACKET: TokenKind.RIGHT_BRACKET} SPECIAL_CHARS = OPERATORS.keys() | {BACKSLASH} +LITERAL_TOKENS = [TokenKind.TEXT, TokenKind.NUMBER] class Token(NamedTuple): @@ -88,18 +90,18 @@ def send_buffer() -> Iterator[Token]: return None value = ''.join(buffer) - for variation, kind in [ - (int, TokenKind.NUMBER), - (check_escaped_int, TokenKind.TEXT), - ]: - try: - value = variation(value) - except ValueError: - continue - else: - break - else: - kind = TokenKind.TEXT + kind = TokenKind.TEXT + if not backslashes: + for variation, kind in [ + (int, TokenKind.NUMBER), + (check_escaped_int, TokenKind.TEXT), + ]: + try: + value = variation(value) + except ValueError: + continue + else: + break yield Token( kind, value, start=cursor - (len(buffer) + backslashes), end=cursor @@ -171,8 +173,8 @@ def reconstruct(self) -> str: def parse(source: str) -> Iterator[Path]: """ - start: literal? path* - + start: root_path path* + root_path: (literal | index_path | append_path) literal: TEXT | NUMBER path: @@ -215,16 +217,47 @@ def expect(*kinds): message = f'Expecting {suffix}' raise HTTPieSyntaxError(source, token, message) - root = Path(PathAction.KEY, '', is_root=True) - if can_advance(): - token = tokens[cursor] - if token.kind in {TokenKind.TEXT, TokenKind.NUMBER}: - token = expect(TokenKind.TEXT, TokenKind.NUMBER) - root.accessor = str(token.value) - root.tokens.append(token) + def parse_root(): + tokens = [] + if not can_advance(): + return Path( + PathAction.KEY, + EMPTY_STRING, + is_root=True + ) - yield root + # (literal | index_path | append_path)? + token = expect(*LITERAL_TOKENS, TokenKind.LEFT_BRACKET) + tokens.append(token) + + if token.kind in LITERAL_TOKENS: + action = PathAction.KEY + value = str(token.value) + elif token.kind is TokenKind.LEFT_BRACKET: + token = expect(TokenKind.NUMBER, TokenKind.RIGHT_BRACKET) + tokens.append(token) + if token.kind is TokenKind.NUMBER: + action = PathAction.INDEX + value = token.value + tokens.append(expect(TokenKind.RIGHT_BRACKET)) + elif token.kind is TokenKind.RIGHT_BRACKET: + action = PathAction.APPEND + value = None + else: + assert_cant_happen() + else: + assert_cant_happen() + return Path( + action, + value, + tokens=tokens, + is_root=True + ) + + yield parse_root() + + # path* while can_advance(): path_tokens = [] path_tokens.append(expect(TokenKind.LEFT_BRACKET)) @@ -296,6 +329,10 @@ def object_for(kind: str) -> Any: assert_cant_happen() for index, (path, next_path) in enumerate(zip(paths, paths[1:])): + # If there is no context yet, set it. + if cursor is None: + context = cursor = object_for(path.kind) + if path.kind is PathAction.KEY: type_check(index, path, dict) if next_path.kind is PathAction.SET: @@ -337,8 +374,19 @@ def object_for(kind: str) -> Any: return context +def wrap_with_dict(context): + if context is None: + return {} + elif isinstance(context, list): + return {EMPTY_STRING: NestedJSONArray(context)} + else: + assert isinstance(context, dict) + return context + + def interpret_nested_json(pairs): - context = {} + context = None for key, value in pairs: - interpret(context, key, value) - return context + context = interpret(context, key, value) + + return wrap_with_dict(context) diff --git a/httpie/cli/options.py b/httpie/cli/options.py index 5bb9f93272..45f669341b 100644 --- a/httpie/cli/options.py +++ b/httpie/cli/options.py @@ -43,7 +43,7 @@ def finalize(self) -> 'ParserSpec': group.finalize() return self - def new_group(self, name: str, **kwargs) -> 'Group': + def add_group(self, name: str, **kwargs) -> 'Group': group = Group(name, **kwargs) self.groups.append(group) return group @@ -100,7 +100,7 @@ def serialize(self) -> Dict[str, Any]: if self.aliases: result['options'] = self.aliases.copy() else: - result['options'] = configuration['metavar'] + result['options'] = [configuration['metavar']] result['is_positional'] = True qualifiers = JSON_QUALIFIER_TO_OPTIONS[configuration.get('nargs', Qualifiers.SUPPRESS)] @@ -108,7 +108,7 @@ def serialize(self) -> Dict[str, Any]: help_msg = configuration.get('help') if help_msg and help_msg is not Qualifiers.SUPPRESS: - result['description'] = help_msg.strip().splitlines() + result['description'] = help_msg.strip() python_type = configuration.get('type') if python_type is not None: diff --git a/httpie/client.py b/httpie/client.py index c2563cbc3e..530d589cae 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -13,12 +13,13 @@ from . import __version__ from .adapters import HTTPieHTTPAdapter from .context import Environment -from .cli.dicts import HTTPHeadersDict +from .cli.constants import EMPTY_STRING +from .cli.dicts import HTTPHeadersDict, NestedJSONArray from .encoding import UTF8 from .models import RequestsMessage from .plugins.registry import plugin_manager from .sessions import get_httpie_session -from .ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter +from .ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieCertificate, HTTPieHTTPSAdapter from .uploads import ( compress_request, prepare_request_body, get_multipart_data_and_content_type, @@ -43,6 +44,7 @@ def collect_messages( httpie_session_headers = None if args.session or args.session_read_only: httpie_session = get_httpie_session( + env=env, config_dir=env.config.directory, session_name=args.session or args.session_read_only, host=args.headers.get('Host'), @@ -129,10 +131,7 @@ def collect_messages( if httpie_session: if httpie_session.is_new() or not args.session_read_only: httpie_session.cookies = requests_session.cookies - httpie_session.remove_cookies( - # TODO: take path & domain into account? - cookie['name'] for cookie in expired_cookies - ) + httpie_session.remove_cookies(expired_cookies) httpie_session.save() @@ -261,7 +260,14 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict: if args.cert: cert = args.cert if args.cert_key: - cert = cert, args.cert_key + # Having a client certificate key passphrase is not supported + # by requests. So we are using our own transportation structure + # which is compatible with their format (a tuple of minimum two + # items). + # + # See: https://github.com/psf/requests/issues/2519 + cert = HTTPieCertificate(cert, args.cert_key, args.cert_key_pass.value) + return { 'proxies': {p.key: p.value for p in args.proxy}, 'stream': True, @@ -280,7 +286,8 @@ def json_dict_to_request_body(data: Dict[str, Any]) -> str: # item in the object, with an en empty key. if len(data) == 1: [(key, value)] = data.items() - if key == '' and isinstance(value, list): + if isinstance(value, NestedJSONArray): + assert key == EMPTY_STRING data = value if data: diff --git a/httpie/config.py b/httpie/config.py index 28574e4ae7..f7fee5bdab 100644 --- a/httpie/config.py +++ b/httpie/config.py @@ -1,7 +1,7 @@ import json import os from pathlib import Path -from typing import Union +from typing import Any, Dict, Union from . import __version__ from .compat import is_windows @@ -62,6 +62,21 @@ class ConfigFileError(Exception): pass +def read_raw_config(config_type: str, path: Path) -> Dict[str, Any]: + try: + with path.open(encoding=UTF8) as f: + try: + return json.load(f) + except ValueError as e: + raise ConfigFileError( + f'invalid {config_type} file: {e} [{path}]' + ) + except FileNotFoundError: + pass + except OSError as e: + raise ConfigFileError(f'cannot read {config_type} file: {e}') + + class BaseConfigDict(dict): name = None helpurl = None @@ -77,26 +92,25 @@ def ensure_directory(self): def is_new(self) -> bool: return not self.path.exists() + def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Hook for processing the incoming config data.""" + return data + + def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Hook for processing the outgoing config data.""" + return data + def load(self): config_type = type(self).__name__.lower() - try: - with self.path.open(encoding=UTF8) as f: - try: - data = json.load(f) - except ValueError as e: - raise ConfigFileError( - f'invalid {config_type} file: {e} [{self.path}]' - ) - self.update(data) - except FileNotFoundError: - pass - except OSError as e: - raise ConfigFileError(f'cannot read {config_type} file: {e}') - - def save(self): - self['__meta__'] = { - 'httpie': __version__ - } + data = read_raw_config(config_type, self.path) + if data is not None: + data = self.pre_process_data(data) + self.update(data) + + def save(self, *, bump_version: bool = False): + self.setdefault('__meta__', {}) + if bump_version or 'httpie' not in self['__meta__']: + self['__meta__']['httpie'] = __version__ if self.helpurl: self['__meta__']['help'] = self.helpurl @@ -106,13 +120,19 @@ def save(self): self.ensure_directory() json_string = json.dumps( - obj=self, + obj=self.post_process_data(self), indent=4, sort_keys=True, ensure_ascii=True, ) self.path.write_text(json_string + '\n', encoding=UTF8) + @property + def version(self): + return self.get( + '__meta__', {} + ).get('httpie', __version__) + class Config(BaseConfigDict): FILENAME = 'config.json' diff --git a/httpie/context.py b/httpie/context.py index 7a6e6a865c..50a8f772cd 100644 --- a/httpie/context.py +++ b/httpie/context.py @@ -1,8 +1,10 @@ import sys import os +import warnings from contextlib import contextmanager from pathlib import Path from typing import Iterator, IO, Optional +from enum import Enum try: @@ -17,6 +19,17 @@ from .utils import repr_dict +class Levels(str, Enum): + WARNING = 'warning' + ERROR = 'error' + + +DISPLAY_THRESHOLDS = { + Levels.WARNING: 2, + Levels.ERROR: float('inf'), # Never hide errors. +} + + class Environment: """ Information about the execution context @@ -87,6 +100,8 @@ def __init__(self, devnull=None, **kwargs): self.stdout_encoding = getattr( actual_stdout, 'encoding', None) or UTF8 + self.quiet = kwargs.pop('quiet', 0) + def __str__(self): defaults = dict(type(self).__dict__) actual = dict(defaults) @@ -134,6 +149,14 @@ def as_silent(self) -> Iterator[None]: self.stdout = original_stdout self.stderr = original_stderr - def log_error(self, msg, level='error'): - assert level in ['error', 'warning'] - self._orig_stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n') + def log_error(self, msg: str, level: Levels = Levels.ERROR) -> None: + if self.stdout_isatty and self.quiet >= DISPLAY_THRESHOLDS[level]: + stderr = self.stderr # Not directly /dev/null, since stderr might be mocked + else: + stderr = self._orig_stderr + + stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n') + + def apply_warnings_filter(self) -> None: + if self.quiet >= DISPLAY_THRESHOLDS[Levels.WARNING]: + warnings.simplefilter("ignore") diff --git a/httpie/core.py b/httpie/core.py index 8831c81632..98f47f344a 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -13,7 +13,7 @@ from .cli.constants import OUT_REQ_BODY from .cli.nested_json import HTTPieSyntaxError from .client import collect_messages -from .context import Environment +from .context import Environment, Levels from .downloads import Downloader from .models import ( RequestsMessageKind, @@ -223,7 +223,7 @@ def request_body_read_callback(chunk: bytes): if args.check_status or downloader: exit_status = http_status_to_exit_status(http_status=message.status_code, follow=args.follow) if exit_status != ExitStatus.SUCCESS and (not env.stdout_isatty or args.quiet == 1): - env.log_error(f'HTTP {message.raw.status} {message.raw.reason}', level='warning') + env.log_error(f'HTTP {message.raw.status} {message.raw.reason}', level=Levels.WARNING) write_message( requests_message=message, env=env, diff --git a/httpie/legacy/__init__.py b/httpie/legacy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/httpie/legacy/cookie_format.py b/httpie/legacy/cookie_format.py new file mode 100644 index 0000000000..b5c6392b7c --- /dev/null +++ b/httpie/legacy/cookie_format.py @@ -0,0 +1,103 @@ +import argparse +from typing import Any, Type, List, Dict, TYPE_CHECKING + +if TYPE_CHECKING: + from httpie.sessions import Session + +INSECURE_COOKIE_JAR_WARNING = '''\ +Outdated layout detected for the current session. Please consider updating it, +in order to not get affected by potential security problems. + +For fixing the current session: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade --bind-cookies {hostname} {session_id} + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade {hostname} {session_id} +''' + + +INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS = '''\ + +For fixing all named sessions: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade-all --bind-cookies + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade-all +''' + +INSECURE_COOKIE_SECURITY_LINK = '\nSee https://pie.co/docs/security for more information.' + + +def pre_process(session: 'Session', cookies: Any) -> List[Dict[str, Any]]: + """Load the given cookies to the cookie jar while maintaining + support for the old cookie layout.""" + + is_old_style = isinstance(cookies, dict) + if is_old_style: + normalized_cookies = [ + { + 'name': key, + **value + } + for key, value in cookies.items() + ] + else: + normalized_cookies = cookies + + should_issue_warning = is_old_style and any( + cookie.get('domain', '') == '' + for cookie in normalized_cookies + ) + + if should_issue_warning and not session.refactor_mode: + warning = INSECURE_COOKIE_JAR_WARNING.format(hostname=session.bound_host, session_id=session.session_id) + if not session.is_anonymous: + warning += INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS + warning += INSECURE_COOKIE_SECURITY_LINK + + session.env.log_error( + warning, + level='warning' + ) + + return normalized_cookies + + +def post_process( + normalized_cookies: List[Dict[str, Any]], + *, + original_type: Type[Any] +) -> Any: + """Convert the cookies to their original format for + maximum compatibility.""" + + if issubclass(original_type, dict): + return { + cookie.pop('name'): cookie + for cookie in normalized_cookies + } + else: + return normalized_cookies + + +def fix_layout(session: 'Session', hostname: str, args: argparse.Namespace) -> None: + if not isinstance(session['cookies'], dict): + return None + + session['cookies'] = [ + { + 'name': key, + **value + } + for key, value in session['cookies'].items() + ] + for cookie in session.cookies: + if cookie.domain == '': + if args.bind_cookies: + cookie.domain = hostname + else: + cookie._rest['is_explicit_none'] = True diff --git a/httpie/manager/cli.py b/httpie/manager/cli.py index 2c10ca2b66..313f145007 100644 --- a/httpie/manager/cli.py +++ b/httpie/manager/cli.py @@ -2,6 +2,15 @@ from httpie.cli.argparser import HTTPieManagerArgumentParser from httpie import __version__ +CLI_SESSION_UPGRADE_FLAGS = [ + { + 'flags': ['--bind-cookies'], + 'action': 'store_true', + 'default': False, + 'help': 'Bind domainless cookies to the host that session belongs.' + } +] + COMMANDS = { 'plugins': { 'help': 'Manage HTTPie plugins.', @@ -36,14 +45,39 @@ }, 'cli': { 'help': 'Manage HTTPie for Terminal', - 'export': [ + 'export-args': [ 'Export available options for the CLI', { - 'variadic': ['-f', '--format'], + 'flags': ['-f', '--format'], 'choices': ['json'], 'default': 'json' } - ] + ], + 'sessions': { + 'help': 'Manage HTTPie sessions', + 'upgrade': [ + 'Upgrade the given HTTPie session with the latest ' + 'layout. A list of changes between different session versions ' + 'can be found in the official documentation.', + { + 'dest': 'hostname', + 'metavar': 'HOSTNAME', + 'help': 'The host this session belongs.' + }, + { + 'dest': 'session', + 'metavar': 'SESSION_NAME_OR_PATH', + 'help': 'The name or the path for the session that will be upgraded.' + }, + *CLI_SESSION_UPGRADE_FLAGS + ], + 'upgrade-all': [ + 'Upgrade all named sessions with the latest layout. A list of ' + 'changes between different session versions can be found in the official ' + 'documentation.', + *CLI_SESSION_UPGRADE_FLAGS + ], + } } } @@ -65,6 +99,8 @@ def generate_subparsers(root, parent_parser, definitions): ) for command, properties in definitions.items(): is_subparser = isinstance(properties, dict) + properties = properties.copy() + descr = properties.pop('help', None) if is_subparser else properties.pop(0) command_parser = actions.add_parser(command, description=descr) command_parser.root = root @@ -73,7 +109,9 @@ def generate_subparsers(root, parent_parser, definitions): continue for argument in properties: - command_parser.add_argument(*argument.pop('variadic', []), **argument) + argument = argument.copy() + flags = argument.pop('flags', []) + command_parser.add_argument(*flags, **argument) parser = HTTPieManagerArgumentParser( diff --git a/httpie/manager/tasks.py b/httpie/manager/tasks.py index 7beb0c5c42..5b726a9bb5 100644 --- a/httpie/manager/tasks.py +++ b/httpie/manager/tasks.py @@ -1,12 +1,13 @@ import argparse -import json -from typing import TypeVar, Callable +from typing import TypeVar, Callable, Tuple +from httpie.sessions import SESSIONS_DIR_NAME, get_httpie_session from httpie.status import ExitStatus from httpie.context import Environment -from httpie.output.writer import write_raw_data +from httpie.legacy import cookie_format as legacy_cookies +from httpie.manager.cli import missing_subcommand, parser -T = TypeVar("T") +T = TypeVar('T') CLI_TASKS = {} @@ -18,20 +19,119 @@ def wrapper(func: T) -> T: return wrapper +@task('sessions') +def cli_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus: + action = args.cli_sessions_action + if action is None: + parser.error(missing_subcommand('cli', 'sessions')) + + if action == 'upgrade': + return cli_upgrade_session(env, args) + elif action == 'upgrade-all': + return cli_upgrade_all_sessions(env, args) + else: + raise ValueError(f'Unexpected action: {action}') + + +def is_version_greater(version_1: str, version_2: str) -> bool: + # In an ideal scenerio, we would depend on `packaging` in order + # to offer PEP 440 compatible parsing. But since it might not be + # commonly available for outside packages, and since we are only + # going to parse HTTPie's own version it should be fine to compare + # this in a SemVer subset fashion. + + def split_version(version: str) -> Tuple[int, ...]: + parts = [] + for part in version.split('.')[:3]: + try: + parts.append(int(part)) + except ValueError: + break + return tuple(parts) + + return split_version(version_1) > split_version(version_2) + + +FIXERS_TO_VERSIONS = { + '3.1.0': legacy_cookies.fix_layout +} + + +def upgrade_session(env: Environment, args: argparse.Namespace, hostname: str, session_name: str): + session = get_httpie_session( + env=env, + config_dir=env.config.directory, + session_name=session_name, + host=hostname, + url=hostname, + refactor_mode=True + ) + + session_name = session.path.stem + if session.is_new(): + env.log_error(f'{session_name!r} @ {hostname!r} does not exist.') + return ExitStatus.ERROR + + fixers = [ + fixer + for version, fixer in FIXERS_TO_VERSIONS.items() + if is_version_greater(version, session.version) + ] + + if len(fixers) == 0: + env.stdout.write(f'{session_name!r} @ {hostname!r} is already up to date.\n') + return ExitStatus.SUCCESS + + for fixer in fixers: + fixer(session, hostname, args) + + session.save(bump_version=True) + env.stdout.write(f'Upgraded {session_name!r} @ {hostname!r} to v{session.version}\n') + return ExitStatus.SUCCESS + + +def cli_upgrade_session(env: Environment, args: argparse.Namespace) -> ExitStatus: + return upgrade_session( + env, + args=args, + hostname=args.hostname, + session_name=args.session + ) + + +def cli_upgrade_all_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus: + session_dir_path = env.config_dir / SESSIONS_DIR_NAME + + status = ExitStatus.SUCCESS + for host_path in session_dir_path.iterdir(): + hostname = host_path.name + for session_path in host_path.glob("*.json"): + session_name = session_path.stem + status |= upgrade_session( + env, + args=args, + hostname=hostname, + session_name=session_name + ) + return status + + FORMAT_TO_CONTENT_TYPE = { 'json': 'application/json' } -@task("export") +@task('export-args') def cli_export(env: Environment, args: argparse.Namespace) -> ExitStatus: + import json from httpie.cli.definition import options from httpie.cli.options import to_data + from httpie.output.writer import write_raw_data if args.format == 'json': data = json.dumps(to_data(options)) else: - raise NotImplementedError(f"Unexpected format value: {args.format}") + raise NotImplementedError(f'Unexpected format value: {args.format}') write_raw_data( env, diff --git a/httpie/output/lexers/http.py b/httpie/output/lexers/http.py index f06a685380..aea827401e 100644 --- a/httpie/output/lexers/http.py +++ b/httpie/output/lexers/http.py @@ -2,7 +2,7 @@ import pygments from httpie.output.lexers.common import precise -RE_STATUS_LINE = re.compile(r'(\d{3})( +)(.+)') +RE_STATUS_LINE = re.compile(r'(\d{3})( +)?(.+)?') STATUS_TYPES = { '1': pygments.token.Number.HTTP.INFO, diff --git a/httpie/sessions.py b/httpie/sessions.py index 176c03e76d..e4a20a5344 100644 --- a/httpie/sessions.py +++ b/httpie/sessions.py @@ -6,16 +6,19 @@ import re from http.cookies import SimpleCookie +from http.cookiejar import Cookie from pathlib import Path -from typing import Iterable, Optional, Union -from urllib.parse import urlsplit +from typing import Any, Dict, List, Optional, Union from requests.auth import AuthBase -from requests.cookies import RequestsCookieJar, create_cookie +from requests.cookies import RequestsCookieJar, remove_cookie_by_name +from .context import Environment from .cli.dicts import HTTPHeadersDict from .config import BaseConfigDict, DEFAULT_CONFIG_DIR +from .utils import url_as_host from .plugins.registry import plugin_manager +from .legacy import cookie_format as legacy_cookies SESSIONS_DIR_NAME = 'sessions' @@ -26,27 +29,72 @@ # SESSION_IGNORED_HEADER_PREFIXES = ['Content-', 'If-'] +# Cookie related options +KEPT_COOKIE_OPTIONS = ['name', 'expires', 'path', 'value', 'domain', 'secure'] +DEFAULT_COOKIE_PATH = '/' + + +def is_anonymous_session(session_name: str) -> bool: + return os.path.sep in session_name + + +def session_hostname_to_dirname(hostname: str, session_name: str) -> str: + # host:port => host_port + hostname = hostname.replace(':', '_') + return os.path.join( + SESSIONS_DIR_NAME, + hostname, + f'{session_name}.json' + ) + + +def strip_port(hostname: str) -> str: + return hostname.split(':')[0] + + +def materialize_cookie(cookie: Cookie) -> Dict[str, Any]: + materialized_cookie = { + option: getattr(cookie, option) + for option in KEPT_COOKIE_OPTIONS + } + + if ( + cookie._rest.get('is_explicit_none') + and materialized_cookie['domain'] == '' + ): + materialized_cookie['domain'] = None + + return materialized_cookie + def get_httpie_session( + env: Environment, config_dir: Path, session_name: str, host: Optional[str], url: str, + *, + refactor_mode: bool = False ) -> 'Session': - if os.path.sep in session_name: + bound_hostname = host or url_as_host(url) + if not bound_hostname: + # HACK/FIXME: httpie-unixsocket's URLs have no hostname. + bound_hostname = 'localhost' + + if is_anonymous_session(session_name): path = os.path.expanduser(session_name) + session_id = path else: - hostname = host or urlsplit(url).netloc.split('@')[-1] - if not hostname: - # HACK/FIXME: httpie-unixsocket's URLs have no hostname. - hostname = 'localhost' - - # host:port => host_port - hostname = hostname.replace(':', '_') - path = ( - config_dir / SESSIONS_DIR_NAME / hostname / f'{session_name}.json' - ) - session = Session(path) + path = config_dir / session_hostname_to_dirname(bound_hostname, session_name) + session_id = session_name + + session = Session( + path, + env=env, + session_id=session_id, + bound_host=strip_port(bound_hostname), + refactor_mode=refactor_mode + ) session.load() return session @@ -55,15 +103,61 @@ class Session(BaseConfigDict): helpurl = 'https://httpie.io/docs#sessions' about = 'HTTPie session file' - def __init__(self, path: Union[str, Path]): + def __init__( + self, + path: Union[str, Path], + env: Environment, + bound_host: str, + session_id: str, + refactor_mode: bool = False, + ): super().__init__(path=Path(path)) self['headers'] = {} - self['cookies'] = {} + self['cookies'] = [] self['auth'] = { 'type': None, 'username': None, 'password': None } + self.env = env + self.cookie_jar = RequestsCookieJar() + self.session_id = session_id + self.bound_host = bound_host + self.refactor_mode = refactor_mode + + def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + cookies = data.get('cookies') + if cookies: + normalized_cookies = legacy_cookies.pre_process(self, cookies) + else: + normalized_cookies = [] + + for cookie in normalized_cookies: + domain = cookie.get('domain', '') + if domain is None: + # domain = None means explicitly lack of cookie, though + # requests requires domain to be a string so we'll cast it + # manually. + cookie['domain'] = '' + cookie['rest'] = {'is_explicit_none': True} + + self.cookie_jar.set(**cookie) + + return data + + def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + cookies = data.get('cookies') + + normalized_cookies = [ + materialize_cookie(cookie) + for cookie in self.cookie_jar + ] + data['cookies'] = legacy_cookies.post_process( + normalized_cookies, + original_type=type(cookies) + ) + + return data def update_headers(self, request_headers: HTTPHeadersDict): """ @@ -73,10 +167,10 @@ def update_headers(self, request_headers: HTTPHeadersDict): """ headers = self.headers for name, value in request_headers.copy().items(): - if value is None: continue # Ignore explicitly unset headers + original_value = value if type(value) is not str: value = value.decode() @@ -85,8 +179,15 @@ def update_headers(self, request_headers: HTTPHeadersDict): if name.lower() == 'cookie': for cookie_name, morsel in SimpleCookie(value).items(): - self['cookies'][cookie_name] = {'value': morsel.value} - del request_headers[name] + if not morsel['path']: + morsel['path'] = DEFAULT_COOKIE_PATH + self.cookie_jar.set(cookie_name, morsel) + + all_cookie_headers = request_headers.getall(name) + if len(all_cookie_headers) > 1: + all_cookie_headers.remove(original_value) + else: + request_headers.popall(name) continue for prefix in SESSION_IGNORED_HEADER_PREFIXES: @@ -103,23 +204,21 @@ def headers(self) -> HTTPHeadersDict: @property def cookies(self) -> RequestsCookieJar: - jar = RequestsCookieJar() - for name, cookie_dict in self['cookies'].items(): - jar.set_cookie(create_cookie( - name, cookie_dict.pop('value'), **cookie_dict)) - jar.clear_expired_cookies() - return jar + self.cookie_jar.clear_expired_cookies() + return self.cookie_jar @cookies.setter def cookies(self, jar: RequestsCookieJar): - # - stored_attrs = ['value', 'path', 'secure', 'expires'] - self['cookies'] = {} - for cookie in jar: - self['cookies'][cookie.name] = { - attname: getattr(cookie, attname) - for attname in stored_attrs - } + self.cookie_jar = jar + + def remove_cookies(self, cookies: List[Dict[str, str]]): + for cookie in cookies: + remove_cookie_by_name( + self.cookie_jar, + cookie['name'], + domain=cookie.get('domain', None), + path=cookie.get('path', None) + ) @property def auth(self) -> Optional[AuthBase]: @@ -155,7 +254,6 @@ def auth(self, auth: dict): assert {'type', 'raw_auth'} == auth.keys() self['auth'] = auth - def remove_cookies(self, names: Iterable[str]): - for name in names: - if name in self['cookies']: - del self['cookies'][name] + @property + def is_anonymous(self): + return is_anonymous_session(self.session_id) diff --git a/httpie/ssl_.py b/httpie/ssl_.py index cdec18f87b..b9438543eb 100644 --- a/httpie/ssl_.py +++ b/httpie/ssl_.py @@ -1,4 +1,5 @@ import ssl +from typing import NamedTuple, Optional from httpie.adapters import HTTPAdapter # noinspection PyPackageRequirements @@ -24,6 +25,17 @@ } +class HTTPieCertificate(NamedTuple): + cert_file: Optional[str] = None + key_file: Optional[str] = None + key_password: Optional[str] = None + + def to_raw_cert(self): + """Synthesize a requests-compatible (2-item tuple of cert and key file) + object from HTTPie's internal representation of a certificate.""" + return (self.cert_file, self.key_file) + + class HTTPieHTTPSAdapter(HTTPAdapter): def __init__( self, @@ -47,6 +59,13 @@ def proxy_manager_for(self, *args, **kwargs): kwargs['ssl_context'] = self._ssl_context return super().proxy_manager_for(*args, **kwargs) + def cert_verify(self, conn, url, verify, cert): + if isinstance(cert, HTTPieCertificate): + conn.key_password = cert.key_password + cert = cert.to_raw_cert() + + return super().cert_verify(conn, url, verify, cert) + @staticmethod def _create_ssl_context( verify: bool, @@ -61,3 +80,17 @@ def _create_ssl_context( # in `super().cert_verify()`. cert_reqs=ssl.CERT_REQUIRED if verify else ssl.CERT_NONE ) + + +def _is_key_file_encrypted(key_file): + """Detects if a key file is encrypted or not. + + Copy of the internal urllib function (urllib3.util.ssl_)""" + + with open(key_file, "r") as f: + for line in f: + # Look for Proc-Type: 4,ENCRYPTED + if "ENCRYPTED" in line: + return True + + return False diff --git a/httpie/uploads.py b/httpie/uploads.py index 4fdb79222d..c9a763df78 100644 --- a/httpie/uploads.py +++ b/httpie/uploads.py @@ -2,6 +2,8 @@ import os import zlib import functools +import time +import threading from typing import Any, Callable, IO, Iterable, Optional, Tuple, Union, TYPE_CHECKING from urllib.parse import urlencode @@ -22,12 +24,20 @@ def __iter__(self) -> Iterable[Union[str, bytes]]: class ChunkedUploadStream(ChunkedStream): - def __init__(self, stream: Iterable, callback: Callable): + def __init__( + self, + stream: Iterable, + callback: Callable, + event: Optional[threading.Event] = None + ) -> None: self.callback = callback self.stream = stream + self.event = event def __iter__(self) -> Iterable[Union[str, bytes]]: for chunk in self.stream: + if self.event: + self.event.set() self.callback(chunk) yield chunk @@ -35,12 +45,19 @@ def __iter__(self) -> Iterable[Union[str, bytes]]: class ChunkedMultipartUploadStream(ChunkedStream): chunk_size = 100 * 1024 - def __init__(self, encoder: 'MultipartEncoder'): + def __init__( + self, + encoder: 'MultipartEncoder', + event: Optional[threading.Event] = None + ) -> None: self.encoder = encoder + self.event = event def __iter__(self) -> Iterable[Union[str, bytes]]: while True: chunk = self.encoder.read(self.chunk_size) + if self.event: + self.event.set() if not chunk: break yield chunk @@ -77,10 +94,10 @@ def is_stdin(file: IO) -> bool: return file_no == sys.stdin.fileno() -READ_THRESHOLD = float(os.getenv("HTTPIE_STDIN_READ_WARN_THRESHOLD", 10.0)) +READ_THRESHOLD = float(os.getenv('HTTPIE_STDIN_READ_WARN_THRESHOLD', 10.0)) -def observe_stdin_for_data_thread(env: Environment, file: IO) -> None: +def observe_stdin_for_data_thread(env: Environment, file: IO, read_event: threading.Event) -> None: # Windows unfortunately does not support select() operation # on regular files, like stdin in our use case. # https://docs.python.org/3/library/select.html#select.select @@ -92,12 +109,9 @@ def observe_stdin_for_data_thread(env: Environment, file: IO) -> None: if READ_THRESHOLD == 0: return None - import select - import threading - - def worker(): - can_read, _, _ = select.select([file], [], [], READ_THRESHOLD) - if not can_read: + def worker(event: threading.Event) -> None: + time.sleep(READ_THRESHOLD) + if not event.is_set(): env.stderr.write( f'> warning: no stdin data read in {READ_THRESHOLD}s ' f'(perhaps you want to --ignore-stdin)\n' @@ -105,11 +119,28 @@ def worker(): ) thread = threading.Thread( - target=worker + target=worker, + args=(read_event,) ) thread.start() +def _read_file_with_selectors(file: IO, read_event: threading.Event) -> bytes: + if is_windows or not is_stdin(file): + return as_bytes(file.read()) + + import select + + # Try checking whether there is any incoming data for READ_THRESHOLD + # seconds. If there isn't anything in the given period, issue + # a warning about a misusage. + read_selectors, _, _ = select.select([file], [], [], READ_THRESHOLD) + if read_selectors: + read_event.set() + + return as_bytes(file.read()) + + def _prepare_file_for_upload( env: Environment, file: Union[IO, 'MultipartEncoder'], @@ -117,9 +148,11 @@ def _prepare_file_for_upload( chunked: bool = False, content_length_header_value: Optional[int] = None, ) -> Union[bytes, IO, ChunkedStream]: + read_event = threading.Event() if not super_len(file): if is_stdin(file): - observe_stdin_for_data_thread(env, file) + observe_stdin_for_data_thread(env, file, read_event) + # Zero-length -> assume stdin. if content_length_header_value is None and not chunked: # Read the whole stdin to determine `Content-Length`. @@ -129,7 +162,7 @@ def _prepare_file_for_upload( # something like --no-chunked. # This would be backwards-incompatible so wait until v3.0.0. # - file = as_bytes(file.read()) + file = _read_file_with_selectors(file, read_event) else: file.read = _wrap_function_with_callback( file.read, @@ -141,11 +174,13 @@ def _prepare_file_for_upload( if isinstance(file, MultipartEncoder): return ChunkedMultipartUploadStream( encoder=file, + event=read_event, ) else: return ChunkedUploadStream( stream=file, callback=callback, + event=read_event ) else: return file diff --git a/httpie/utils.py b/httpie/utils.py index fa19fa7cde..4fffb2826e 100644 --- a/httpie/utils.py +++ b/httpie/utils.py @@ -9,6 +9,7 @@ from http.cookiejar import parse_ns_headers from pathlib import Path from pprint import pformat +from urllib.parse import urlsplit from typing import Any, List, Optional, Tuple, Callable, Iterable, TypeVar import requests.auth @@ -237,3 +238,7 @@ def unwrap_context(exc: Exception) -> Optional[Exception]: return unwrap_context(context) else: return exc + + +def url_as_host(url: str) -> str: + return urlsplit(url).netloc.split('@')[-1] diff --git a/pytest.ini b/pytest.ini index 7c6209701b..ced65979b1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,7 @@ [pytest] markers = + # If you want to run tests without a full HTTPie installation + # we advise you to disable the markers below, e.g: + # pytest -m 'not requires_installation and not requires_external_processes' requires_installation + requires_external_processes diff --git a/setup.py b/setup.py index 5316ff73d3..8f9a93140f 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ tests_require = [ 'pytest', 'pytest-httpbin>=0.0.6', + 'pytest-lazy-fixture>=0.0.6', 'responses', ] dev_require = [ diff --git a/tests/client_certs/password_protected/client.key b/tests/client_certs/password_protected/client.key new file mode 100644 index 0000000000..1634352f90 --- /dev/null +++ b/tests/client_certs/password_protected/client.key @@ -0,0 +1,42 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,93DA845817852FB576163AA829C232E9 + +VauvxiyC0lQbLJFoEGlnIIZO2/+b66DjTwSccqSdVg+Zaxvbc0jeVhS43SQ01ft0 +hB/oISgJB/1I/oKbGwx07T9j78Q8G9AxQV6hzvozK5Etjew4RrvV4DYyOSzwZNQr +qB9S0qhBKyemA2vx4aH/8nazHh+zrRD3y0oMbuCHLxSGuqncNXIKCTYgMb8NUucJ +fEArYHijZ0iotoOEpP31JOUPCpKhEewQxzNK0HLws0lv6nl6fmBlkdi603qmsG5U +uinuiGodrh9SpCUc/A4OhVWKwoiQSxGnz+SiNaXyUByf9CR8RLPWqi5pTGHC8xrJ +uHI6Cw8ZfrJ2clYtuCWv6g6c4F7sz6eAJHqCZNnU32kKu3uH/9E/7Z8uH7JOVyFa +9DlBHCWHdyaHs8mY+/pDcxeMyWeC837sBelIBF1iEwU/sMw43HipZBNhrekMLAkx +y5HRYQDstTvk1Nvj8fKysYuhGCiF/V6PWYo5RaQszZLhS+uyFEBwa0ojYNZh4LyB +5uIdBaqtL9FD4RXqTYfN96eEyoYaUUY5KXqQMZkuZpotGYmH9OGMTVCgR7eU0a62 +dgbQw4UCQd4YTNx1PyboH72oIi+Rqp2LEYEQSHP/dIUtBiA/kmWhgapZVGvfJ+fF +u9MPgPUDvH3oLVm4Mr+biLX/oUQVEup85q8++E2csDe2HoC4JdmJ0D9rZM2OqpYV +YZAPcPhx2pYnK5d6RvMFwtLPNfHxgYQXMVg6BFtu5GCxxqr+dhF7TGrN5s6AKC8U +bkVQIXwO8bYVTLj2Sb44fe+Xl1X/09yHnkZC0u/Kb2KvUm7Gnltn3tUmj7fGI0I6 +aI6G3T1xc0jz9WhjdnM3uDYYI66GpgRgv81n7IkfRjclNArW4OStf30K4pXXjGeP +vgopPJ1yNpaM4QNbx3cqzP0eBy+Ss7aCXca4I3BzjXtuo9ZcEzGb+1FkS7ASEdex +cAroJOmm9KJ+3KOxsVs5fxXtQqzzeD8cdZeGV0eckJNfjWSBH2zyhaxwdlCvG1I9 +dTvdd6q31FjlnUq9SvGEkfoy4myIUtt4DJQ4lSktvKQv9qepUjoX0k3xipgSmiPO +yxE+VdJdJ9/tDUf3psD01XLIss7hOX9aED3svN3uXB2ZVCSH6e2l4IrBMirdKNwR +fB4Yrul0qt9knmn11p2aWav055hb1Il5Tm8/WnaXkgtr20zP4RgR7P19mSjTBxUm +7iUIiWqU43Sx2LWsYpg7Lbj5XGLcvxv5WjYsE4Km0ltZCLKzMHfQ76qv4ZOQkHcR +9UevRmzU45095eASztedrYyxDNwU6YSdUcOYTP6383G9azbStlQY+w2Em++UoNoH +3eYj6KHKx+hkZOdc8PLaLg2f98jOiADpKYJTGnkKoLjTCfr9nzBeNxwRCQ4F4vO/ ++tuRo3i1ODpJQbbZys9Mz+9PSwBH31UAib0+v0GYLDJN2rJcyGal/0DH5zON9Ogi +5bZQ9oS91p9K5hUAnHpd3zOzeX1lCoZnmtOI8wah79SVSpK1xoE6BAxAHfRiYiS3 +1tDmkThJBOGXmkpLjtgNW3MqYKBnO3tRzrDDCjTKi5jFX/SD2FPpExOyA2+I0lrr +a9b+Sjbl1Z7B1yZmmTGMKB7prwK00LaF6yqKOhE+bx1yJAaWrbdPCD6vDmbq5YV6 +87woIiA16Q2I1x77/Kg3TDO9LMDiwI5BFyjR+4Q5SvufIaxtsmTBuaBuPif+f4DT +MPQcfk5ozQIKY4qiSqMAOXAf2t+/UQROjgYvayRz0fOv2rV0vS4i9ELj/8Dn65Dq +7aQzLwM0psToGIVyzAV+hF3jeQP+Xu7VjtSxTJ+ajz7PeIXeBH/mwJKMk7hpRwGj +4fZ92S00Iat2kA6wn55u6EGewgcaQrN2zr75a9gvXQwMDmsjszq2uWWxxJg6pAPZ +rNqhM9tJ2UAJ1lLZzUDfhK4wU4pGWIhT+BmdDgJ40hI4b1WEmKSTxsj8AYNcVDRf +i2Ox1QhZQX9bH5kTOX373/6cALFR5DcU8qh2FJtf+3uiZHNloEeID//H2Gdoxz0Y +5CC/VDiIa4Gj4D+ATsLMgTDt4eUOinMeC1H6w+QBd9UvceqEvrgu+1WB8UCK/Hm/ +7fZ0srsGg/WRqdSuO8/7998PEHgP8+wnTbxi9Y3EEbkaKUL6esJfeOjBibuGPyaf +2Y9QLcpVKaD7pmVeb97qExZZjEiID6QYmFUO8j0koS2fki0l+z8XEZ3JLZKa9XS+ +uiMPQKg41j+9ZrGmwPNj7brjwA0cdSb4CLgxg4FwuwB660XaXpW3aRsiRryi0YcM +hn2l6b4JgBz8gUkFiTXQ8wRvAKDC1hUkUysqCAC+Yg3cWxlDZVeSeqVGr5jhHgN1 +-----END RSA PRIVATE KEY----- diff --git a/tests/client_certs/password_protected/client.pem b/tests/client_certs/password_protected/client.pem new file mode 100644 index 0000000000..08ee652254 --- /dev/null +++ b/tests/client_certs/password_protected/client.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEazCCAtOgAwIBAgIUIWojJySwCenyvcJrZY1aKOMQfTUwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAzMDExMzM5MzRaFw00OTA3 +MTcxMzM5MzRaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggGiMA0GCSqGSIb3DQEB +AQUAA4IBjwAwggGKAoIBgQC0+8GOIhknLgLmWEQGgdDJffntbDfGdtG7JFUFfvuN +sibTHL4RPNhe+UrT/LR+JBfIfmdeb+NlmKzsTeR0+8PkX5ZjXMShf5icghVukK7G +OoQS7olQqlGzpIX76VqktVr4tFNXmMJeBO0NIaXHm0ASsoz3fIfDx9ttJETPs6lm +Wv/PUPemvtUgcbAb+kjz9QqcUV8B1xcCvQma6NSpxcmJHqAuI6HkdbDzyedKuuSi +M6yNFjh3EJjsufReQgkcfDsqh+RA3zQoIyPXLNqjzGD71z24jUtvIxb5ZNEtv0xp +5zCOCavuRNNyKGvvnlIeyup7bMe0QIds566miG49osVpPVvVmg+q+w2YYAE+7svb +nJp7NYn2tryRqsmvnASLVQD6T9wTWUa8w/tT1+ltnhfqbwDcVACzsw/U4FFwcfWw +5BnUcJacoDkj/3TCqgkA8XFe1/DVU8XCcsvEaoLzwHhHu2+QDpqal8rNouyTpFGA +/wioVBQGpksPZjl8lumsz3kCAwEAAaNTMFEwHQYDVR0OBBYEFGJhl1BPOXCVqRo3 +U/ruuedvlDqsMB8GA1UdIwQYMBaAFGJhl1BPOXCVqRo3U/ruuedvlDqsMA8GA1Ud +EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggGBAE9NtrE5qSHlK9DXdH6bPW6z +GO9uWBl3rJVtqVvPoH8RxJG/jtaD/Pnc3MkIxMoliCleNK6BdYVGV9u8x9W1jQo8 +H+mnH3/Ise8ZY1zpblZJF0z9xs5sWW7qO8U06GmJWRSPn3LKEZjLsNmThhUW09wN +8EZX914zCWtzCrUTNg8Au1Dz9zA9ScfpCVPhKORTCnrpoTL6iXsPxmCx+5awmNLE +uh9kw4NScEyq33RTPosMpwSMlXGRuASltx/J7Rn0DNR0r1p0XzDS4CG1iDwXHlEF +MwsOvSahNyz5RInrU3cgN70tafoRIHScLYycnRml8dydxrDoFgdJk5sI4zgq24Sg +TktTq9ShrT4yQX+lrGS6eZQK/YZEBPD7BdTLYp3vlfYQMJ4Jz9SyQ8b9/9jIFVFS +dFfWiCqEuhTvGfptAzYX+K9OaegZnIk3X7R6O+YQ3oHCbLbnV3bpKlgNnOKBwa2X +kJ5GRp+rZOJ97yjrspKjpR5tNCiJnp7NnnA5VA6mfw== +-----END CERTIFICATE----- diff --git a/tests/conftest.py b/tests/conftest.py index 5e8c511072..7ca0e60440 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,11 @@ import pytest from pytest_httpbin import certs -from .utils import HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, HTTPBIN_WITH_CHUNKED_SUPPORT +from .utils import ( # noqa + HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, + HTTPBIN_WITH_CHUNKED_SUPPORT, + mock_env +) from .utils.plugins_cli import ( # noqa broken_plugin, dummy_plugin, diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 126b13276e..6e6e73676e 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -1,6 +1,9 @@ """Test data""" +import json from pathlib import Path +from typing import Optional, Dict, Any +import httpie from httpie.encoding import UTF8 from httpie.output.formatters.xml import pretty_xml, parse_xml @@ -19,10 +22,20 @@ def patharg(path): JSON_FILE_PATH = FIXTURES_ROOT / 'test.json' JSON_WITH_DUPE_KEYS_FILE_PATH = FIXTURES_ROOT / 'test_with_dupe_keys.json' BIN_FILE_PATH = FIXTURES_ROOT / 'test.bin' + XML_FILES_PATH = FIXTURES_ROOT / 'xmldata' XML_FILES_VALID = list((XML_FILES_PATH / 'valid').glob('*_raw.xml')) XML_FILES_INVALID = list((XML_FILES_PATH / 'invalid').glob('*.xml')) +SESSION_FILES_PATH = FIXTURES_ROOT / 'session_data' +SESSION_FILES_OLD = sorted((SESSION_FILES_PATH / 'old').glob('*.json')) +SESSION_FILES_NEW = sorted((SESSION_FILES_PATH / 'new').glob('*.json')) + +SESSION_VARIABLES = { + '__version__': httpie.__version__, + '__host__': 'null', +} + FILE_PATH_ARG = patharg(FILE_PATH) BIN_FILE_PATH_ARG = patharg(BIN_FILE_PATH) JSON_FILE_PATH_ARG = patharg(JSON_FILE_PATH) @@ -40,3 +53,14 @@ def patharg(path): UNICODE = FILE_CONTENT XML_DATA_RAW = 'text' XML_DATA_FORMATTED = pretty_xml(parse_xml(XML_DATA_RAW)) + + +def read_session_file(session_file: Path, *, extra_variables: Optional[Dict[str, str]] = None) -> Any: + with open(session_file) as stream: + data = stream.read() + + session_vars = {**SESSION_VARIABLES, **(extra_variables or {})} + for variable, value in session_vars.items(): + data = data.replace(variable, value) + + return json.loads(data) diff --git a/tests/fixtures/session_data/new/cookies_dict.json b/tests/fixtures/session_data/new/cookies_dict.json new file mode 100644 index 0000000000..8a4d5f2e13 --- /dev/null +++ b/tests/fixtures/session_data/new/cookies_dict.json @@ -0,0 +1,31 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [ + { + "domain": __host__, + "expires": null, + "name": "baz", + "path": "/", + "secure": false, + "value": "quux" + }, + { + "domain": __host__, + "expires": null, + "name": "foo", + "path": "/", + "secure": false, + "value": "bar" + } + ], + "headers": {} +} diff --git a/tests/fixtures/session_data/new/cookies_dict_dev_version.json b/tests/fixtures/session_data/new/cookies_dict_dev_version.json new file mode 100644 index 0000000000..8a4d5f2e13 --- /dev/null +++ b/tests/fixtures/session_data/new/cookies_dict_dev_version.json @@ -0,0 +1,31 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [ + { + "domain": __host__, + "expires": null, + "name": "baz", + "path": "/", + "secure": false, + "value": "quux" + }, + { + "domain": __host__, + "expires": null, + "name": "foo", + "path": "/", + "secure": false, + "value": "bar" + } + ], + "headers": {} +} diff --git a/tests/fixtures/session_data/new/cookies_dict_with_extras.json b/tests/fixtures/session_data/new/cookies_dict_with_extras.json new file mode 100644 index 0000000000..9a99f15268 --- /dev/null +++ b/tests/fixtures/session_data/new/cookies_dict_with_extras.json @@ -0,0 +1,33 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "raw_auth": "foo:bar", + "type": "basic" + }, + "cookies": [ + { + "domain": __host__, + "expires": null, + "name": "baz", + "path": "/", + "secure": false, + "value": "quux" + }, + { + "domain": __host__, + "expires": null, + "name": "foo", + "path": "/", + "secure": false, + "value": "bar" + } + ], + "headers": { + "X-Data": "value", + "X-Foo": "bar" + } +} diff --git a/tests/fixtures/session_data/new/empty_cookies_dict.json b/tests/fixtures/session_data/new/empty_cookies_dict.json new file mode 100644 index 0000000000..1d01661a06 --- /dev/null +++ b/tests/fixtures/session_data/new/empty_cookies_dict.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [], + "headers": {} +} diff --git a/tests/fixtures/session_data/new/empty_cookies_list.json b/tests/fixtures/session_data/new/empty_cookies_list.json new file mode 100644 index 0000000000..1d01661a06 --- /dev/null +++ b/tests/fixtures/session_data/new/empty_cookies_list.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [], + "headers": {} +} diff --git a/tests/fixtures/session_data/old/cookies_dict.json b/tests/fixtures/session_data/old/cookies_dict.json new file mode 100644 index 0000000000..9c4fd21476 --- /dev/null +++ b/tests/fixtures/session_data/old/cookies_dict.json @@ -0,0 +1,27 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": { + "baz": { + "expires": null, + "path": "/", + "secure": false, + "value": "quux" + }, + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + }, + "headers": {} +} diff --git a/tests/fixtures/session_data/old/cookies_dict_dev_version.json b/tests/fixtures/session_data/old/cookies_dict_dev_version.json new file mode 100644 index 0000000000..935b43f083 --- /dev/null +++ b/tests/fixtures/session_data/old/cookies_dict_dev_version.json @@ -0,0 +1,27 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "2.7.0.dev0" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": { + "baz": { + "expires": null, + "path": "/", + "secure": false, + "value": "quux" + }, + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + }, + "headers": {} +} diff --git a/tests/fixtures/session_data/old/cookies_dict_with_extras.json b/tests/fixtures/session_data/old/cookies_dict_with_extras.json new file mode 100644 index 0000000000..42968e52a9 --- /dev/null +++ b/tests/fixtures/session_data/old/cookies_dict_with_extras.json @@ -0,0 +1,29 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "raw_auth": "foo:bar", + "type": "basic" + }, + "cookies": { + "baz": { + "expires": null, + "path": "/", + "secure": false, + "value": "quux" + }, + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + }, + "headers": { + "X-Data": "value", + "X-Foo": "bar" + } +} diff --git a/tests/fixtures/session_data/old/empty_cookies_dict.json b/tests/fixtures/session_data/old/empty_cookies_dict.json new file mode 100644 index 0000000000..8de1a9217c --- /dev/null +++ b/tests/fixtures/session_data/old/empty_cookies_dict.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": {}, + "headers": {} +} diff --git a/tests/fixtures/session_data/old/empty_cookies_list.json b/tests/fixtures/session_data/old/empty_cookies_list.json new file mode 100644 index 0000000000..12194f7ed2 --- /dev/null +++ b/tests/fixtures/session_data/old/empty_cookies_list.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [], + "headers": {} +} diff --git a/tests/test_cli_ui.py b/tests/test_cli_ui.py new file mode 100644 index 0000000000..760077a47f --- /dev/null +++ b/tests/test_cli_ui.py @@ -0,0 +1,81 @@ +import pytest +import shutil +import os +import sys +from tests.utils import http + + +if sys.version_info >= (3, 9): + REQUEST_ITEM_MSG = "[REQUEST_ITEM ...]" +else: + REQUEST_ITEM_MSG = "[REQUEST_ITEM [REQUEST_ITEM ...]]" + + +NAKED_HELP_MESSAGE = f"""\ +usage: + http [METHOD] URL {REQUEST_ITEM_MSG} + +error: + the following arguments are required: URL + +for more information: + run 'http --help' or visit https://httpie.io/docs/cli + +""" + +NAKED_HELP_MESSAGE_PRETTY_WITH_NO_ARG = f"""\ +usage: + http [--pretty {{all,colors,format,none}}] [METHOD] URL {REQUEST_ITEM_MSG} + +error: + argument --pretty: expected one argument + +for more information: + run 'http --help' or visit https://httpie.io/docs/cli + +""" + +NAKED_HELP_MESSAGE_PRETTY_WITH_INVALID_ARG = f"""\ +usage: + http [--pretty {{all,colors,format,none}}] [METHOD] URL {REQUEST_ITEM_MSG} + +error: + argument --pretty: invalid choice: '$invalid' (choose from 'all', 'colors', 'format', 'none') + +for more information: + run 'http --help' or visit https://httpie.io/docs/cli + +""" + + +PREDEFINED_TERMINAL_SIZE = (160, 80) + + +@pytest.fixture(scope="function") +def ignore_terminal_size(monkeypatch): + """Some tests wrap/crop the output depending on the + size of the executed terminal, which might not be consistent + through all runs. + + This fixture ensures every run uses the same exact configuration. + """ + + def fake_terminal_size(*args, **kwargs): + return os.terminal_size(PREDEFINED_TERMINAL_SIZE) + + # Setting COLUMNS as an env var is required for 3.8< + monkeypatch.setitem(os.environ, 'COLUMNS', str(PREDEFINED_TERMINAL_SIZE[0])) + monkeypatch.setattr(shutil, 'get_terminal_size', fake_terminal_size) + + +@pytest.mark.parametrize( + 'args, expected_msg', [ + ([], NAKED_HELP_MESSAGE), + (['--pretty'], NAKED_HELP_MESSAGE_PRETTY_WITH_NO_ARG), + (['pie.dev', '--pretty'], NAKED_HELP_MESSAGE_PRETTY_WITH_NO_ARG), + (['--pretty', '$invalid'], NAKED_HELP_MESSAGE_PRETTY_WITH_INVALID_ARG), + ] +) +def test_naked_invocation(ignore_terminal_size, args, expected_msg): + result = http(*args, tolerate_error_exit_status=True) + assert result.stderr == expected_msg diff --git a/tests/test_cookie_on_redirects.py b/tests/test_cookie_on_redirects.py new file mode 100644 index 0000000000..e22f833048 --- /dev/null +++ b/tests/test_cookie_on_redirects.py @@ -0,0 +1,262 @@ +import pytest +from .utils import http + + +@pytest.fixture +def remote_httpbin(httpbin_with_chunked_support): + return httpbin_with_chunked_support + + +def _stringify(fixture): + return fixture + '' + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +def test_explicit_user_set_cookie(httpbin, instance): + # User set cookies ARE NOT persisted within redirects + # when there is no session, even on the same domain. + + r = http( + '--follow', + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/cookies', + 'Cookie:a=b' + ) + assert r.json == {'cookies': {}} + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +def test_explicit_user_set_cookie_in_session(tmp_path, httpbin, instance): + # User set cookies ARE persisted within redirects + # when there is A session, even on the same domain. + + r = http( + '--follow', + '--session', + str(tmp_path / 'session.json'), + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/cookies', + 'Cookie:a=b' + ) + assert r.json == {'cookies': {'a': 'b'}} + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +def test_saved_user_set_cookie_in_session(tmp_path, httpbin, instance): + # User set cookies ARE persisted within redirects + # when there is A session, even on the same domain. + + http( + '--follow', + '--session', + str(tmp_path / 'session.json'), + httpbin + '/get', + 'Cookie:a=b' + ) + r = http( + '--follow', + '--session', + str(tmp_path / 'session.json'), + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/cookies', + ) + assert r.json == {'cookies': {'a': 'b'}} + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +@pytest.mark.parametrize('session', [True, False]) +def test_explicit_user_set_headers(httpbin, tmp_path, instance, session): + # User set headers ARE persisted within redirects + # even on different domains domain with or without + # an active session. + session_args = [] + if session: + session_args.extend([ + '--session', + str(tmp_path / 'session.json') + ]) + + r = http( + '--follow', + *session_args, + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/get', + 'X-Custom-Header:value' + ) + assert 'X-Custom-Header' in r.json['headers'] + + +@pytest.mark.parametrize('session', [True, False]) +def test_server_set_cookie_on_redirect_same_domain(tmp_path, httpbin, session): + # Server set cookies ARE persisted on the same domain + # when they are forwarded. + + session_args = [] + if session: + session_args.extend([ + '--session', + str(tmp_path / 'session.json') + ]) + + r = http( + '--follow', + *session_args, + httpbin + '/cookies/set/a/b', + ) + assert r.json['cookies'] == {'a': 'b'} + + +@pytest.mark.parametrize('session', [True, False]) +def test_server_set_cookie_on_redirect_different_domain(tmp_path, http_server, httpbin, session): + # Server set cookies ARE persisted on different domains + # when they are forwarded. + + session_args = [] + if session: + session_args.extend([ + '--session', + str(tmp_path / 'session.json') + ]) + + r = http( + '--follow', + *session_args, + http_server + '/cookies/set-and-redirect', + f"X-Redirect-To:{httpbin + '/cookies'}", + 'X-Cookies:a=b' + ) + assert r.json['cookies'] == {'a': 'b'} + + +def test_saved_session_cookies_on_same_domain(tmp_path, httpbin): + # Saved session cookies ARE persisted when making a new + # request to the same domain. + http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies/set/a/b' + ) + r = http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies' + ) + assert r.json == {'cookies': {'a': 'b'}} + + +def test_saved_session_cookies_on_different_domain(tmp_path, httpbin, remote_httpbin): + # Saved session cookies ARE persisted when making a new + # request to a different domain. + http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies/set/a/b' + ) + r = http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies' + ) + assert r.json == {'cookies': {}} + + +@pytest.mark.parametrize('initial_domain, first_request_domain, second_request_domain, expect_cookies', [ + ( + # Cookies are set by Domain A + # Initial domain is Domain A + # Redirected domain is Domain A + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('httpbin'), + True, + ), + ( + # Cookies are set by Domain A + # Initial domain is Domain B + # Redirected domain is Domain B + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), + pytest.lazy_fixture('remote_httpbin'), + False, + ), + ( + # Cookies are set by Domain A + # Initial domain is Domain A + # Redirected domain is Domain B + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), + False, + ), + ( + # Cookies are set by Domain A + # Initial domain is Domain B + # Redirected domain is Domain A + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), + pytest.lazy_fixture('httpbin'), + True, + ), +]) +def test_saved_session_cookies_on_redirect(tmp_path, initial_domain, first_request_domain, second_request_domain, expect_cookies): + http( + '--session', + str(tmp_path / 'session.json'), + initial_domain + '/cookies/set/a/b' + ) + r = http( + '--session', + str(tmp_path / 'session.json'), + '--follow', + first_request_domain + '/redirect-to', + f'url=={_stringify(second_request_domain)}/cookies' + ) + if expect_cookies: + expected_data = {'cookies': {'a': 'b'}} + else: + expected_data = {'cookies': {}} + assert r.json == expected_data + + +def test_saved_session_cookie_pool(tmp_path, httpbin, remote_httpbin): + http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies/set/a/b' + ) + http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies/set/a/c' + ) + http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies/set/b/d' + ) + + response = http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies' + ) + assert response.json['cookies'] == {'a': 'b'} + + response = http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies' + ) + assert response.json['cookies'] == {'a': 'c', 'b': 'd'} diff --git a/tests/test_httpie_cli.py b/tests/test_httpie_cli.py index 377f6e1651..9da7163922 100644 --- a/tests/test_httpie_cli.py +++ b/tests/test_httpie_cli.py @@ -1,8 +1,14 @@ import pytest +import shutil import json +from httpie.sessions import SESSIONS_DIR_NAME from httpie.status import ExitStatus from httpie.cli.options import PARSER_SPEC_VERSION -from tests.utils import httpie +from tests.utils import DUMMY_HOST, httpie +from tests.fixtures import SESSION_FILES_PATH, SESSION_FILES_NEW, SESSION_FILES_OLD, read_session_file + + +OLD_SESSION_FILES_PATH = SESSION_FILES_PATH / 'old' @pytest.mark.requires_installation @@ -16,11 +22,12 @@ def test_plugins_cli_error_message_without_args(): @pytest.mark.parametrize( - 'example', [ + 'example', + [ 'pie.dev/get', 'DELETE localhost:8000/delete', - 'POST pie.dev/post header:value a=b header_2:value x:=1' - ] + 'POST pie.dev/post header:value a=b header_2:value x:=1', + ], ) @pytest.mark.requires_installation def test_plugins_cli_error_messages_with_example(example): @@ -32,15 +39,15 @@ def test_plugins_cli_error_messages_with_example(example): @pytest.mark.parametrize( - 'example', [ + 'example', + [ 'cli', 'plugins', 'cli foo', - 'cli export --format=foo', 'plugins unknown', 'plugins unknown.com A:B c=d', 'unknown.com UNPARSABLE????SYNTAX', - ] + ], ) @pytest.mark.requires_installation def test_plugins_cli_error_messages_invalid_example(example): @@ -51,6 +58,74 @@ def test_plugins_cli_error_messages_invalid_example(example): assert f'https {example}' not in result.stderr +HTTPIE_CLI_SESSIONS_UPGRADE_OPTIONS = [ + ( + # Default settings + [], + {'__host__': json.dumps(None)}, + ), + ( + # When --bind-cookies is applied, the __host__ becomes DUMMY_URL. + ['--bind-cookies'], + {'__host__': json.dumps(DUMMY_HOST)}, + ), +] + + +@pytest.mark.parametrize( + 'old_session_file, new_session_file', zip(SESSION_FILES_OLD, SESSION_FILES_NEW) +) +@pytest.mark.parametrize( + 'extra_args, extra_variables', + HTTPIE_CLI_SESSIONS_UPGRADE_OPTIONS, +) +def test_httpie_sessions_upgrade(tmp_path, old_session_file, new_session_file, extra_args, extra_variables): + session_path = tmp_path / 'session.json' + shutil.copyfile(old_session_file, session_path) + + result = httpie( + 'cli', 'sessions', 'upgrade', *extra_args, DUMMY_HOST, str(session_path) + ) + assert result.exit_status == ExitStatus.SUCCESS + assert read_session_file(session_path) == read_session_file( + new_session_file, extra_variables=extra_variables + ) + + +def test_httpie_sessions_upgrade_on_non_existent_file(tmp_path): + session_path = tmp_path / 'session.json' + result = httpie('cli', 'sessions', 'upgrade', DUMMY_HOST, str(session_path)) + assert result.exit_status == ExitStatus.ERROR + assert 'does not exist' in result.stderr + + +@pytest.mark.parametrize( + 'extra_args, extra_variables', + HTTPIE_CLI_SESSIONS_UPGRADE_OPTIONS, +) +def test_httpie_sessions_upgrade_all(tmp_path, mock_env, extra_args, extra_variables): + mock_env._create_temp_config_dir = False + mock_env.config_dir = tmp_path / "config" + + session_dir = mock_env.config_dir / SESSIONS_DIR_NAME / DUMMY_HOST + session_dir.mkdir(parents=True) + for original_session_file in SESSION_FILES_OLD: + shutil.copy(original_session_file, session_dir) + + result = httpie( + 'cli', 'sessions', 'upgrade-all', *extra_args, env=mock_env + ) + assert result.exit_status == ExitStatus.SUCCESS + + for refactored_session_file, expected_session_file in zip( + sorted(session_dir.glob("*.json")), + SESSION_FILES_NEW + ): + assert read_session_file(refactored_session_file) == read_session_file( + expected_session_file, extra_variables=extra_variables + ) + + @pytest.mark.parametrize( 'load_func, extra_options', [ (json.loads, []), @@ -58,6 +133,6 @@ def test_plugins_cli_error_messages_invalid_example(example): ] ) def test_cli_export(load_func, extra_options): - response = httpie('cli', 'export', *extra_options) + response = httpie('cli', 'export-args', *extra_options) assert response.exit_status == ExitStatus.SUCCESS assert load_func(response)['version'] == PARSER_SPEC_VERSION diff --git a/tests/test_json.py b/tests/test_json.py index b454c03408..2ba603a680 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -321,7 +321,7 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value): 'foo[][key]=value', 'foo[2][key 2]=value 2', r'foo[2][key \[]=value 3', - r'[nesting][under][!][empty][?][\\key]:=4', + r'bar[nesting][under][!][empty][?][\\key]:=4', ], { 'foo': [ @@ -329,7 +329,7 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value): 2, {'key': 'value', 'key 2': 'value 2', 'key [': 'value 3'}, ], - '': { + 'bar': { 'nesting': {'under': {'!': {'empty': {'?': {'\\key': 4}}}}} }, }, @@ -397,6 +397,58 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value): '2012': {'x': 2, '[3]': 4}, }, ), + ( + [ + r'a[\0]:=0', + r'a[\\1]:=1', + r'a[\\\2]:=2', + r'a[\\\\\3]:=3', + r'a[-1\\]:=-1', + r'a[-2\\\\]:=-2', + r'a[\\-3\\\\]:=-3', + ], + { + 'a': { + '0': 0, + r'\1': 1, + r'\\2': 2, + r'\\\3': 3, + '-1\\': -1, + '-2\\\\': -2, + '\\-3\\\\': -3, + } + }, + ), + ( + ['[]:=0', '[]:=1', '[5]:=5', '[]:=6', '[9]:=9'], + [0, 1, None, None, None, 5, 6, None, None, 9], + ), + ( + ['=empty', 'foo=bar', 'bar[baz][quux]=tuut'], + {'': 'empty', 'foo': 'bar', 'bar': {'baz': {'quux': 'tuut'}}}, + ), + ( + [ + r'\1=top level int', + r'\\1=escaped top level int', + r'\2[\3][\4]:=5', + ], + { + '1': 'top level int', + '\\1': 'escaped top level int', + '2': {'3': {'4': 5}}, + }, + ), + ( + [':={"foo": {"bar": "baz"}}', 'top=val'], + {'': {'foo': {'bar': 'baz'}}, 'top': 'val'}, + ), + ( + ['[][a][b][]:=1', '[0][a][b][]:=2', '[][]:=2'], + [{'a': {'b': [1, 2]}}, [2]], + ), + ([':=[1,2,3]'], {'': [1, 2, 3]}), + ([':=[1,2,3]', 'foo=bar'], {'': [1, 2, 3], 'foo': 'bar'}), ], ) def test_nested_json_syntax(input_json, expected_json, httpbin): @@ -494,13 +546,36 @@ def test_nested_json_syntax(input_json, expected_json, httpbin): ['foo[\\1]:=2', 'foo[5]:=3'], "HTTPie Type Error: Can't perform 'index' based access on 'foo' which has a type of 'object' but this operation requires a type of 'array'.\nfoo[5]\n ^^^", ), + ( + ['x=y', '[]:=2'], + "HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.", + ), + ( + ['[]:=2', 'x=y'], + "HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.", + ), + ( + [':=[1,2,3]', '[]:=4'], + "HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.", + ), + ( + ['[]:=4', ':=[1,2,3]'], + "HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.", + ), ], ) def test_nested_json_errors(input_json, expected_error, httpbin): with pytest.raises(HTTPieSyntaxError) as exc: http(httpbin + '/post', *input_json) - assert str(exc.value) == expected_error + exc_lines = str(exc.value).splitlines() + expected_lines = expected_error.splitlines() + if len(expected_lines) == 1: + # When the error offsets are not important, we'll just compare the actual + # error message. + exc_lines = exc_lines[:1] + + assert expected_lines == exc_lines def test_nested_json_sparse_array(httpbin_both): diff --git a/tests/test_output.py b/tests/test_output.py index c68bfa9e38..470673bf91 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -5,6 +5,7 @@ import json import os import io +import warnings from urllib.request import urlopen import pytest @@ -17,10 +18,15 @@ ) from httpie.cli.definition import parser from httpie.encoding import UTF8 -from httpie.output.formatters.colors import get_lexer, PIE_STYLE_NAMES +from httpie.output.formatters.colors import get_lexer, PIE_STYLE_NAMES, BUNDLED_STYLES from httpie.status import ExitStatus from .fixtures import XML_DATA_RAW, XML_DATA_FORMATTED -from .utils import COLOR, CRLF, HTTP_OK, MockEnvironment, http, DUMMY_URL +from .utils import COLOR, CRLF, HTTP_OK, MockEnvironment, http, DUMMY_URL, strip_colors + + +# For ensuring test reproducibility, avoid using the unsorted +# BUNDLED_STYLES set. +SORTED_BUNDLED_STYLES = sorted(BUNDLED_STYLES) @pytest.mark.parametrize('stdout_isatty', [True, False]) @@ -85,6 +91,31 @@ def test_quiet_quiet_with_check_status_non_zero_pipe(self, httpbin): ) assert 'http: warning: HTTP 500' in r.stderr + @mock.patch('httpie.core.program') + @pytest.mark.parametrize('flags, expected_warnings', [ + ([], 1), + (['-q'], 1), + (['-qq'], 0), + ]) + def test_quiet_on_python_warnings(self, test_patch, httpbin, flags, expected_warnings): + def warn_and_run(*args, **kwargs): + warnings.warn('warning!!') + return ExitStatus.SUCCESS + + test_patch.side_effect = warn_and_run + with pytest.warns(None) as record: + http(*flags, httpbin + '/get') + + assert len(record) == expected_warnings + + def test_double_quiet_on_error(self, httpbin): + r = http( + '-qq', '--check-status', '$$$this.does.not.exist$$$', + tolerate_error_exit_status=True, + ) + assert not r + assert 'Couldn’t resolve the given hostname' in r.stderr + @pytest.mark.parametrize('quiet_flags', QUIET_SCENARIOS) @mock.patch('httpie.cli.argtypes.AuthCredentials._getpass', new=lambda self, prompt: 'password') @@ -234,6 +265,24 @@ def test_ensure_meta_is_colored(httpbin, style): assert COLOR in r +@pytest.mark.parametrize('style', SORTED_BUNDLED_STYLES) +@pytest.mark.parametrize('msg', [ + '', + ' ', + ' OK', + ' OK ', + ' CUSTOM ', +]) +def test_ensure_status_code_is_shown_on_all_themes(http_server, style, msg): + env = MockEnvironment(colors=256) + r = http('--style', style, + http_server + '/status/msg', + '--raw', msg, env=env) + + # Trailing space is stripped away. + assert 'HTTP/1.0 200' + msg.rstrip() in strip_colors(r) + + class TestPrettyOptions: """Test the --pretty handling.""" diff --git a/tests/test_parser_schema.py b/tests/test_parser_schema.py index 366c93189a..3dc7d7b0e5 100644 --- a/tests/test_parser_schema.py +++ b/tests/test_parser_schema.py @@ -4,7 +4,7 @@ def test_parser_serialization(): small_parser = ParserSpec("test_parser") - group_1 = small_parser.new_group("group_1") + group_1 = small_parser.add_group("group_1") group_1.add_argument("regular_arg", help="regular arg") group_1.add_argument( "variadic_arg", @@ -20,7 +20,7 @@ def test_parser_serialization(): help_formatter=lambda state: ", ".join(state), ) - group_2 = small_parser.new_group("group_2") + group_2 = small_parser.add_group("group_2") group_2.add_argument("--typed", action="store_true", type=int) definition = small_parser.finalize() @@ -35,7 +35,7 @@ def test_parser_serialization(): "args": [ { "options": ["regular_arg"], - "description": ["regular arg"], + "description": "regular arg", }, { "options": ["variadic_arg"], @@ -45,7 +45,7 @@ def test_parser_serialization(): }, { "options": ["-O", "--opt-arg"], - "description": ["opt_1, opt_2"], + "description": "opt_1, opt_2", "choices": ["opt_1", "opt_2"], }, ], diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 5835993605..8bcd906327 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -1,12 +1,16 @@ import json import os import shutil +from contextlib import contextmanager from datetime import datetime from unittest import mock +from pathlib import Path +from typing import Iterator import pytest from .fixtures import FILE_PATH_ARG, UNICODE +from httpie.context import Environment from httpie.encoding import UTF8 from httpie.plugins import AuthPlugin from httpie.plugins.builtin import HTTPBasicAuth @@ -14,7 +18,7 @@ from httpie.sessions import Session from httpie.utils import get_expired_cookies from .test_auth_plugins import basic_auth -from .utils import HTTP_OK, MockEnvironment, http, mk_config_dir +from .utils import DUMMY_HOST, HTTP_OK, MockEnvironment, http, mk_config_dir from base64 import b64encode @@ -203,9 +207,9 @@ def test_session_with_cookie_followed_by_another_header(self, httpbin): """ self.start_session(httpbin) session_data = { - "headers": { - "cookie": "...", - "zzz": "..." + 'headers': { + 'cookie': '...', + 'zzz': '...' } } session_path = self.config_dir / 'session-data.json' @@ -307,7 +311,7 @@ class Plugin(AuthPlugin): auth_type = 'test-prompted' def get_auth(self, username=None, password=None): - basic_auth_header = "Basic " + b64encode(self.raw_auth.encode()).strip().decode('latin1') + basic_auth_header = 'Basic ' + b64encode(self.raw_auth.encode()).strip().decode('latin1') return basic_auth(basic_auth_header) plugin_manager.register(Plugin) @@ -359,7 +363,7 @@ def get_auth(self, username=None, password=None): ) updated_session = json.loads(self.session_path.read_text(encoding=UTF8)) assert updated_session['auth']['type'] == 'test-saved' - assert updated_session['auth']['raw_auth'] == "user:password" + assert updated_session['auth']['raw_auth'] == 'user:password' plugin_manager.unregister(Plugin) @@ -368,12 +372,12 @@ class TestExpiredCookies(CookieTestBase): @pytest.mark.parametrize( 'initial_cookie, expired_cookie', [ - ({'id': {'value': 123}}, 'id'), - ({'id': {'value': 123}}, 'token') + ({'id': {'value': 123}}, {'name': 'id'}), + ({'id': {'value': 123}}, {'name': 'token'}) ] ) - def test_removes_expired_cookies_from_session_obj(self, initial_cookie, expired_cookie, httpbin): - session = Session(self.config_dir) + def test_removes_expired_cookies_from_session_obj(self, initial_cookie, expired_cookie, httpbin, mock_env): + session = Session(self.config_dir, env=mock_env, session_id=None, bound_host=None) session['cookies'] = initial_cookie session.remove_cookies([expired_cookie]) assert expired_cookie not in session.cookies @@ -524,3 +528,165 @@ def test_cookie_storage_priority(self, cli_cookie, set_cookie, expected, httpbin updated_session = json.loads(self.session_path.read_text(encoding=UTF8)) assert updated_session['cookies']['cookie1']['value'] == expected + + +@pytest.fixture +def basic_session(httpbin, tmp_path): + session_path = tmp_path / 'session.json' + http( + '--session', str(session_path), + httpbin + '/get' + ) + return session_path + + +@contextmanager +def open_session(path: Path, env: Environment, read_only: bool = False) -> Iterator[Session]: + session = Session(path, env, session_id='test', bound_host=DUMMY_HOST) + session.load() + yield session + if not read_only: + session.save() + + +@contextmanager +def open_raw_session(path: Path, read_only: bool = False) -> None: + with open(path) as stream: + raw_session = json.load(stream) + + yield raw_session + + if not read_only: + with open(path, 'w') as stream: + json.dump(raw_session, stream) + + +def read_stderr(env: Environment) -> bytes: + env.stderr.seek(0) + stderr_data = env.stderr.read() + if isinstance(stderr_data, str): + return stderr_data.encode() + else: + return stderr_data + + +def test_old_session_version_saved_as_is(basic_session, mock_env): + with open_session(basic_session, mock_env) as session: + session['__meta__'] = {'httpie': '0.0.1'} + + with open_session(basic_session, mock_env, read_only=True) as session: + assert session['__meta__']['httpie'] == '0.0.1' + + +def test_old_session_cookie_layout_warning(basic_session, mock_env): + with open_session(basic_session, mock_env) as session: + # Use the old layout & set a cookie + session['cookies'] = {} + session.cookies.set('foo', 'bar') + + assert read_stderr(mock_env) == b'' + + with open_session(basic_session, mock_env, read_only=True) as session: + assert b'Outdated layout detected' in read_stderr(mock_env) + + +@pytest.mark.parametrize('cookies, expect_warning', [ + # Old-style cookie format + ( + # Without 'domain' set + {'foo': {'value': 'bar'}}, + True + ), + ( + # With 'domain' set to empty string + {'foo': {'value': 'bar', 'domain': ''}}, + True + ), + ( + # With 'domain' set to null + {'foo': {'value': 'bar', 'domain': None}}, + False, + ), + ( + # With 'domain' set to a URL + {'foo': {'value': 'bar', 'domain': DUMMY_HOST}}, + False, + ), + # New style cookie format + ( + # Without 'domain' set + [{'name': 'foo', 'value': 'bar'}], + False + ), + ( + # With 'domain' set to empty string + [{'name': 'foo', 'value': 'bar', 'domain': ''}], + False + ), + ( + # With 'domain' set to null + [{'name': 'foo', 'value': 'bar', 'domain': None}], + False, + ), + ( + # With 'domain' set to a URL + [{'name': 'foo', 'value': 'bar', 'domain': DUMMY_HOST}], + False, + ), +]) +def test_cookie_security_warnings_on_raw_cookies(basic_session, mock_env, cookies, expect_warning): + with open_raw_session(basic_session) as raw_session: + raw_session['cookies'] = cookies + + with open_session(basic_session, mock_env, read_only=True): + warning = b'Outdated layout detected' + stderr = read_stderr(mock_env) + + if expect_warning: + assert warning in stderr + else: + assert warning not in stderr + + +def test_old_session_cookie_layout_loading(basic_session, httpbin, mock_env): + with open_session(basic_session, mock_env) as session: + # Use the old layout & set a cookie + session['cookies'] = {} + session.cookies.set('foo', 'bar') + + response = http( + '--session', str(basic_session), + httpbin + '/cookies' + ) + assert response.json['cookies'] == {'foo': 'bar'} + + +@pytest.mark.parametrize('layout_type', [ + dict, list +]) +def test_session_cookie_layout_preservance(basic_session, mock_env, layout_type): + with open_session(basic_session, mock_env) as session: + session['cookies'] = layout_type() + session.cookies.set('foo', 'bar') + session.save() + + with open_session(basic_session, mock_env, read_only=True) as session: + assert isinstance(session['cookies'], layout_type) + + +@pytest.mark.parametrize('layout_type', [ + dict, list +]) +def test_session_cookie_layout_preservance_on_new_cookies(basic_session, httpbin, mock_env, layout_type): + with open_session(basic_session, mock_env) as session: + session['cookies'] = layout_type() + session.cookies.set('foo', 'bar') + session.save() + + http( + '--session', str(basic_session), + httpbin + '/cookies/set/baz/quux' + ) + + with open_session(basic_session, mock_env, read_only=True) as session: + assert isinstance(session['cookies'], layout_type) diff --git a/tests/test_ssl.py b/tests/test_ssl.py index f930bf2826..fc587064d7 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -6,6 +6,8 @@ import requests.exceptions import urllib3 +from unittest import mock + from httpie.ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS from httpie.status import ExitStatus @@ -32,6 +34,15 @@ CLIENT_KEY = str(CERTS_ROOT / 'client.key') CLIENT_PEM = str(CERTS_ROOT / 'client.pem') +# In case of a regeneration, use the following commands +# in the PWD_TESTS_ROOT: +# $ openssl genrsa -aes128 -passout pass:password 3072 > client.pem +# $ openssl req -new -x509 -nodes -days 10000 -key client.pem > client.pem +PWD_TESTS_ROOT = CERTS_ROOT / 'password_protected' +PWD_CLIENT_PEM = str(PWD_TESTS_ROOT / 'client.pem') +PWD_CLIENT_KEY = str(PWD_TESTS_ROOT / 'client.key') +PWD_CLIENT_PASS = 'password' +PWD_CLIENT_INVALID_PASS = PWD_CLIENT_PASS + 'invalid' # We test against a local httpbin instance which uses a self-signed cert. # Requests without --verify= will fail with a verification error. @@ -165,3 +176,37 @@ def test_pyopenssl_presence(): else: assert urllib3.util.ssl_.IS_PYOPENSSL assert urllib3.util.IS_PYOPENSSL + + +@mock.patch('httpie.cli.argtypes.SSLCredentials._prompt_password', + new=lambda self, prompt: PWD_CLIENT_PASS) +def test_password_protected_cert_prompt(httpbin_secure): + r = http(httpbin_secure + '/get', + '--cert', PWD_CLIENT_PEM, + '--cert-key', PWD_CLIENT_KEY) + assert HTTP_OK in r + + +@mock.patch('httpie.cli.argtypes.SSLCredentials._prompt_password', + new=lambda self, prompt: PWD_CLIENT_INVALID_PASS) +def test_password_protected_cert_prompt_invalid(httpbin_secure): + with pytest.raises(ssl_errors): + http(httpbin_secure + '/get', + '--cert', PWD_CLIENT_PEM, + '--cert-key', PWD_CLIENT_KEY) + + +def test_password_protected_cert_cli_arg(httpbin_secure): + r = http(httpbin_secure + '/get', + '--cert', PWD_CLIENT_PEM, + '--cert-key', PWD_CLIENT_KEY, + '--cert-key-pass', PWD_CLIENT_PASS) + assert HTTP_OK in r + + +def test_password_protected_cert_cli_arg_invalid(httpbin_secure): + with pytest.raises(ssl_errors): + http(httpbin_secure + '/get', + '--cert', PWD_CLIENT_PEM, + '--cert-key', PWD_CLIENT_KEY, + '--cert-key-pass', PWD_CLIENT_INVALID_PASS) diff --git a/tests/test_uploads.py b/tests/test_uploads.py index d9de3ac984..5695d0c8c2 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -121,6 +121,7 @@ def stdin_processes(httpbin, *args): @pytest.mark.parametrize("wait", (True, False)) +@pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") def test_reading_from_stdin(httpbin, wait): with stdin_processes(httpbin) as (process_1, process_2): @@ -138,6 +139,7 @@ def test_reading_from_stdin(httpbin, wait): assert b'> warning: no stdin data read in 0.1s' not in errs +@pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") def test_stdin_read_warning(httpbin): with stdin_processes(httpbin) as (process_1, process_2): @@ -153,6 +155,7 @@ def test_stdin_read_warning(httpbin): assert b'> warning: no stdin data read in 0.1s' in errs +@pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") def test_stdin_read_warning_with_quiet(httpbin): with stdin_processes(httpbin, "-qq") as (process_1, process_2): diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index f40a3f327d..d3359820c1 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -5,6 +5,8 @@ import time import json import tempfile +import warnings +import pytest from contextlib import suppress from io import BytesIO from pathlib import Path @@ -16,6 +18,7 @@ from httpie.status import ExitStatus from httpie.config import Config from httpie.context import Environment +from httpie.utils import url_as_host # pytest-httpbin currently does not support chunked requests: @@ -28,6 +31,8 @@ TESTS_ROOT = Path(__file__).parent.parent CRLF = '\r\n' COLOR = '\x1b[' +COLOR_RE = re.compile(r'\x1b\[\d+(;\d+)*?m', re.MULTILINE) + HTTP_OK = '200 OK' # noinspection GrazieInspection HTTP_OK_COLOR = ( @@ -37,6 +42,11 @@ ) DUMMY_URL = 'http://this-should.never-resolve' # Note: URL never fetched +DUMMY_HOST = url_as_host(DUMMY_URL) + + +def strip_colors(colorized_msg: str) -> str: + return COLOR_RE.sub('', colorized_msg) def mk_config_dir() -> Path: @@ -91,6 +101,7 @@ def create_temp_config_dir(self): def cleanup(self): self.stdout.close() self.stderr.close() + warnings.resetwarnings() if self._delete_config_dir: assert self._temp_dir in self.config_dir.parents from shutil import rmtree @@ -180,6 +191,13 @@ class ExitStatusError(Exception): pass +@pytest.fixture +def mock_env() -> MockEnvironment: + env = MockEnvironment(stdout_mode='') + yield env + env.cleanup() + + def normalize_args(args: Iterable[Any]) -> List[str]: return [str(arg) for arg in args] @@ -194,7 +212,7 @@ def httpie( status. """ - env = kwargs.setdefault('env', MockEnvironment()) + env = kwargs.setdefault('env', MockEnvironment(stdout_mode='')) cli_args = ['httpie'] if not kwargs.pop('no_debug', False): cli_args.append('--debug') diff --git a/tests/utils/http_server.py b/tests/utils/http_server.py index fc8f2b07a2..ecc14966b9 100644 --- a/tests/utils/http_server.py +++ b/tests/utils/http_server.py @@ -18,14 +18,17 @@ def inner(func): return func return inner - def do_GET(self): + def do_generic(self): parse_result = urlparse(self.path) - func = self.handlers['GET'].get(parse_result.path) + func = self.handlers[self.command].get(parse_result.path) if func is None: return self.send_error(HTTPStatus.NOT_FOUND) return func(self) + do_GET = do_generic + do_POST = do_generic + @TestHandler.handler('GET', '/headers') def get_headers(handler): @@ -73,6 +76,28 @@ def random_encoding(handler): handler.wfile.write('0\r\n\r\n'.encode('utf-8')) +@TestHandler.handler('POST', '/status/msg') +def status_custom_msg(handler): + content_len = int(handler.headers.get('content-length', 0)) + post_body = handler.rfile.read(content_len).decode() + + handler.send_response(200, post_body) + handler.end_headers() + + +@TestHandler.handler('GET', '/cookies/set-and-redirect') +def set_cookie_and_redirect(handler): + handler.send_response(302) + + redirect_to = handler.headers.get('X-Redirect-To', '/headers') + handler.send_header('Location', redirect_to) + + raw_cookies = handler.headers.get('X-Cookies', 'a=b') + for cookie in raw_cookies.split(', '): + handler.send_header('Set-Cookie', cookie) + handler.end_headers() + + @pytest.fixture(scope="function") def http_server(): """A custom HTTP server implementation for our tests, that is