From 86ba995ad881a1a09b04c8088211c40329932256 Mon Sep 17 00:00:00 2001 From: Marcos Chicote Date: Wed, 26 Jan 2022 15:45:03 +0100 Subject: [PATCH 01/36] 2022 (#1259) --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index a6270bbf89..4a53e9d679 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright © 2012-2021 Jakub Roztocil +Copyright © 2012-2022 Jakub Roztocil Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: From 7abddfe3500a777240889529ce32f2ecbb55169c Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 1 Feb 2022 12:52:07 +0300 Subject: [PATCH 02/36] Mark stdin warning related tests with `requires_external_processes` (#1289) * Mark test_stdin_read_warning with requires_installation * Mark stdin tests with requires_external_processes Co-authored-by: Nilushan Costa <19643850+nilushancosta@users.noreply.github.com> --- pytest.ini | 4 ++++ tests/test_uploads.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/pytest.ini b/pytest.ini index 7c6209701b..ced65979b1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,7 @@ [pytest] markers = + # If you want to run tests without a full HTTPie installation + # we advise you to disable the markers below, e.g: + # pytest -m 'not requires_installation and not requires_external_processes' requires_installation + requires_external_processes diff --git a/tests/test_uploads.py b/tests/test_uploads.py index d9de3ac984..5695d0c8c2 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -121,6 +121,7 @@ def stdin_processes(httpbin, *args): @pytest.mark.parametrize("wait", (True, False)) +@pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") def test_reading_from_stdin(httpbin, wait): with stdin_processes(httpbin) as (process_1, process_2): @@ -138,6 +139,7 @@ def test_reading_from_stdin(httpbin, wait): assert b'> warning: no stdin data read in 0.1s' not in errs +@pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") def test_stdin_read_warning(httpbin): with stdin_processes(httpbin) as (process_1, process_2): @@ -153,6 +155,7 @@ def test_stdin_read_warning(httpbin): assert b'> warning: no stdin data read in 0.1s' in errs +@pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") def test_stdin_read_warning_with_quiet(httpbin): with stdin_processes(httpbin, "-qq") as (process_1, process_2): From f1ea4860254bebefb363d2fda2e5a75fc6226f3e Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 1 Feb 2022 13:10:55 +0300 Subject: [PATCH 03/36] Fix escaping of integer indexes with multiple backslashes (#1288) --- CHANGELOG.md | 4 ++++ httpie/cli/nested_json.py | 24 ++++++++++++------------ tests/test_json.py | 22 ++++++++++++++++++++++ 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 045d406cfe..b580104564 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ This document records all notable changes to [HTTPie](https://httpie.io). This project adheres to [Semantic Versioning](https://semver.org/). +## Unreleased + +- Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285)) + ## [3.0.2](https://github.com/httpie/httpie/compare/3.0.1...3.0.2) (2022-01-24) [What’s new in HTTPie for Terminal 3.0 →](https://httpie.io/blog/httpie-3.0.0) diff --git a/httpie/cli/nested_json.py b/httpie/cli/nested_json.py index 501e3b594f..beb5205843 100644 --- a/httpie/cli/nested_json.py +++ b/httpie/cli/nested_json.py @@ -88,18 +88,18 @@ def send_buffer() -> Iterator[Token]: return None value = ''.join(buffer) - for variation, kind in [ - (int, TokenKind.NUMBER), - (check_escaped_int, TokenKind.TEXT), - ]: - try: - value = variation(value) - except ValueError: - continue - else: - break - else: - kind = TokenKind.TEXT + kind = TokenKind.TEXT + if not backslashes: + for variation, kind in [ + (int, TokenKind.NUMBER), + (check_escaped_int, TokenKind.TEXT), + ]: + try: + value = variation(value) + except ValueError: + continue + else: + break yield Token( kind, value, start=cursor - (len(buffer) + backslashes), end=cursor diff --git a/tests/test_json.py b/tests/test_json.py index b454c03408..7b4dff41e8 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -397,6 +397,28 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value): '2012': {'x': 2, '[3]': 4}, }, ), + ( + [ + r'a[\0]:=0', + r'a[\\1]:=1', + r'a[\\\2]:=2', + r'a[\\\\\3]:=3', + r'a[-1\\]:=-1', + r'a[-2\\\\]:=-2', + r'a[\\-3\\\\]:=-3', + ], + { + "a": { + "0": 0, + r"\1": 1, + r"\\2": 2, + r"\\\3": 3, + "-1\\": -1, + "-2\\\\": -2, + "\\-3\\\\": -3, + } + } + ), ], ) def test_nested_json_syntax(input_json, expected_json, httpbin): From d45f413f12198137869e403a3a77ffdf073206db Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Thu, 3 Feb 2022 12:47:06 +0300 Subject: [PATCH 04/36] Make the version point to `3.0.3.dev0` (#1291) --- CHANGELOG.md | 2 +- docs/README.md | 2 +- httpie/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b580104564..fa90f10714 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ This document records all notable changes to [HTTPie](https://httpie.io). This project adheres to [Semantic Versioning](https://semver.org/). -## Unreleased +## [3.0.3.dev0](https://github.com/httpie/httpie/compare/3.0.2...HEAD) (Unreleased) - Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285)) diff --git a/docs/README.md b/docs/README.md index 66bb4d6a9d..5b4f6493da 100644 --- a/docs/README.md +++ b/docs/README.md @@ -260,7 +260,7 @@ Verify that now you have the [current development version identifier](https://gi ```bash $ http --version -# 3.0.0 +# 3.0.3.dev0 ``` ## Usage diff --git a/httpie/__init__.py b/httpie/__init__.py index 7d48b3499c..683e7b98a7 100644 --- a/httpie/__init__.py +++ b/httpie/__init__.py @@ -3,6 +3,6 @@ """ -__version__ = '3.0.2' +__version__ = '3.0.3.dev0' __author__ = 'Jakub Roztocil' __licence__ = 'BSD' From 42edb1eb767110d9dbe883f2ada469d798d675e0 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Sat, 5 Feb 2022 22:07:28 +0300 Subject: [PATCH 05/36] Use 3.0.0 blog post as the changelog --- docs/packaging/windows-chocolatey/httpie.nuspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/packaging/windows-chocolatey/httpie.nuspec b/docs/packaging/windows-chocolatey/httpie.nuspec index ce69afa5a7..6a3e4fc8c5 100644 --- a/docs/packaging/windows-chocolatey/httpie.nuspec +++ b/docs/packaging/windows-chocolatey/httpie.nuspec @@ -33,7 +33,7 @@ Main features: https://raw.githubusercontent.com/httpie/httpie/master/LICENSE https://pie-assets.s3.eu-central-1.amazonaws.com/LogoIcons/GB.png false - See the [changelog](https://github.com/httpie/httpie/blob/2.6.0/CHANGELOG.md). + See the [changelog](https://httpie.io/blog/httpie-3.0.0). httpie http https rest api client curl python ssl cli foss oss url https://httpie.io https://github.com/httpie/httpie/tree/master/docs/packaging/windows-chocolatey From 46e782bf75092a94a7a4f966e29f4f9c2b1d2970 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Sat, 5 Feb 2022 22:07:51 +0300 Subject: [PATCH 06/36] Point package to 3.0.2 --- docs/packaging/windows-chocolatey/httpie.nuspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/packaging/windows-chocolatey/httpie.nuspec b/docs/packaging/windows-chocolatey/httpie.nuspec index 6a3e4fc8c5..3dc8358727 100644 --- a/docs/packaging/windows-chocolatey/httpie.nuspec +++ b/docs/packaging/windows-chocolatey/httpie.nuspec @@ -2,7 +2,7 @@ httpie - 2.6.0 + 3.0.2 Modern, user-friendly command-line HTTP client for the API era HTTPie *aitch-tee-tee-pie* is a user-friendly command-line HTTP client for the API era. From 37ef6708760015ffed51d3197bd3ae5fba6df9ed Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Sat, 5 Feb 2022 22:08:03 +0300 Subject: [PATCH 07/36] Update copyright year --- docs/packaging/windows-chocolatey/httpie.nuspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/packaging/windows-chocolatey/httpie.nuspec b/docs/packaging/windows-chocolatey/httpie.nuspec index 3dc8358727..0406a1f21e 100644 --- a/docs/packaging/windows-chocolatey/httpie.nuspec +++ b/docs/packaging/windows-chocolatey/httpie.nuspec @@ -29,7 +29,7 @@ Main features: HTTPie HTTPie jakubroztocil - 2012-2021 Jakub Roztocil + 2012-2022 Jakub Roztocil https://raw.githubusercontent.com/httpie/httpie/master/LICENSE https://pie-assets.s3.eu-central-1.amazonaws.com/LogoIcons/GB.png false From 5fd48e31377edd3083f06fa00759814ea7cd0c32 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 8 Feb 2022 12:21:58 +0300 Subject: [PATCH 08/36] Use the lastest fedora spec in the packit --- .packit.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.packit.yaml b/.packit.yaml index 8fc33379d6..afb1dc1a20 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -2,9 +2,9 @@ # https://packit.dev/docs/configuration/ specfile_path: httpie.spec actions: - # get the current Fedora Rawhide specfile: - # post-upstream-clone: "wget https://src.fedoraproject.org/rpms/httpie/raw/rawhide/f/httpie.spec -O httpie.spec" - post-upstream-clone: "cp docs/packaging/linux-fedora/httpie.spec.txt httpie.spec" + get the current Fedora Rawhide specfile: + post-upstream-clone: "wget https://src.fedoraproject.org/rpms/httpie/raw/rawhide/f/httpie.spec -O httpie.spec" + # post-upstream-clone: "cp docs/packaging/linux-fedora/httpie.spec.txt httpie.spec" jobs: - job: copr_build trigger: pull_request From 384d3869f6b8e9e926ca5be04d2154c4d4101ac9 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 8 Feb 2022 12:22:27 +0300 Subject: [PATCH 09/36] Update the local copy fore 3.0.2 --- docs/packaging/linux-fedora/httpie.spec.txt | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/packaging/linux-fedora/httpie.spec.txt b/docs/packaging/linux-fedora/httpie.spec.txt index b0ab5b7b3a..f18f487801 100644 --- a/docs/packaging/linux-fedora/httpie.spec.txt +++ b/docs/packaging/linux-fedora/httpie.spec.txt @@ -1,5 +1,5 @@ Name: httpie -Version: 2.6.0 +Version: 3.0.2 Release: 1%{?dist} Summary: A Curl-like tool for humans @@ -78,6 +78,21 @@ help2man %{buildroot}%{_bindir}/httpie > %{buildroot}%{_mandir}/man1/httpie.1 %changelog +* Mon Jan 24 2022 Miro Hrončok - 3.0.2-1 +- Update to 3.0.2 +- Fixes: rhbz#2044572 + +* Mon Jan 24 2022 Miro Hrončok - 3.0.1-1 +- Update to 3.0.1 +- Fixes: rhbz#2044058 + +* Fri Jan 21 2022 Miro Hrončok - 3.0.0-1 +- Update to 3.0.0 +- Fixes: rhbz#2043680 + +* Thu Jan 20 2022 Fedora Release Engineering - 2.6.0-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_36_Mass_Rebuild + * Fri Oct 15 2021 Miro Hrončok - 2.6.0-1 - Update to 2.6.0 - Fixes: rhbz#2014022 From e306667436f6a81054034c20297e137d4a433986 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 8 Feb 2022 12:23:35 +0300 Subject: [PATCH 10/36] Leave a note for the local spec --- .packit.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.packit.yaml b/.packit.yaml index afb1dc1a20..32eae1f0e0 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -4,6 +4,7 @@ specfile_path: httpie.spec actions: get the current Fedora Rawhide specfile: post-upstream-clone: "wget https://src.fedoraproject.org/rpms/httpie/raw/rawhide/f/httpie.spec -O httpie.spec" + # Use this when the latest spec is not up-to-date. # post-upstream-clone: "cp docs/packaging/linux-fedora/httpie.spec.txt httpie.spec" jobs: - job: copr_build From 0a9d3d3c54c316c17d3558443f44b5be7e2968f7 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 8 Feb 2022 12:28:45 +0300 Subject: [PATCH 11/36] Fix the packit syntax --- .packit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.packit.yaml b/.packit.yaml index 32eae1f0e0..b002afa67e 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -2,7 +2,7 @@ # https://packit.dev/docs/configuration/ specfile_path: httpie.spec actions: - get the current Fedora Rawhide specfile: + # get the current Fedora Rawhide specfile: post-upstream-clone: "wget https://src.fedoraproject.org/rpms/httpie/raw/rawhide/f/httpie.spec -O httpie.spec" # Use this when the latest spec is not up-to-date. # post-upstream-clone: "cp docs/packaging/linux-fedora/httpie.spec.txt httpie.spec" From cafa11665b2d1d4f8d8e6358913a4a9c5a97a484 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 8 Feb 2022 12:37:45 +0300 Subject: [PATCH 12/36] Disable additional repos --- .packit.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.packit.yaml b/.packit.yaml index b002afa67e..0ae2100cbc 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -12,8 +12,6 @@ jobs: metadata: targets: - fedora-all - additional_repos: - - "https://kojipkgs.fedoraproject.org/repos/f$releasever-build/latest/$basearch/" - job: propose_downstream trigger: release metadata: From 225dccb2186f14f871695b6c4e0bfbcdb2e3aa28 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 9 Feb 2022 02:18:40 +0300 Subject: [PATCH 13/36] Regulate top-level arrays (#1292) * Redesign the starting path * Do not cast `:=[1,2,3]` to a top-level array --- docs/README.md | 41 +++++++++++++++++++++ httpie/cli/constants.py | 1 + httpie/cli/dicts.py | 4 ++ httpie/cli/nested_json.py | 76 +++++++++++++++++++++++++++++++------- httpie/client.py | 6 ++- tests/test_json.py | 77 +++++++++++++++++++++++++++++++++------ 6 files changed, 177 insertions(+), 28 deletions(-) diff --git a/docs/README.md b/docs/README.md index 5b4f6493da..bc7f0e7af7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -854,6 +854,47 @@ $ http PUT pie.dev/put \ #### Advanced usage +##### Top level arrays + +If you want to send an array instead of a regular object, you can simply +do that by omitting the starting key: + +```bash +$ http --offline --print=B pie.dev/post \ + []:=1 \ + []:=2 \ + []:=3 +``` + +```json +[ + 1, + 2, + 3 +] +``` + +You can also apply the nesting to the items by referencing their index: + +```bash +http --offline --print=B pie.dev/post \ + [0][type]=platform [0][name]=terminal \ + [1][type]=platform [1][name]=desktop +``` + +```json +[ + { + "type": "platform", + "name": "terminal" + }, + { + "type": "platform", + "name": "desktop" + } +] +``` + ##### Escaping behavior Nested JSON syntax uses the same [escaping rules](#escaping-rules) as diff --git a/httpie/cli/constants.py b/httpie/cli/constants.py index 897e806b4d..067aaabdf7 100644 --- a/httpie/cli/constants.py +++ b/httpie/cli/constants.py @@ -127,6 +127,7 @@ class RequestType(enum.Enum): JSON = enum.auto() +EMPTY_STRING = '' OPEN_BRACKET = '[' CLOSE_BRACKET = ']' BACKSLASH = '\\' diff --git a/httpie/cli/dicts.py b/httpie/cli/dicts.py index 3d0cab5a45..434a396672 100644 --- a/httpie/cli/dicts.py +++ b/httpie/cli/dicts.py @@ -82,3 +82,7 @@ class MultipartRequestDataDict(MultiValueOrderedDict): class RequestFilesDict(RequestDataDict): pass + + +class NestedJSONArray(list): + """Denotes a top-level JSON array.""" diff --git a/httpie/cli/nested_json.py b/httpie/cli/nested_json.py index beb5205843..c75798929d 100644 --- a/httpie/cli/nested_json.py +++ b/httpie/cli/nested_json.py @@ -9,7 +9,8 @@ Type, Union, ) -from httpie.cli.constants import OPEN_BRACKET, CLOSE_BRACKET, BACKSLASH, HIGHLIGHTER +from httpie.cli.dicts import NestedJSONArray +from httpie.cli.constants import EMPTY_STRING, OPEN_BRACKET, CLOSE_BRACKET, BACKSLASH, HIGHLIGHTER class HTTPieSyntaxError(ValueError): @@ -52,6 +53,7 @@ def to_name(self) -> str: OPERATORS = {OPEN_BRACKET: TokenKind.LEFT_BRACKET, CLOSE_BRACKET: TokenKind.RIGHT_BRACKET} SPECIAL_CHARS = OPERATORS.keys() | {BACKSLASH} +LITERAL_TOKENS = [TokenKind.TEXT, TokenKind.NUMBER] class Token(NamedTuple): @@ -171,8 +173,8 @@ def reconstruct(self) -> str: def parse(source: str) -> Iterator[Path]: """ - start: literal? path* - + start: root_path path* + root_path: (literal | index_path | append_path) literal: TEXT | NUMBER path: @@ -215,16 +217,47 @@ def expect(*kinds): message = f'Expecting {suffix}' raise HTTPieSyntaxError(source, token, message) - root = Path(PathAction.KEY, '', is_root=True) - if can_advance(): - token = tokens[cursor] - if token.kind in {TokenKind.TEXT, TokenKind.NUMBER}: - token = expect(TokenKind.TEXT, TokenKind.NUMBER) - root.accessor = str(token.value) - root.tokens.append(token) + def parse_root(): + tokens = [] + if not can_advance(): + return Path( + PathAction.KEY, + EMPTY_STRING, + is_root=True + ) + + # (literal | index_path | append_path)? + token = expect(*LITERAL_TOKENS, TokenKind.LEFT_BRACKET) + tokens.append(token) + + if token.kind in LITERAL_TOKENS: + action = PathAction.KEY + value = str(token.value) + elif token.kind is TokenKind.LEFT_BRACKET: + token = expect(TokenKind.NUMBER, TokenKind.RIGHT_BRACKET) + tokens.append(token) + if token.kind is TokenKind.NUMBER: + action = PathAction.INDEX + value = token.value + tokens.append(expect(TokenKind.RIGHT_BRACKET)) + elif token.kind is TokenKind.RIGHT_BRACKET: + action = PathAction.APPEND + value = None + else: + assert_cant_happen() + else: + assert_cant_happen() + + return Path( + action, + value, + tokens=tokens, + is_root=True + ) - yield root + yield parse_root() + # path* while can_advance(): path_tokens = [] path_tokens.append(expect(TokenKind.LEFT_BRACKET)) @@ -296,6 +329,10 @@ def object_for(kind: str) -> Any: assert_cant_happen() for index, (path, next_path) in enumerate(zip(paths, paths[1:])): + # If there is no context yet, set it. + if cursor is None: + context = cursor = object_for(path.kind) + if path.kind is PathAction.KEY: type_check(index, path, dict) if next_path.kind is PathAction.SET: @@ -337,8 +374,19 @@ def object_for(kind: str) -> Any: return context +def wrap_with_dict(context): + if context is None: + return {} + elif isinstance(context, list): + return {EMPTY_STRING: NestedJSONArray(context)} + else: + assert isinstance(context, dict) + return context + + def interpret_nested_json(pairs): - context = {} + context = None for key, value in pairs: - interpret(context, key, value) - return context + context = interpret(context, key, value) + + return wrap_with_dict(context) diff --git a/httpie/client.py b/httpie/client.py index c2563cbc3e..06235d249b 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -13,7 +13,8 @@ from . import __version__ from .adapters import HTTPieHTTPAdapter from .context import Environment -from .cli.dicts import HTTPHeadersDict +from .cli.constants import EMPTY_STRING +from .cli.dicts import HTTPHeadersDict, NestedJSONArray from .encoding import UTF8 from .models import RequestsMessage from .plugins.registry import plugin_manager @@ -280,7 +281,8 @@ def json_dict_to_request_body(data: Dict[str, Any]) -> str: # item in the object, with an en empty key. if len(data) == 1: [(key, value)] = data.items() - if key == '' and isinstance(value, list): + if isinstance(value, NestedJSONArray): + assert key == EMPTY_STRING data = value if data: diff --git a/tests/test_json.py b/tests/test_json.py index 7b4dff41e8..2ba603a680 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -321,7 +321,7 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value): 'foo[][key]=value', 'foo[2][key 2]=value 2', r'foo[2][key \[]=value 3', - r'[nesting][under][!][empty][?][\\key]:=4', + r'bar[nesting][under][!][empty][?][\\key]:=4', ], { 'foo': [ @@ -329,7 +329,7 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value): 2, {'key': 'value', 'key 2': 'value 2', 'key [': 'value 3'}, ], - '': { + 'bar': { 'nesting': {'under': {'!': {'empty': {'?': {'\\key': 4}}}}} }, }, @@ -408,17 +408,47 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value): r'a[\\-3\\\\]:=-3', ], { - "a": { - "0": 0, - r"\1": 1, - r"\\2": 2, - r"\\\3": 3, - "-1\\": -1, - "-2\\\\": -2, - "\\-3\\\\": -3, + 'a': { + '0': 0, + r'\1': 1, + r'\\2': 2, + r'\\\3': 3, + '-1\\': -1, + '-2\\\\': -2, + '\\-3\\\\': -3, } - } + }, + ), + ( + ['[]:=0', '[]:=1', '[5]:=5', '[]:=6', '[9]:=9'], + [0, 1, None, None, None, 5, 6, None, None, 9], + ), + ( + ['=empty', 'foo=bar', 'bar[baz][quux]=tuut'], + {'': 'empty', 'foo': 'bar', 'bar': {'baz': {'quux': 'tuut'}}}, + ), + ( + [ + r'\1=top level int', + r'\\1=escaped top level int', + r'\2[\3][\4]:=5', + ], + { + '1': 'top level int', + '\\1': 'escaped top level int', + '2': {'3': {'4': 5}}, + }, + ), + ( + [':={"foo": {"bar": "baz"}}', 'top=val'], + {'': {'foo': {'bar': 'baz'}}, 'top': 'val'}, + ), + ( + ['[][a][b][]:=1', '[0][a][b][]:=2', '[][]:=2'], + [{'a': {'b': [1, 2]}}, [2]], ), + ([':=[1,2,3]'], {'': [1, 2, 3]}), + ([':=[1,2,3]', 'foo=bar'], {'': [1, 2, 3], 'foo': 'bar'}), ], ) def test_nested_json_syntax(input_json, expected_json, httpbin): @@ -516,13 +546,36 @@ def test_nested_json_syntax(input_json, expected_json, httpbin): ['foo[\\1]:=2', 'foo[5]:=3'], "HTTPie Type Error: Can't perform 'index' based access on 'foo' which has a type of 'object' but this operation requires a type of 'array'.\nfoo[5]\n ^^^", ), + ( + ['x=y', '[]:=2'], + "HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.", + ), + ( + ['[]:=2', 'x=y'], + "HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.", + ), + ( + [':=[1,2,3]', '[]:=4'], + "HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.", + ), + ( + ['[]:=4', ':=[1,2,3]'], + "HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.", + ), ], ) def test_nested_json_errors(input_json, expected_error, httpbin): with pytest.raises(HTTPieSyntaxError) as exc: http(httpbin + '/post', *input_json) - assert str(exc.value) == expected_error + exc_lines = str(exc.value).splitlines() + expected_lines = expected_error.splitlines() + if len(expected_lines) == 1: + # When the error offsets are not important, we'll just compare the actual + # error message. + exc_lines = exc_lines[:1] + + assert expected_lines == exc_lines def test_nested_json_sparse_array(httpbin_both): From ad613f29d229d36f8b6b998c05e54bbd1d63a3a7 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Fri, 25 Feb 2022 12:51:34 +0300 Subject: [PATCH 14/36] Add a changelog entry for the top-level array regulation --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa90f10714..73186f6f79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [3.0.3.dev0](https://github.com/httpie/httpie/compare/3.0.2...HEAD) (Unreleased) - Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285)) +- Improved regulation of top-level arrays. ([#1292](https://github.com/httpie/httpie/commit/225dccb2186f14f871695b6c4e0bfbcdb2e3aa28)) ## [3.0.2](https://github.com/httpie/httpie/compare/3.0.1...3.0.2) (2022-01-24) From 30cd862fc0e173698fc17487c4b96d8f64b701ea Mon Sep 17 00:00:00 2001 From: Sebastian Stasiak Date: Mon, 28 Feb 2022 22:57:23 +0100 Subject: [PATCH 15/36] Update commands for Arch (#1306) --- docs/README.md | 4 ++-- docs/installation/methods.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/README.md b/docs/README.md index bc7f0e7af7..590e0de4a4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -205,12 +205,12 @@ Also works for other Arch-derived distributions like ArcoLinux, EndeavourOS, Art ```bash # Install httpie -$ pacman -Sy httpie +$ pacman -Syu httpie ``` ```bash # Upgrade httpie -$ pacman -Syu httpie +$ pacman -Syu ``` ### FreeBSD diff --git a/docs/installation/methods.yml b/docs/installation/methods.yml index 1583b2364a..0828b0d603 100644 --- a/docs/installation/methods.yml +++ b/docs/installation/methods.yml @@ -106,9 +106,9 @@ tools: package: https://archlinux.org/packages/community/any/httpie/ commands: install: - - pacman -Sy httpie - upgrade: - pacman -Syu httpie + upgrade: + - pacman -Syu pkg: title: FreshPorts From 6bf39e469faf08ea74bb40875da2fd5cbbb23226 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Mar 2022 17:17:41 +0300 Subject: [PATCH 16/36] Bump actions/setup-python from 2 to 3 (#1307) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 3. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/benchmark.yml | 2 +- .github/workflows/code-style.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/docs-update-install.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/tests.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 560835dab2..e5b40a95be 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: "3.9" diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml index 3d220d72f0..e25e9515e3 100644 --- a/.github/workflows/code-style.yml +++ b/.github/workflows/code-style.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: 3.9 - run: make venv diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 5edb47fa46..de0063f604 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: "3.10" - run: make install diff --git a/.github/workflows/docs-update-install.yml b/.github/workflows/docs-update-install.yml index 3a22aaa0bb..d92f76c11e 100644 --- a/.github/workflows/docs-update-install.yml +++ b/.github/workflows/docs-update-install.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: 3.9 - run: make install diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 12753b49f5..944d3a3faa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: - name: PyPI configuration run: | echo "[distutils]\nindex-servers=\n httpie\n\n[httpie]\nrepository = https://upload.pypi.org/legacy/\n" > $HOME/.pypirc - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: 3.9 - run: make publish diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3cde99669..f8946c0904 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Windows setup From 6f77e144e4e2ed73ace47cb7547c006861917c11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 08:16:56 -0800 Subject: [PATCH 17/36] Bump actions/checkout from 2 to 3 (#1311) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/benchmark.yml | 2 +- .github/workflows/code-style.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/docs-check-markdown.yml | 2 +- .github/workflows/docs-update-install.yml | 2 +- .github/workflows/release-snap.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test-package-linux-snap.yml | 2 +- .github/workflows/test-package-mac-brew.yml | 2 +- .github/workflows/tests.yml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e5b40a95be..08d24700ea 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -13,7 +13,7 @@ jobs: if: github.event.label.name == 'benchmark' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: "3.9" diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml index e25e9515e3..b53b353b04 100644 --- a/.github/workflows/code-style.yml +++ b/.github/workflows/code-style.yml @@ -13,7 +13,7 @@ jobs: code-style: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: 3.9 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index de0063f604..acd9aeb9da 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,7 +12,7 @@ jobs: coverage: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: "3.10" diff --git a/.github/workflows/docs-check-markdown.yml b/.github/workflows/docs-check-markdown.yml index f019bf7017..a19c25916a 100644 --- a/.github/workflows/docs-check-markdown.yml +++ b/.github/workflows/docs-check-markdown.yml @@ -10,7 +10,7 @@ jobs: doc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Ruby uses: ruby/setup-ruby@v1 with: diff --git a/.github/workflows/docs-update-install.yml b/.github/workflows/docs-update-install.yml index d92f76c11e..2d3fbe09d9 100644 --- a/.github/workflows/docs-update-install.yml +++ b/.github/workflows/docs-update-install.yml @@ -15,7 +15,7 @@ jobs: doc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: 3.9 diff --git a/.github/workflows/release-snap.yml b/.github/workflows/release-snap.yml index f67c0e2577..149adc50e7 100644 --- a/.github/workflows/release-snap.yml +++ b/.github/workflows/release-snap.yml @@ -16,7 +16,7 @@ jobs: snap: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.branch }} - uses: snapcore/action-build@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 944d3a3faa..30561369d1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: new-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.branch }} - name: PyPI configuration diff --git a/.github/workflows/test-package-linux-snap.yml b/.github/workflows/test-package-linux-snap.yml index 12a2766a40..ac9640a06d 100644 --- a/.github/workflows/test-package-linux-snap.yml +++ b/.github/workflows/test-package-linux-snap.yml @@ -12,7 +12,7 @@ jobs: snap: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build uses: snapcore/action-build@v1 id: snapcraft diff --git a/.github/workflows/test-package-mac-brew.yml b/.github/workflows/test-package-mac-brew.yml index 6570a2afd9..babdaa5def 100644 --- a/.github/workflows/test-package-mac-brew.yml +++ b/.github/workflows/test-package-mac-brew.yml @@ -11,7 +11,7 @@ jobs: brew: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup brew run: | brew developer on diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f8946c0904..17d03d3eef 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: pyopenssl: [0, 1] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} From 25bd817bb2ea3dddbb9682766948495bb0b4efe3 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Thu, 3 Mar 2022 19:28:04 +0300 Subject: [PATCH 18/36] Fix displaying of status code without a status message. (#1301) Co-authored-by: Jakub Roztocil --- CHANGELOG.md | 2 ++ httpie/output/lexers/http.py | 2 +- tests/test_output.py | 27 +++++++++++++++++++++++++-- tests/utils/__init__.py | 6 ++++++ tests/utils/http_server.py | 16 ++++++++++++++-- 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73186f6f79..026916d2f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,10 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [3.0.3.dev0](https://github.com/httpie/httpie/compare/3.0.2...HEAD) (Unreleased) - Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285)) +- Fixed displaying of status code without a status message on non-`auto` themes. ([#1300](https://github.com/httpie/httpie/issues/1300)) - Improved regulation of top-level arrays. ([#1292](https://github.com/httpie/httpie/commit/225dccb2186f14f871695b6c4e0bfbcdb2e3aa28)) + ## [3.0.2](https://github.com/httpie/httpie/compare/3.0.1...3.0.2) (2022-01-24) [What’s new in HTTPie for Terminal 3.0 →](https://httpie.io/blog/httpie-3.0.0) diff --git a/httpie/output/lexers/http.py b/httpie/output/lexers/http.py index f06a685380..aea827401e 100644 --- a/httpie/output/lexers/http.py +++ b/httpie/output/lexers/http.py @@ -2,7 +2,7 @@ import pygments from httpie.output.lexers.common import precise -RE_STATUS_LINE = re.compile(r'(\d{3})( +)(.+)') +RE_STATUS_LINE = re.compile(r'(\d{3})( +)?(.+)?') STATUS_TYPES = { '1': pygments.token.Number.HTTP.INFO, diff --git a/tests/test_output.py b/tests/test_output.py index c68bfa9e38..716ceb097d 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -17,10 +17,15 @@ ) from httpie.cli.definition import parser from httpie.encoding import UTF8 -from httpie.output.formatters.colors import get_lexer, PIE_STYLE_NAMES +from httpie.output.formatters.colors import get_lexer, PIE_STYLE_NAMES, BUNDLED_STYLES from httpie.status import ExitStatus from .fixtures import XML_DATA_RAW, XML_DATA_FORMATTED -from .utils import COLOR, CRLF, HTTP_OK, MockEnvironment, http, DUMMY_URL +from .utils import COLOR, CRLF, HTTP_OK, MockEnvironment, http, DUMMY_URL, strip_colors + + +# For ensuring test reproducibility, avoid using the unsorted +# BUNDLED_STYLES set. +SORTED_BUNDLED_STYLES = sorted(BUNDLED_STYLES) @pytest.mark.parametrize('stdout_isatty', [True, False]) @@ -234,6 +239,24 @@ def test_ensure_meta_is_colored(httpbin, style): assert COLOR in r +@pytest.mark.parametrize('style', SORTED_BUNDLED_STYLES) +@pytest.mark.parametrize('msg', [ + '', + ' ', + ' OK', + ' OK ', + ' CUSTOM ', +]) +def test_ensure_status_code_is_shown_on_all_themes(http_server, style, msg): + env = MockEnvironment(colors=256) + r = http('--style', style, + http_server + '/status/msg', + '--raw', msg, env=env) + + # Trailing space is stripped away. + assert 'HTTP/1.0 200' + msg.rstrip() in strip_colors(r) + + class TestPrettyOptions: """Test the --pretty handling.""" diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index f8954565c5..2bd376eef4 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -27,6 +27,8 @@ TESTS_ROOT = Path(__file__).parent.parent CRLF = '\r\n' COLOR = '\x1b[' +COLOR_RE = re.compile(r'\x1b\[\d+(;\d+)*?m', re.MULTILINE) + HTTP_OK = '200 OK' # noinspection GrazieInspection HTTP_OK_COLOR = ( @@ -38,6 +40,10 @@ DUMMY_URL = 'http://this-should.never-resolve' # Note: URL never fetched +def strip_colors(colorized_msg: str) -> str: + return COLOR_RE.sub('', colorized_msg) + + def mk_config_dir() -> Path: dirname = tempfile.mkdtemp(prefix='httpie_config_') return Path(dirname) diff --git a/tests/utils/http_server.py b/tests/utils/http_server.py index fc8f2b07a2..0a96dd8b07 100644 --- a/tests/utils/http_server.py +++ b/tests/utils/http_server.py @@ -18,14 +18,17 @@ def inner(func): return func return inner - def do_GET(self): + def do_generic(self): parse_result = urlparse(self.path) - func = self.handlers['GET'].get(parse_result.path) + func = self.handlers[self.command].get(parse_result.path) if func is None: return self.send_error(HTTPStatus.NOT_FOUND) return func(self) + do_GET = do_generic + do_POST = do_generic + @TestHandler.handler('GET', '/headers') def get_headers(handler): @@ -73,6 +76,15 @@ def random_encoding(handler): handler.wfile.write('0\r\n\r\n'.encode('utf-8')) +@TestHandler.handler('POST', '/status/msg') +def status_custom_msg(handler): + content_len = int(handler.headers.get('content-length', 0)) + post_body = handler.rfile.read(content_len).decode() + + handler.send_response(200, post_body) + handler.end_headers() + + @pytest.fixture(scope="function") def http_server(): """A custom HTTP server implementation for our tests, that is From c901e704633cb79c686b3edaae497ed38b2978ba Mon Sep 17 00:00:00 2001 From: Hoylen Sue Date: Fri, 4 Mar 2022 02:31:06 +1000 Subject: [PATCH 19/36] Replaced unmaintained OAuth plugin with new httpie-oauth1 plugin. (#1302) --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 590e0de4a4..b5d139aad9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1377,7 +1377,7 @@ Here are a few picks: - [httpie-jwt-auth](https://github.com/teracyhq/httpie-jwt-auth): JWTAuth (JSON Web Tokens) - [httpie-negotiate](https://github.com/ndzou/httpie-negotiate): SPNEGO (GSS Negotiate) - [httpie-ntlm](https://github.com/httpie/httpie-ntlm): NTLM (NT LAN Manager) -- [httpie-oauth](https://github.com/httpie/httpie-oauth): OAuth +- [httpie-oauth1](https://github.com/qcif/httpie-oauth1): OAuth 1.0a - [requests-hawk](https://github.com/mozilla-services/requests-hawk): Hawk See [plugin manager](#plugin-manager) for more details. From 55087a901e86eeda2a62e33765bb48299f004a78 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 7 Mar 2022 15:40:35 +0300 Subject: [PATCH 20/36] Introduce a mode to suppress all warnings (#1283) --- CHANGELOG.md | 2 +- httpie/cli/argparser.py | 2 ++ httpie/context.py | 29 ++++++++++++++++++++++++++--- httpie/core.py | 4 ++-- tests/test_output.py | 26 ++++++++++++++++++++++++++ tests/utils/__init__.py | 2 ++ 6 files changed, 59 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 026916d2f4..e74b331772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285)) - Fixed displaying of status code without a status message on non-`auto` themes. ([#1300](https://github.com/httpie/httpie/issues/1300)) - Improved regulation of top-level arrays. ([#1292](https://github.com/httpie/httpie/commit/225dccb2186f14f871695b6c4e0bfbcdb2e3aa28)) - +- Double `--quiet` flags will now suppress all python level warnings. ([#1271](https://github.com/httpie/httpie/issues/1271)) ## [3.0.2](https://github.com/httpie/httpie/compare/3.0.1...3.0.2) (2022-01-24) diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index 64481096c7..b1ab8de155 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -230,9 +230,11 @@ def _setup_standard_streams(self): self.env.stdout_isatty = False if self.args.quiet: + self.env.quiet = self.args.quiet self.env.stderr = self.env.devnull if not (self.args.output_file_specified and not self.args.download): self.env.stdout = self.env.devnull + self.env.apply_warnings_filter() def _process_auth(self): # TODO: refactor & simplify this method. diff --git a/httpie/context.py b/httpie/context.py index 7a6e6a865c..50a8f772cd 100644 --- a/httpie/context.py +++ b/httpie/context.py @@ -1,8 +1,10 @@ import sys import os +import warnings from contextlib import contextmanager from pathlib import Path from typing import Iterator, IO, Optional +from enum import Enum try: @@ -17,6 +19,17 @@ from .utils import repr_dict +class Levels(str, Enum): + WARNING = 'warning' + ERROR = 'error' + + +DISPLAY_THRESHOLDS = { + Levels.WARNING: 2, + Levels.ERROR: float('inf'), # Never hide errors. +} + + class Environment: """ Information about the execution context @@ -87,6 +100,8 @@ def __init__(self, devnull=None, **kwargs): self.stdout_encoding = getattr( actual_stdout, 'encoding', None) or UTF8 + self.quiet = kwargs.pop('quiet', 0) + def __str__(self): defaults = dict(type(self).__dict__) actual = dict(defaults) @@ -134,6 +149,14 @@ def as_silent(self) -> Iterator[None]: self.stdout = original_stdout self.stderr = original_stderr - def log_error(self, msg, level='error'): - assert level in ['error', 'warning'] - self._orig_stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n') + def log_error(self, msg: str, level: Levels = Levels.ERROR) -> None: + if self.stdout_isatty and self.quiet >= DISPLAY_THRESHOLDS[level]: + stderr = self.stderr # Not directly /dev/null, since stderr might be mocked + else: + stderr = self._orig_stderr + + stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n') + + def apply_warnings_filter(self) -> None: + if self.quiet >= DISPLAY_THRESHOLDS[Levels.WARNING]: + warnings.simplefilter("ignore") diff --git a/httpie/core.py b/httpie/core.py index 079de17d28..3dbed19523 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -13,7 +13,7 @@ from .cli.constants import OUT_REQ_BODY from .cli.nested_json import HTTPieSyntaxError from .client import collect_messages -from .context import Environment +from .context import Environment, Levels from .downloads import Downloader from .models import ( RequestsMessageKind, @@ -221,7 +221,7 @@ def request_body_read_callback(chunk: bytes): if args.check_status or downloader: exit_status = http_status_to_exit_status(http_status=message.status_code, follow=args.follow) if exit_status != ExitStatus.SUCCESS and (not env.stdout_isatty or args.quiet == 1): - env.log_error(f'HTTP {message.raw.status} {message.raw.reason}', level='warning') + env.log_error(f'HTTP {message.raw.status} {message.raw.reason}', level=Levels.WARNING) write_message(requests_message=message, env=env, args=args, output_options=output_options._replace( body=do_write_body )) diff --git a/tests/test_output.py b/tests/test_output.py index 716ceb097d..470673bf91 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -5,6 +5,7 @@ import json import os import io +import warnings from urllib.request import urlopen import pytest @@ -90,6 +91,31 @@ def test_quiet_quiet_with_check_status_non_zero_pipe(self, httpbin): ) assert 'http: warning: HTTP 500' in r.stderr + @mock.patch('httpie.core.program') + @pytest.mark.parametrize('flags, expected_warnings', [ + ([], 1), + (['-q'], 1), + (['-qq'], 0), + ]) + def test_quiet_on_python_warnings(self, test_patch, httpbin, flags, expected_warnings): + def warn_and_run(*args, **kwargs): + warnings.warn('warning!!') + return ExitStatus.SUCCESS + + test_patch.side_effect = warn_and_run + with pytest.warns(None) as record: + http(*flags, httpbin + '/get') + + assert len(record) == expected_warnings + + def test_double_quiet_on_error(self, httpbin): + r = http( + '-qq', '--check-status', '$$$this.does.not.exist$$$', + tolerate_error_exit_status=True, + ) + assert not r + assert 'Couldn’t resolve the given hostname' in r.stderr + @pytest.mark.parametrize('quiet_flags', QUIET_SCENARIOS) @mock.patch('httpie.cli.argtypes.AuthCredentials._getpass', new=lambda self, prompt: 'password') diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 2bd376eef4..cf90d684b9 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -5,6 +5,7 @@ import time import json import tempfile +import warnings from io import BytesIO from pathlib import Path from typing import Any, Optional, Union, List, Iterable @@ -96,6 +97,7 @@ def create_temp_config_dir(self): def cleanup(self): self.stdout.close() self.stderr.close() + warnings.resetwarnings() if self._delete_config_dir: assert self._temp_dir in self.config_dir.parents from shutil import rmtree From b0f5b8ab2600016b9ecf479abd4eb8961ea8fbf3 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Fri, 25 Feb 2022 15:42:13 +0300 Subject: [PATCH 21/36] Prevent data race happening between `select.select` and `file.read()` --- httpie/uploads.py | 59 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/httpie/uploads.py b/httpie/uploads.py index 4fdb79222d..9ddc89c0ca 100644 --- a/httpie/uploads.py +++ b/httpie/uploads.py @@ -2,6 +2,8 @@ import os import zlib import functools +import time +import threading from typing import Any, Callable, IO, Iterable, Optional, Tuple, Union, TYPE_CHECKING from urllib.parse import urlencode @@ -22,12 +24,20 @@ def __iter__(self) -> Iterable[Union[str, bytes]]: class ChunkedUploadStream(ChunkedStream): - def __init__(self, stream: Iterable, callback: Callable): + def __init__( + self, + stream: Iterable, + callback: Callable, + event: Optional[threading.Event] = None + ) -> None: self.callback = callback self.stream = stream + self.event = event def __iter__(self) -> Iterable[Union[str, bytes]]: for chunk in self.stream: + if self.event: + self.event.set() self.callback(chunk) yield chunk @@ -35,12 +45,19 @@ def __iter__(self) -> Iterable[Union[str, bytes]]: class ChunkedMultipartUploadStream(ChunkedStream): chunk_size = 100 * 1024 - def __init__(self, encoder: 'MultipartEncoder'): + def __init__( + self, + encoder: 'MultipartEncoder', + event: Optional[threading.Event] = None + ) -> None: self.encoder = encoder + self.event = event def __iter__(self) -> Iterable[Union[str, bytes]]: while True: chunk = self.encoder.read(self.chunk_size) + if self.event: + self.event.set() if not chunk: break yield chunk @@ -80,7 +97,7 @@ def is_stdin(file: IO) -> bool: READ_THRESHOLD = float(os.getenv("HTTPIE_STDIN_READ_WARN_THRESHOLD", 10.0)) -def observe_stdin_for_data_thread(env: Environment, file: IO) -> None: +def observe_stdin_for_data_thread(env: Environment, file: IO, read_event: threading.Event) -> None: # Windows unfortunately does not support select() operation # on regular files, like stdin in our use case. # https://docs.python.org/3/library/select.html#select.select @@ -92,12 +109,9 @@ def observe_stdin_for_data_thread(env: Environment, file: IO) -> None: if READ_THRESHOLD == 0: return None - import select - import threading - - def worker(): - can_read, _, _ = select.select([file], [], [], READ_THRESHOLD) - if not can_read: + def worker(event: threading.Event) -> None: + time.sleep(READ_THRESHOLD) + if not event.is_set(): env.stderr.write( f'> warning: no stdin data read in {READ_THRESHOLD}s ' f'(perhaps you want to --ignore-stdin)\n' @@ -105,11 +119,28 @@ def worker(): ) thread = threading.Thread( - target=worker + target=worker, + args=(read_event,) ) thread.start() +def _read_file_with_selectors(file: IO, read_event: threading.Event) -> bytes: + if is_windows or not is_stdin(file): + return as_bytes(file.read()) + + import select + + # Try checking whether there is any incoming data for READ_THRESHOLD + # seconds. If there isn't anything in the given period, issue + # a warning about a misusage. + read_selectors, _, _ = select.select([file], [], [], READ_THRESHOLD) + if read_selectors: + read_event.set() + + return as_bytes(file.read()) + + def _prepare_file_for_upload( env: Environment, file: Union[IO, 'MultipartEncoder'], @@ -117,9 +148,11 @@ def _prepare_file_for_upload( chunked: bool = False, content_length_header_value: Optional[int] = None, ) -> Union[bytes, IO, ChunkedStream]: + read_event = threading.Event() if not super_len(file): if is_stdin(file): - observe_stdin_for_data_thread(env, file) + observe_stdin_for_data_thread(env, file, read_event) + # Zero-length -> assume stdin. if content_length_header_value is None and not chunked: # Read the whole stdin to determine `Content-Length`. @@ -129,7 +162,7 @@ def _prepare_file_for_upload( # something like --no-chunked. # This would be backwards-incompatible so wait until v3.0.0. # - file = as_bytes(file.read()) + file = _read_file_with_selectors(file, read_event) else: file.read = _wrap_function_with_callback( file.read, @@ -141,11 +174,13 @@ def _prepare_file_for_upload( if isinstance(file, MultipartEncoder): return ChunkedMultipartUploadStream( encoder=file, + event=read_event, ) else: return ChunkedUploadStream( stream=file, callback=callback, + event=read_event ) else: return file From 5c982533777ed054a7cef2398b542473f8df3b3d Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Thu, 3 Mar 2022 08:22:17 -0800 Subject: [PATCH 22/36] Update httpie/uploads.py --- httpie/uploads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpie/uploads.py b/httpie/uploads.py index 9ddc89c0ca..c9a763df78 100644 --- a/httpie/uploads.py +++ b/httpie/uploads.py @@ -94,7 +94,7 @@ def is_stdin(file: IO) -> bool: return file_no == sys.stdin.fileno() -READ_THRESHOLD = float(os.getenv("HTTPIE_STDIN_READ_WARN_THRESHOLD", 10.0)) +READ_THRESHOLD = float(os.getenv('HTTPIE_STDIN_READ_WARN_THRESHOLD', 10.0)) def observe_stdin_for_data_thread(env: Environment, file: IO, read_event: threading.Event) -> None: From 5ac05e95149eaafb9dd1a1c1f3971da00ce0d466 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 7 Mar 2022 15:51:15 +0300 Subject: [PATCH 23/36] Add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e74b331772..4a05cbc9ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285)) - Fixed displaying of status code without a status message on non-`auto` themes. ([#1300](https://github.com/httpie/httpie/issues/1300)) +- Fixed redundant issuance of stdin detection warnings on some rare cases due to underlying implementation. ([#1303](https://github.com/httpie/httpie/pull/1303)) - Improved regulation of top-level arrays. ([#1292](https://github.com/httpie/httpie/commit/225dccb2186f14f871695b6c4e0bfbcdb2e3aa28)) - Double `--quiet` flags will now suppress all python level warnings. ([#1271](https://github.com/httpie/httpie/issues/1271)) From 98688b2f2dec7d0f3fb78a5d14c41e94dce58cb3 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 7 Mar 2022 15:51:51 +0300 Subject: [PATCH 24/36] Style fix on the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a05cbc9ca..fa83b97232 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285)) - Fixed displaying of status code without a status message on non-`auto` themes. ([#1300](https://github.com/httpie/httpie/issues/1300)) -- Fixed redundant issuance of stdin detection warnings on some rare cases due to underlying implementation. ([#1303](https://github.com/httpie/httpie/pull/1303)) +- Fixed redundant issuance of stdin detection warnings on some rare cases due to underlying implementation. ([#1303](https://github.com/httpie/httpie/pull/1303)) - Improved regulation of top-level arrays. ([#1292](https://github.com/httpie/httpie/commit/225dccb2186f14f871695b6c4e0bfbcdb2e3aa28)) - Double `--quiet` flags will now suppress all python level warnings. ([#1271](https://github.com/httpie/httpie/issues/1271)) From 15013fd609db6b23551a4691429a61d8e2632fe2 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 1 Mar 2022 17:16:37 +0300 Subject: [PATCH 25/36] Implement support for private key passphrases --- CHANGELOG.md | 1 + docs/README.md | 15 +++++++ httpie/cli/argparser.py | 17 ++++++- httpie/cli/argtypes.py | 33 ++++++++++---- httpie/cli/definition.py | 13 +++++- httpie/client.py | 11 ++++- httpie/ssl_.py | 33 ++++++++++++++ .../password_protected/client.key | 42 +++++++++++++++++ .../password_protected/client.pem | 26 +++++++++++ tests/test_ssl.py | 45 +++++++++++++++++++ 10 files changed, 223 insertions(+), 13 deletions(-) create mode 100644 tests/client_certs/password_protected/client.key create mode 100644 tests/client_certs/password_protected/client.pem diff --git a/CHANGELOG.md b/CHANGELOG.md index fa83b97232..77a1d1f9a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [3.0.3.dev0](https://github.com/httpie/httpie/compare/3.0.2...HEAD) (Unreleased) +- Added support for specifying certificate private key passphrases through `--cert-key-pass` and prompts. ([#946](https://github.com/httpie/httpie/issues/946)) - Fixed escaping of integer indexes with multiple backslashes in the nested JSON builder. ([#1285](https://github.com/httpie/httpie/issues/1285)) - Fixed displaying of status code without a status message on non-`auto` themes. ([#1300](https://github.com/httpie/httpie/issues/1300)) - Fixed redundant issuance of stdin detection warnings on some rare cases due to underlying implementation. ([#1303](https://github.com/httpie/httpie/pull/1303)) diff --git a/docs/README.md b/docs/README.md index b5d139aad9..00aff4b842 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1489,6 +1489,21 @@ path of the key file with `--cert-key`: $ http --cert=client.crt --cert-key=client.key https://example.org ``` +If the given private key requires a passphrase, HTTPie will automatically detect it +and ask it through a prompt: + +```bash +$ http --cert=client.pem --cert-key=client.key https://example.org +http: passphrase for client.key: **** +``` + +If you don't want to see a prompt, you can supply the passphrase with the `--cert-key-pass` +argument: + +```bash +$ http --cert=client.pem --cert-key=client.key --cert-key-pass=my_password https://example.org +``` + ### SSL version Use the `--ssl=` option to specify the desired protocol version to use. diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index b1ab8de155..f9d6674b7e 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -10,7 +10,8 @@ from requests.utils import get_netrc_auth from .argtypes import ( - AuthCredentials, KeyValueArgType, PARSED_DEFAULT_FORMAT_OPTIONS, + AuthCredentials, SSLCredentials, KeyValueArgType, + PARSED_DEFAULT_FORMAT_OPTIONS, parse_auth, parse_format_options, ) @@ -148,6 +149,7 @@ def parse_args( self._parse_items() self._process_url() self._process_auth() + self._process_ssl_cert() if self.args.raw is not None: self._body_from_input(self.args.raw) @@ -236,6 +238,19 @@ def _setup_standard_streams(self): self.env.stdout = self.env.devnull self.env.apply_warnings_filter() + def _process_ssl_cert(self): + from httpie.ssl_ import _is_key_file_encrypted + + if self.args.cert_key_pass is None: + self.args.cert_key_pass = SSLCredentials(None) + + if ( + self.args.cert_key is not None + and self.args.cert_key_pass.value is None + and _is_key_file_encrypted(self.args.cert_key) + ): + self.args.cert_key_pass.prompt_password(self.args.cert_key) + def _process_auth(self): # TODO: refactor & simplify this method. self.args.auth_plugin = None diff --git a/httpie/cli/argtypes.py b/httpie/cli/argtypes.py index 7bc487ea81..8f19c3c51e 100644 --- a/httpie/cli/argtypes.py +++ b/httpie/cli/argtypes.py @@ -130,16 +130,11 @@ def tokenize(self, s: str) -> List[Union[str, Escaped]]: return tokens -class AuthCredentials(KeyValueArg): - """Represents parsed credentials.""" - - def has_password(self) -> bool: - return self.value is not None - - def prompt_password(self, host: str): - prompt_text = f'http: password for {self.key}@{host}: ' +class PromptMixin: + def _prompt_password(self, prompt: str) -> str: + prompt_text = f'http: {prompt}: ' try: - self.value = self._getpass(prompt_text) + return self._getpass(prompt_text) except (EOFError, KeyboardInterrupt): sys.stderr.write('\n') sys.exit(0) @@ -150,6 +145,26 @@ def _getpass(prompt): return getpass.getpass(str(prompt)) +class SSLCredentials(PromptMixin): + """Represents the passphrase for the certificate's key.""" + + def __init__(self, value: Optional[str]) -> None: + self.value = value + + def prompt_password(self, key_file: str) -> None: + self.value = self._prompt_password(f'passphrase for {key_file}') + + +class AuthCredentials(KeyValueArg, PromptMixin): + """Represents parsed credentials.""" + + def has_password(self) -> bool: + return self.value is not None + + def prompt_password(self, host: str) -> None: + self.value = self._prompt_password(f'password for {self.key}@{host}:') + + class AuthCredentialsArgType(KeyValueArgType): """A key-value arg type that parses credentials.""" diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 5ccc3a16f9..0a0efafa6f 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -8,7 +8,7 @@ from .. import __doc__, __version__ from .argparser import HTTPieArgumentParser from .argtypes import ( - KeyValueArgType, SessionNameValidator, + KeyValueArgType, SessionNameValidator, SSLCredentials, readable_file_arg, response_charset_type, response_mime_type, ) from .constants import ( @@ -803,6 +803,17 @@ def format_auth_help(auth_plugins_mapping): ''' ) +ssl.add_argument( + '--cert-key-pass', + default=None, + type=SSLCredentials, + help=''' + The passphrase to be used to with the given private key. Only needed if --cert-key + is given and the key file requires a passphrase. + + ''' +) + ####################################################################### # Troubleshooting ####################################################################### diff --git a/httpie/client.py b/httpie/client.py index 06235d249b..1984537c2b 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -19,7 +19,7 @@ from .models import RequestsMessage from .plugins.registry import plugin_manager from .sessions import get_httpie_session -from .ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter +from .ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieCertificate, HTTPieHTTPSAdapter from .uploads import ( compress_request, prepare_request_body, get_multipart_data_and_content_type, @@ -262,7 +262,14 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict: if args.cert: cert = args.cert if args.cert_key: - cert = cert, args.cert_key + # Having a client certificate key passphrase is not supported + # by requests. So we are using our own transportation structure + # which is compatible with their format (a tuple of minimum two + # items). + # + # See: https://github.com/psf/requests/issues/2519 + cert = HTTPieCertificate(cert, args.cert_key, args.cert_key_pass.value) + return { 'proxies': {p.key: p.value for p in args.proxy}, 'stream': True, diff --git a/httpie/ssl_.py b/httpie/ssl_.py index cdec18f87b..b9438543eb 100644 --- a/httpie/ssl_.py +++ b/httpie/ssl_.py @@ -1,4 +1,5 @@ import ssl +from typing import NamedTuple, Optional from httpie.adapters import HTTPAdapter # noinspection PyPackageRequirements @@ -24,6 +25,17 @@ } +class HTTPieCertificate(NamedTuple): + cert_file: Optional[str] = None + key_file: Optional[str] = None + key_password: Optional[str] = None + + def to_raw_cert(self): + """Synthesize a requests-compatible (2-item tuple of cert and key file) + object from HTTPie's internal representation of a certificate.""" + return (self.cert_file, self.key_file) + + class HTTPieHTTPSAdapter(HTTPAdapter): def __init__( self, @@ -47,6 +59,13 @@ def proxy_manager_for(self, *args, **kwargs): kwargs['ssl_context'] = self._ssl_context return super().proxy_manager_for(*args, **kwargs) + def cert_verify(self, conn, url, verify, cert): + if isinstance(cert, HTTPieCertificate): + conn.key_password = cert.key_password + cert = cert.to_raw_cert() + + return super().cert_verify(conn, url, verify, cert) + @staticmethod def _create_ssl_context( verify: bool, @@ -61,3 +80,17 @@ def _create_ssl_context( # in `super().cert_verify()`. cert_reqs=ssl.CERT_REQUIRED if verify else ssl.CERT_NONE ) + + +def _is_key_file_encrypted(key_file): + """Detects if a key file is encrypted or not. + + Copy of the internal urllib function (urllib3.util.ssl_)""" + + with open(key_file, "r") as f: + for line in f: + # Look for Proc-Type: 4,ENCRYPTED + if "ENCRYPTED" in line: + return True + + return False diff --git a/tests/client_certs/password_protected/client.key b/tests/client_certs/password_protected/client.key new file mode 100644 index 0000000000..1634352f90 --- /dev/null +++ b/tests/client_certs/password_protected/client.key @@ -0,0 +1,42 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,93DA845817852FB576163AA829C232E9 + +VauvxiyC0lQbLJFoEGlnIIZO2/+b66DjTwSccqSdVg+Zaxvbc0jeVhS43SQ01ft0 +hB/oISgJB/1I/oKbGwx07T9j78Q8G9AxQV6hzvozK5Etjew4RrvV4DYyOSzwZNQr +qB9S0qhBKyemA2vx4aH/8nazHh+zrRD3y0oMbuCHLxSGuqncNXIKCTYgMb8NUucJ +fEArYHijZ0iotoOEpP31JOUPCpKhEewQxzNK0HLws0lv6nl6fmBlkdi603qmsG5U +uinuiGodrh9SpCUc/A4OhVWKwoiQSxGnz+SiNaXyUByf9CR8RLPWqi5pTGHC8xrJ +uHI6Cw8ZfrJ2clYtuCWv6g6c4F7sz6eAJHqCZNnU32kKu3uH/9E/7Z8uH7JOVyFa +9DlBHCWHdyaHs8mY+/pDcxeMyWeC837sBelIBF1iEwU/sMw43HipZBNhrekMLAkx +y5HRYQDstTvk1Nvj8fKysYuhGCiF/V6PWYo5RaQszZLhS+uyFEBwa0ojYNZh4LyB +5uIdBaqtL9FD4RXqTYfN96eEyoYaUUY5KXqQMZkuZpotGYmH9OGMTVCgR7eU0a62 +dgbQw4UCQd4YTNx1PyboH72oIi+Rqp2LEYEQSHP/dIUtBiA/kmWhgapZVGvfJ+fF +u9MPgPUDvH3oLVm4Mr+biLX/oUQVEup85q8++E2csDe2HoC4JdmJ0D9rZM2OqpYV +YZAPcPhx2pYnK5d6RvMFwtLPNfHxgYQXMVg6BFtu5GCxxqr+dhF7TGrN5s6AKC8U +bkVQIXwO8bYVTLj2Sb44fe+Xl1X/09yHnkZC0u/Kb2KvUm7Gnltn3tUmj7fGI0I6 +aI6G3T1xc0jz9WhjdnM3uDYYI66GpgRgv81n7IkfRjclNArW4OStf30K4pXXjGeP +vgopPJ1yNpaM4QNbx3cqzP0eBy+Ss7aCXca4I3BzjXtuo9ZcEzGb+1FkS7ASEdex +cAroJOmm9KJ+3KOxsVs5fxXtQqzzeD8cdZeGV0eckJNfjWSBH2zyhaxwdlCvG1I9 +dTvdd6q31FjlnUq9SvGEkfoy4myIUtt4DJQ4lSktvKQv9qepUjoX0k3xipgSmiPO +yxE+VdJdJ9/tDUf3psD01XLIss7hOX9aED3svN3uXB2ZVCSH6e2l4IrBMirdKNwR +fB4Yrul0qt9knmn11p2aWav055hb1Il5Tm8/WnaXkgtr20zP4RgR7P19mSjTBxUm +7iUIiWqU43Sx2LWsYpg7Lbj5XGLcvxv5WjYsE4Km0ltZCLKzMHfQ76qv4ZOQkHcR +9UevRmzU45095eASztedrYyxDNwU6YSdUcOYTP6383G9azbStlQY+w2Em++UoNoH +3eYj6KHKx+hkZOdc8PLaLg2f98jOiADpKYJTGnkKoLjTCfr9nzBeNxwRCQ4F4vO/ ++tuRo3i1ODpJQbbZys9Mz+9PSwBH31UAib0+v0GYLDJN2rJcyGal/0DH5zON9Ogi +5bZQ9oS91p9K5hUAnHpd3zOzeX1lCoZnmtOI8wah79SVSpK1xoE6BAxAHfRiYiS3 +1tDmkThJBOGXmkpLjtgNW3MqYKBnO3tRzrDDCjTKi5jFX/SD2FPpExOyA2+I0lrr +a9b+Sjbl1Z7B1yZmmTGMKB7prwK00LaF6yqKOhE+bx1yJAaWrbdPCD6vDmbq5YV6 +87woIiA16Q2I1x77/Kg3TDO9LMDiwI5BFyjR+4Q5SvufIaxtsmTBuaBuPif+f4DT +MPQcfk5ozQIKY4qiSqMAOXAf2t+/UQROjgYvayRz0fOv2rV0vS4i9ELj/8Dn65Dq +7aQzLwM0psToGIVyzAV+hF3jeQP+Xu7VjtSxTJ+ajz7PeIXeBH/mwJKMk7hpRwGj +4fZ92S00Iat2kA6wn55u6EGewgcaQrN2zr75a9gvXQwMDmsjszq2uWWxxJg6pAPZ +rNqhM9tJ2UAJ1lLZzUDfhK4wU4pGWIhT+BmdDgJ40hI4b1WEmKSTxsj8AYNcVDRf +i2Ox1QhZQX9bH5kTOX373/6cALFR5DcU8qh2FJtf+3uiZHNloEeID//H2Gdoxz0Y +5CC/VDiIa4Gj4D+ATsLMgTDt4eUOinMeC1H6w+QBd9UvceqEvrgu+1WB8UCK/Hm/ +7fZ0srsGg/WRqdSuO8/7998PEHgP8+wnTbxi9Y3EEbkaKUL6esJfeOjBibuGPyaf +2Y9QLcpVKaD7pmVeb97qExZZjEiID6QYmFUO8j0koS2fki0l+z8XEZ3JLZKa9XS+ +uiMPQKg41j+9ZrGmwPNj7brjwA0cdSb4CLgxg4FwuwB660XaXpW3aRsiRryi0YcM +hn2l6b4JgBz8gUkFiTXQ8wRvAKDC1hUkUysqCAC+Yg3cWxlDZVeSeqVGr5jhHgN1 +-----END RSA PRIVATE KEY----- diff --git a/tests/client_certs/password_protected/client.pem b/tests/client_certs/password_protected/client.pem new file mode 100644 index 0000000000..08ee652254 --- /dev/null +++ b/tests/client_certs/password_protected/client.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEazCCAtOgAwIBAgIUIWojJySwCenyvcJrZY1aKOMQfTUwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAzMDExMzM5MzRaFw00OTA3 +MTcxMzM5MzRaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggGiMA0GCSqGSIb3DQEB +AQUAA4IBjwAwggGKAoIBgQC0+8GOIhknLgLmWEQGgdDJffntbDfGdtG7JFUFfvuN +sibTHL4RPNhe+UrT/LR+JBfIfmdeb+NlmKzsTeR0+8PkX5ZjXMShf5icghVukK7G +OoQS7olQqlGzpIX76VqktVr4tFNXmMJeBO0NIaXHm0ASsoz3fIfDx9ttJETPs6lm +Wv/PUPemvtUgcbAb+kjz9QqcUV8B1xcCvQma6NSpxcmJHqAuI6HkdbDzyedKuuSi +M6yNFjh3EJjsufReQgkcfDsqh+RA3zQoIyPXLNqjzGD71z24jUtvIxb5ZNEtv0xp +5zCOCavuRNNyKGvvnlIeyup7bMe0QIds566miG49osVpPVvVmg+q+w2YYAE+7svb +nJp7NYn2tryRqsmvnASLVQD6T9wTWUa8w/tT1+ltnhfqbwDcVACzsw/U4FFwcfWw +5BnUcJacoDkj/3TCqgkA8XFe1/DVU8XCcsvEaoLzwHhHu2+QDpqal8rNouyTpFGA +/wioVBQGpksPZjl8lumsz3kCAwEAAaNTMFEwHQYDVR0OBBYEFGJhl1BPOXCVqRo3 +U/ruuedvlDqsMB8GA1UdIwQYMBaAFGJhl1BPOXCVqRo3U/ruuedvlDqsMA8GA1Ud +EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggGBAE9NtrE5qSHlK9DXdH6bPW6z +GO9uWBl3rJVtqVvPoH8RxJG/jtaD/Pnc3MkIxMoliCleNK6BdYVGV9u8x9W1jQo8 +H+mnH3/Ise8ZY1zpblZJF0z9xs5sWW7qO8U06GmJWRSPn3LKEZjLsNmThhUW09wN +8EZX914zCWtzCrUTNg8Au1Dz9zA9ScfpCVPhKORTCnrpoTL6iXsPxmCx+5awmNLE +uh9kw4NScEyq33RTPosMpwSMlXGRuASltx/J7Rn0DNR0r1p0XzDS4CG1iDwXHlEF +MwsOvSahNyz5RInrU3cgN70tafoRIHScLYycnRml8dydxrDoFgdJk5sI4zgq24Sg +TktTq9ShrT4yQX+lrGS6eZQK/YZEBPD7BdTLYp3vlfYQMJ4Jz9SyQ8b9/9jIFVFS +dFfWiCqEuhTvGfptAzYX+K9OaegZnIk3X7R6O+YQ3oHCbLbnV3bpKlgNnOKBwa2X +kJ5GRp+rZOJ97yjrspKjpR5tNCiJnp7NnnA5VA6mfw== +-----END CERTIFICATE----- diff --git a/tests/test_ssl.py b/tests/test_ssl.py index f930bf2826..fc587064d7 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -6,6 +6,8 @@ import requests.exceptions import urllib3 +from unittest import mock + from httpie.ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS from httpie.status import ExitStatus @@ -32,6 +34,15 @@ CLIENT_KEY = str(CERTS_ROOT / 'client.key') CLIENT_PEM = str(CERTS_ROOT / 'client.pem') +# In case of a regeneration, use the following commands +# in the PWD_TESTS_ROOT: +# $ openssl genrsa -aes128 -passout pass:password 3072 > client.pem +# $ openssl req -new -x509 -nodes -days 10000 -key client.pem > client.pem +PWD_TESTS_ROOT = CERTS_ROOT / 'password_protected' +PWD_CLIENT_PEM = str(PWD_TESTS_ROOT / 'client.pem') +PWD_CLIENT_KEY = str(PWD_TESTS_ROOT / 'client.key') +PWD_CLIENT_PASS = 'password' +PWD_CLIENT_INVALID_PASS = PWD_CLIENT_PASS + 'invalid' # We test against a local httpbin instance which uses a self-signed cert. # Requests without --verify= will fail with a verification error. @@ -165,3 +176,37 @@ def test_pyopenssl_presence(): else: assert urllib3.util.ssl_.IS_PYOPENSSL assert urllib3.util.IS_PYOPENSSL + + +@mock.patch('httpie.cli.argtypes.SSLCredentials._prompt_password', + new=lambda self, prompt: PWD_CLIENT_PASS) +def test_password_protected_cert_prompt(httpbin_secure): + r = http(httpbin_secure + '/get', + '--cert', PWD_CLIENT_PEM, + '--cert-key', PWD_CLIENT_KEY) + assert HTTP_OK in r + + +@mock.patch('httpie.cli.argtypes.SSLCredentials._prompt_password', + new=lambda self, prompt: PWD_CLIENT_INVALID_PASS) +def test_password_protected_cert_prompt_invalid(httpbin_secure): + with pytest.raises(ssl_errors): + http(httpbin_secure + '/get', + '--cert', PWD_CLIENT_PEM, + '--cert-key', PWD_CLIENT_KEY) + + +def test_password_protected_cert_cli_arg(httpbin_secure): + r = http(httpbin_secure + '/get', + '--cert', PWD_CLIENT_PEM, + '--cert-key', PWD_CLIENT_KEY, + '--cert-key-pass', PWD_CLIENT_PASS) + assert HTTP_OK in r + + +def test_password_protected_cert_cli_arg_invalid(httpbin_secure): + with pytest.raises(ssl_errors): + http(httpbin_secure + '/get', + '--cert', PWD_CLIENT_PEM, + '--cert-key', PWD_CLIENT_KEY, + '--cert-key-pass', PWD_CLIENT_INVALID_PASS) From 9241a093605cf6afbd6a52b42db4c946badea420 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 7 Mar 2022 16:05:13 +0300 Subject: [PATCH 26/36] Mention about interactive prompt on key passphrases --- httpie/cli/definition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 0a0efafa6f..806b04bc60 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -810,7 +810,7 @@ def format_auth_help(auth_plugins_mapping): help=''' The passphrase to be used to with the given private key. Only needed if --cert-key is given and the key file requires a passphrase. - + If not provided, you’ll be prompted interactively. ''' ) From 350abe30338ffb3fd29129ec12a94ddd69b64080 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 8 Feb 2022 12:13:17 +0300 Subject: [PATCH 27/36] Make the naked invocation display a compacted help --- CHANGELOG.md | 1 + httpie/cli/argparser.py | 53 ++++++++++++++++++++++++-- tests/test_cli_ui.py | 84 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 tests/test_cli_ui.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 77a1d1f9a9..7ecf797d29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Fixed displaying of status code without a status message on non-`auto` themes. ([#1300](https://github.com/httpie/httpie/issues/1300)) - Fixed redundant issuance of stdin detection warnings on some rare cases due to underlying implementation. ([#1303](https://github.com/httpie/httpie/pull/1303)) - Improved regulation of top-level arrays. ([#1292](https://github.com/httpie/httpie/commit/225dccb2186f14f871695b6c4e0bfbcdb2e3aa28)) +- Improved UI layout for standalone invocations. ([#1296](https://github.com/httpie/httpie/pull/1296)) - Double `--quiet` flags will now suppress all python level warnings. ([#1271](https://github.com/httpie/httpie/issues/1271)) ## [3.0.2](https://github.com/httpie/httpie/compare/3.0.1...3.0.2) (2022-01-24) diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index f9d6674b7e..c632774d5d 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -48,12 +48,39 @@ def _split_lines(self, text, width): text = dedent(text).strip() + '\n\n' return text.splitlines() + def add_usage(self, usage, actions, groups, prefix=None): + # Only display the positional arguments + displayed_actions = [ + action + for action in actions + if not action.option_strings + ] + + _, exception, _ = sys.exc_info() + if ( + isinstance(exception, argparse.ArgumentError) + and len(exception.args) >= 1 + and isinstance(exception.args[0], argparse.Action) + ): + # add_usage path is also taken when you pass an invalid option, + # e.g --style=invalid. If something like that happens, we want + # to include to action that caused to the invalid usage into + # the list of actions we are displaying. + displayed_actions.insert(0, exception.args[0]) + + super().add_usage( + usage, + displayed_actions, + groups, + prefix="usage:\n " + ) + # TODO: refactor and design type-annotated data structures # for raw args + parsed args and keep things immutable. class BaseHTTPieArgumentParser(argparse.ArgumentParser): - def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs): - super().__init__(*args, formatter_class=formatter_class, **kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.env = None self.args = None self.has_stdin_data = False @@ -116,9 +143,9 @@ class HTTPieArgumentParser(BaseHTTPieArgumentParser): """ - def __init__(self, *args, **kwargs): + def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs): kwargs.setdefault('add_help', False) - super().__init__(*args, **kwargs) + super().__init__(*args, formatter_class=formatter_class, **kwargs) # noinspection PyMethodOverriding def parse_args( @@ -529,3 +556,21 @@ def _process_format_options(self): for options_group in format_options: parsed_options = parse_format_options(options_group, defaults=parsed_options) self.args.format_options = parsed_options + + def error(self, message): + """Prints a usage message incorporating the message to stderr and + exits.""" + self.print_usage(sys.stderr) + self.exit( + 2, + dedent( + f''' + error: + {message} + + For more information: + - Try running {self.prog} --help + - Or visiting https://httpie.io/docs/cli + ''' + ) + ) diff --git a/tests/test_cli_ui.py b/tests/test_cli_ui.py new file mode 100644 index 0000000000..35faf37f3e --- /dev/null +++ b/tests/test_cli_ui.py @@ -0,0 +1,84 @@ +import pytest +import shutil +import os +import sys +from tests.utils import http + + +if sys.version_info >= (3, 9): + REQUEST_ITEM_MSG = "[REQUEST_ITEM ...]" +else: + REQUEST_ITEM_MSG = "[REQUEST_ITEM [REQUEST_ITEM ...]]" + + +NAKED_HELP_MESSAGE = f"""\ +usage: + http [METHOD] URL {REQUEST_ITEM_MSG} + +error: + the following arguments are required: URL + +For more information: + - Try running http --help + - Or visiting https://httpie.io/docs/cli + +""" + +NAKED_HELP_MESSAGE_PRETTY_WITH_NO_ARG = f"""\ +usage: + http [--pretty {{all,colors,format,none}}] [METHOD] URL {REQUEST_ITEM_MSG} + +error: + argument --pretty: expected one argument + +For more information: + - Try running http --help + - Or visiting https://httpie.io/docs/cli + +""" + +NAKED_HELP_MESSAGE_PRETTY_WITH_INVALID_ARG = f"""\ +usage: + http [--pretty {{all,colors,format,none}}] [METHOD] URL {REQUEST_ITEM_MSG} + +error: + argument --pretty: invalid choice: '$invalid' (choose from 'all', 'colors', 'format', 'none') + +For more information: + - Try running http --help + - Or visiting https://httpie.io/docs/cli + +""" + + +PREDEFINED_TERMINAL_SIZE = (160, 80) + + +@pytest.fixture(scope="function") +def ignore_terminal_size(monkeypatch): + """Some tests wrap/crop the output depending on the + size of the executed terminal, which might not be consistent + through all runs. + + This fixture ensures every run uses the same exact configuration. + """ + + def fake_terminal_size(*args, **kwargs): + return os.terminal_size(PREDEFINED_TERMINAL_SIZE) + + # Setting COLUMNS as an env var is required for 3.8< + monkeypatch.setitem(os.environ, 'COLUMNS', str(PREDEFINED_TERMINAL_SIZE[0])) + monkeypatch.setattr(shutil, 'get_terminal_size', fake_terminal_size) + + +@pytest.mark.parametrize( + 'args, expected_msg', [ + ([], NAKED_HELP_MESSAGE), + (['--pretty'], NAKED_HELP_MESSAGE_PRETTY_WITH_NO_ARG), + (['pie.dev', '--pretty'], NAKED_HELP_MESSAGE_PRETTY_WITH_NO_ARG), + (['--pretty', '$invalid'], NAKED_HELP_MESSAGE_PRETTY_WITH_INVALID_ARG), + ] +) +def test_naked_invocation(ignore_terminal_size, args, expected_msg): + result = http(*args, tolerate_error_exit_status=True) + assert result.stderr == expected_msg From ec203b1face4fb7e606a60880b6e80b4051bc41c Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 7 Mar 2022 16:52:04 +0100 Subject: [PATCH 28/36] Tweak compact help --- httpie/cli/argparser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index c632774d5d..a312b8ba10 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -568,9 +568,8 @@ def error(self, message): error: {message} - For more information: - - Try running {self.prog} --help - - Or visiting https://httpie.io/docs/cli + for more information: + run '{self.prog} --help' or visit https://httpie.io/docs/cli ''' ) ) From b5623ccc8797312de60e4ca37dea249fa0a1ea39 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 7 Mar 2022 19:01:37 +0300 Subject: [PATCH 29/36] Fix the tests with the latest layout --- tests/test_cli_ui.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_cli_ui.py b/tests/test_cli_ui.py index 35faf37f3e..760077a47f 100644 --- a/tests/test_cli_ui.py +++ b/tests/test_cli_ui.py @@ -18,9 +18,8 @@ error: the following arguments are required: URL -For more information: - - Try running http --help - - Or visiting https://httpie.io/docs/cli +for more information: + run 'http --help' or visit https://httpie.io/docs/cli """ @@ -31,9 +30,8 @@ error: argument --pretty: expected one argument -For more information: - - Try running http --help - - Or visiting https://httpie.io/docs/cli +for more information: + run 'http --help' or visit https://httpie.io/docs/cli """ @@ -44,9 +42,8 @@ error: argument --pretty: invalid choice: '$invalid' (choose from 'all', 'colors', 'format', 'none') -For more information: - - Try running http --help - - Or visiting https://httpie.io/docs/cli +for more information: + run 'http --help' or visit https://httpie.io/docs/cli """ From 65ab7d5caaaf2f95e61f9dd65441801c2ddee38b Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 1 Feb 2022 12:14:24 +0300 Subject: [PATCH 30/36] Implement new style cookies --- docs/README.md | 140 ++++++++++ httpie/client.py | 6 +- httpie/config.py | 60 ++-- httpie/manager/cli.py | 43 ++- httpie/manager/core.py | 11 + httpie/manager/tasks.py | 134 +++++++++ httpie/sessions.py | 206 +++++++++++--- httpie/utils.py | 5 + setup.py | 1 + tests/conftest.py | 6 +- tests/fixtures/__init__.py | 24 ++ .../session_data/new/cookies_dict.json | 31 +++ .../new/cookies_dict_dev_version.json | 31 +++ .../new/cookies_dict_with_extras.json | 33 +++ .../session_data/new/empty_cookies_dict.json | 14 + .../session_data/new/empty_cookies_list.json | 14 + .../session_data/old/cookies_dict.json | 27 ++ .../old/cookies_dict_dev_version.json | 27 ++ .../old/cookies_dict_with_extras.json | 29 ++ .../session_data/old/empty_cookies_dict.json | 14 + .../session_data/old/empty_cookies_list.json | 14 + tests/test_cookie_on_redirects.py | 262 ++++++++++++++++++ tests/test_httpie_cli.py | 125 +++++++++ tests/test_plugins_cli.py | 43 --- tests/test_sessions.py | 186 ++++++++++++- tests/utils/__init__.py | 24 +- tests/utils/http_server.py | 13 + 27 files changed, 1406 insertions(+), 117 deletions(-) create mode 100644 httpie/manager/tasks.py create mode 100644 tests/fixtures/session_data/new/cookies_dict.json create mode 100644 tests/fixtures/session_data/new/cookies_dict_dev_version.json create mode 100644 tests/fixtures/session_data/new/cookies_dict_with_extras.json create mode 100644 tests/fixtures/session_data/new/empty_cookies_dict.json create mode 100644 tests/fixtures/session_data/new/empty_cookies_list.json create mode 100644 tests/fixtures/session_data/old/cookies_dict.json create mode 100644 tests/fixtures/session_data/old/cookies_dict_dev_version.json create mode 100644 tests/fixtures/session_data/old/cookies_dict_with_extras.json create mode 100644 tests/fixtures/session_data/old/empty_cookies_dict.json create mode 100644 tests/fixtures/session_data/old/empty_cookies_list.json create mode 100644 tests/test_cookie_on_redirects.py create mode 100644 tests/test_httpie_cli.py diff --git a/docs/README.md b/docs/README.md index 00aff4b842..efd579a343 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2157,6 +2157,85 @@ $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:orig- $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:new-value ``` +### Host-based Cookie Policy + +Cookies in stored HTTPie sessions have a `domain` field which is binding them to the +specified hostname. For example, in the following session: + +```json +{ + "cookies": [ + { + "domain": "pie.dev", + "name": "secret_cookie", + "value": "value_1" + }, + { + "domain": "httpbin.org", + "name": "secret_cookie", + "value": "value_2" + } + ] +} +``` + +we will send `Cookie:secret_cookie=value_1` only when you are making a request against `pie.dev` (it +also includes the domains, like `api.pie.dev`), and `Cookie:secret_cookie=value_2` when you use `httpbin.org`. + +```bash +$ http --session=./session.json pie.dev/cookies +``` + +```json +{ + "cookies": { + "secret_cookie": "value_1" + } +} +``` + +```bash +$ http --session=./session.json httpbin.org/cookies +``` + +```json +{ + "cookies": { + "secret_cookie": "value_2" + } +} +``` + +If you want to make a cookie domain unbound, you can simply set the `domain` +field to `null` by editing the session file directly: + +```json +{ + "cookies": [ + { + "domain": null, + "expires": null, + "name": "generic_cookie", + "path": "/", + "secure": false, + "value": "generic_value" + } + ] +} +``` + +```bash +$ http --session=./session.json pie.dev/cookies +``` + +```json +{ + "cookies": { + "generic_cookie": "generic_value" + } +} +``` + ### Cookie Storage Behavior **TL;DR:** Cookie storage priority: Server response > Command line request > Session file @@ -2208,6 +2287,50 @@ Expired cookies are never stored. If a cookie in a session file expires, it will be removed before sending a new request. If the server expires an existing cookie, it will also be removed from the session file. +### Upgrading Sessions + +In rare circumstances, HTTPie makes changes in it's session layout. For allowing a smoother transition of existing files +from the old layout to the new layout we offer 2 interfaces: + +- `httpie cli sessions upgrade` +- `httpie cli sessions upgrade-all` + + +With `httpie cli sessions upgrade`, you can upgrade a single session with it's name (or it's path, if it is an +[anonymous session](#anonymous-sessions)) and the hostname it belongs to. For example: + +([named session](#named-sessions)) + +```bash +$ httpie cli sessions upgrade pie.dev api_auth +Refactored 'api_auth' (for 'pie.dev') to the version 3.1.0. +``` + +([anonymous session](#anonymous-sessions)) + +```bash +$ httpie cli sessions upgrade pie.dev ./session.json +Refactored 'session' (for 'pie.dev') to the version 3.1.0. +``` + +If you want to upgrade every existing [named session](#named-sessions), you can use `httpie cli sessions upgrade-all` (be aware +that this won't upgrade [anonymous sessions](#anonymous-sessions)): + +```bash +$ httpie cli sessions upgrade-all +Refactored 'api_auth' (for 'pie.dev') to the version 3.1.0. +Refactored 'login_cookies' (for 'httpie.io') to the version 3.1.0. +``` + +#### Additional Customizations + +| Flag | Description | +|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--bind-cookies` | Bind all the unbound cookies to the hostname that session belongs. By default, if the cookie is unbound (the `domain` attribute does not exist / set to an empty string) then it will still continue to be a generic cookie. | + +These flags can be used to customize the defaults during an `upgrade` operation. They can +be used in both `sessions upgrade` and `sessions upgrade-all`. + ## Config HTTPie uses a simple `config.json` file. @@ -2299,6 +2422,23 @@ And since there’s neither data nor `EOF`, it will get stuck. So unless you’r Also, it might be good to set a connection `--timeout` limit to prevent your program from hanging if the server never responds. +### Security + +#### Exposure of Cookies To The 3rd Party Hosts On Redirects + +*Vulnerability Type*: [CWE-200](https://cwe.mitre.org/data/definitions/200.html) +*Severity Level*: LOW +*Affected Versions*: `<3.1.0` + +The handling of [cookies](#cookies) was not compatible with the [RFC 6265](https://datatracker.ietf.org/doc/html/rfc6265) +on the point of handling the `Domain` attribute when they were saved into [session](#sessions) files. All cookies were shared +across all hosts during the runtime, including redirects to the 3rd party hosts. + +This vulnerability has been fixed in [3.1.0](https://github.com/httpie/httpie/releases/tag/3.1.0) and the +[`httpie cli sessions upgrade`](#upgrading-sessions)/[`httpie cli sessions upgrade-all`]((#upgrading-sessions) commands +have been put in place in order to allow a smooth transition to the new session layout from the existing [session](#sessions) +files. + ## Plugin manager HTTPie offers extensibility through a [plugin API](https://github.com/httpie/httpie/blob/master/httpie/plugins/base.py), diff --git a/httpie/client.py b/httpie/client.py index 1984537c2b..530d589cae 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -44,6 +44,7 @@ def collect_messages( httpie_session_headers = None if args.session or args.session_read_only: httpie_session = get_httpie_session( + env=env, config_dir=env.config.directory, session_name=args.session or args.session_read_only, host=args.headers.get('Host'), @@ -130,10 +131,7 @@ def collect_messages( if httpie_session: if httpie_session.is_new() or not args.session_read_only: httpie_session.cookies = requests_session.cookies - httpie_session.remove_cookies( - # TODO: take path & domain into account? - cookie['name'] for cookie in expired_cookies - ) + httpie_session.remove_cookies(expired_cookies) httpie_session.save() diff --git a/httpie/config.py b/httpie/config.py index 28574e4ae7..f7fee5bdab 100644 --- a/httpie/config.py +++ b/httpie/config.py @@ -1,7 +1,7 @@ import json import os from pathlib import Path -from typing import Union +from typing import Any, Dict, Union from . import __version__ from .compat import is_windows @@ -62,6 +62,21 @@ class ConfigFileError(Exception): pass +def read_raw_config(config_type: str, path: Path) -> Dict[str, Any]: + try: + with path.open(encoding=UTF8) as f: + try: + return json.load(f) + except ValueError as e: + raise ConfigFileError( + f'invalid {config_type} file: {e} [{path}]' + ) + except FileNotFoundError: + pass + except OSError as e: + raise ConfigFileError(f'cannot read {config_type} file: {e}') + + class BaseConfigDict(dict): name = None helpurl = None @@ -77,26 +92,25 @@ def ensure_directory(self): def is_new(self) -> bool: return not self.path.exists() + def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Hook for processing the incoming config data.""" + return data + + def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Hook for processing the outgoing config data.""" + return data + def load(self): config_type = type(self).__name__.lower() - try: - with self.path.open(encoding=UTF8) as f: - try: - data = json.load(f) - except ValueError as e: - raise ConfigFileError( - f'invalid {config_type} file: {e} [{self.path}]' - ) - self.update(data) - except FileNotFoundError: - pass - except OSError as e: - raise ConfigFileError(f'cannot read {config_type} file: {e}') - - def save(self): - self['__meta__'] = { - 'httpie': __version__ - } + data = read_raw_config(config_type, self.path) + if data is not None: + data = self.pre_process_data(data) + self.update(data) + + def save(self, *, bump_version: bool = False): + self.setdefault('__meta__', {}) + if bump_version or 'httpie' not in self['__meta__']: + self['__meta__']['httpie'] = __version__ if self.helpurl: self['__meta__']['help'] = self.helpurl @@ -106,13 +120,19 @@ def save(self): self.ensure_directory() json_string = json.dumps( - obj=self, + obj=self.post_process_data(self), indent=4, sort_keys=True, ensure_ascii=True, ) self.path.write_text(json_string + '\n', encoding=UTF8) + @property + def version(self): + return self.get( + '__meta__', {} + ).get('httpie', __version__) + class Config(BaseConfigDict): FILENAME = 'config.json' diff --git a/httpie/manager/cli.py b/httpie/manager/cli.py index 11c63d0a31..9ad4eca6ba 100644 --- a/httpie/manager/cli.py +++ b/httpie/manager/cli.py @@ -2,6 +2,15 @@ from httpie.cli.argparser import HTTPieManagerArgumentParser from httpie import __version__ +CLI_SESSION_UPGRADE_FLAGS = [ + { + 'variadic': ['--bind-cookies'], + 'action': 'store_true', + 'default': False, + 'help': 'Bind domainless cookies to the host that session belongs.' + } +] + COMMANDS = { 'plugins': { 'help': 'Manage HTTPie plugins.', @@ -34,6 +43,34 @@ 'List all installed HTTPie plugins.' ], }, + 'cli': { + 'help': 'Manage HTTPie for Terminal', + 'sessions': { + 'help': 'Manage HTTPie sessions', + 'upgrade': [ + 'Upgrade the given HTTPie session with the latest ' + 'layout. A list of changes between different session versions ' + 'can be found in the official documentation.', + { + 'dest': 'hostname', + 'metavar': 'HOSTNAME', + 'help': 'The host this session belongs.' + }, + { + 'dest': 'session', + 'metavar': 'SESSION_NAME_OR_PATH', + 'help': 'The name or the path for the session that will be upgraded.' + }, + *CLI_SESSION_UPGRADE_FLAGS + ], + 'upgrade-all': [ + 'Upgrade all named sessions with the latest layout. A list of ' + 'changes between different session versions can be found in the official ' + 'documentation.', + *CLI_SESSION_UPGRADE_FLAGS + ], + } + } } @@ -54,6 +91,8 @@ def generate_subparsers(root, parent_parser, definitions): ) for command, properties in definitions.items(): is_subparser = isinstance(properties, dict) + properties = properties.copy() + descr = properties.pop('help', None) if is_subparser else properties.pop(0) command_parser = actions.add_parser(command, description=descr) command_parser.root = root @@ -62,7 +101,9 @@ def generate_subparsers(root, parent_parser, definitions): continue for argument in properties: - command_parser.add_argument(**argument) + argument = argument.copy() + variadic = argument.pop('variadic', []) + command_parser.add_argument(*variadic, **argument) parser = HTTPieManagerArgumentParser( diff --git a/httpie/manager/core.py b/httpie/manager/core.py index e2134b5527..1289fef1a4 100644 --- a/httpie/manager/core.py +++ b/httpie/manager/core.py @@ -1,9 +1,11 @@ import argparse +from typing import Optional from httpie.context import Environment from httpie.manager.plugins import PluginInstaller from httpie.status import ExitStatus from httpie.manager.cli import missing_subcommand, parser +from httpie.manager.tasks import CLI_TASKS MSG_COMMAND_CONFUSION = '''\ This command is only for managing HTTPie plugins. @@ -22,6 +24,13 @@ '''.rstrip("\n").format(args='POST pie.dev/post hello=world') +def dispatch_cli_task(env: Environment, action: Optional[str], args: argparse.Namespace) -> ExitStatus: + if action is None: + parser.error(missing_subcommand('cli')) + + return CLI_TASKS[action](env, args) + + def program(args: argparse.Namespace, env: Environment) -> ExitStatus: if args.action is None: parser.error(MSG_NAKED_INVOCATION) @@ -29,5 +38,7 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus: if args.action == 'plugins': plugins = PluginInstaller(env, debug=args.debug) return plugins.run(args.plugins_action, args) + elif args.action == 'cli': + return dispatch_cli_task(env, args.cli_action, args) return ExitStatus.SUCCESS diff --git a/httpie/manager/tasks.py b/httpie/manager/tasks.py new file mode 100644 index 0000000000..c04ed9bc3d --- /dev/null +++ b/httpie/manager/tasks.py @@ -0,0 +1,134 @@ +import argparse +from typing import TypeVar, Callable, Tuple + +from httpie.sessions import SESSIONS_DIR_NAME, Session, get_httpie_session +from httpie.status import ExitStatus +from httpie.context import Environment +from httpie.manager.cli import missing_subcommand, parser + +T = TypeVar('T') + +CLI_TASKS = {} + + +def task(name: str) -> Callable[[T], T]: + def wrapper(func: T) -> T: + CLI_TASKS[name] = func + return func + return wrapper + + +@task('sessions') +def cli_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus: + action = args.cli_sessions_action + if action is None: + parser.error(missing_subcommand('cli', 'sessions')) + + if action == 'upgrade': + return cli_upgrade_session(env, args) + elif action == 'upgrade-all': + return cli_upgrade_all_sessions(env, args) + else: + raise ValueError(f'Unexpected action: {action}') + + +def is_version_greater(version_1: str, version_2: str) -> bool: + # In an ideal scenerio, we would depend on `packaging` in order + # to offer PEP 440 compatible parsing. But since it might not be + # commonly available for outside packages, and since we are only + # going to parse HTTPie's own version it should be fine to compare + # this in a SemVer subset fashion. + + def split_version(version: str) -> Tuple[int, ...]: + parts = [] + for part in version.split('.')[:3]: + try: + parts.append(int(part)) + except ValueError: + break + return tuple(parts) + + return split_version(version_1) > split_version(version_2) + + +def fix_cookie_layout(session: Session, hostname: str, args: argparse.Namespace) -> None: + if not isinstance(session['cookies'], dict): + return None + + session['cookies'] = [ + { + 'name': key, + **value + } + for key, value in session['cookies'].items() + ] + for cookie in session.cookies: + if cookie.domain == '': + if args.bind_cookies: + cookie.domain = hostname + else: + cookie._rest['is_explicit_none'] = True + + +FIXERS_TO_VERSIONS = { + '3.1.0': fix_cookie_layout +} + + +def upgrade_session(env: Environment, args: argparse.Namespace, hostname: str, session_name: str): + session = get_httpie_session( + env=env, + config_dir=env.config.directory, + session_name=session_name, + host=hostname, + url=hostname, + refactor_mode=True + ) + + session_name = session.path.stem + if session.is_new(): + env.log_error(f'{session_name!r} (for {hostname!r}) does not exist.') + return ExitStatus.ERROR + + fixers = [ + fixer + for version, fixer in FIXERS_TO_VERSIONS.items() + if is_version_greater(version, session.version) + ] + + if len(fixers) == 0: + env.stdout.write(f'{session_name!r} (for {hostname!r}) is already up-to-date.\n') + return ExitStatus.SUCCESS + + for fixer in fixers: + fixer(session, hostname, args) + + session.save(bump_version=True) + env.stdout.write(f'Refactored {session_name!r} (for {hostname!r}) to the version {session.version}.\n') + return ExitStatus.SUCCESS + + +def cli_upgrade_session(env: Environment, args: argparse.Namespace) -> ExitStatus: + return upgrade_session( + env, + args=args, + hostname=args.hostname, + session_name=args.session + ) + + +def cli_upgrade_all_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus: + session_dir_path = env.config_dir / SESSIONS_DIR_NAME + + status = ExitStatus.SUCCESS + for host_path in session_dir_path.iterdir(): + hostname = host_path.name + for session_path in host_path.glob("*.json"): + session_name = session_path.stem + status |= upgrade_session( + env, + args=args, + hostname=hostname, + session_name=session_name + ) + return status diff --git a/httpie/sessions.py b/httpie/sessions.py index 176c03e76d..c23cb56852 100644 --- a/httpie/sessions.py +++ b/httpie/sessions.py @@ -6,15 +6,17 @@ import re from http.cookies import SimpleCookie +from http.cookiejar import Cookie from pathlib import Path -from typing import Iterable, Optional, Union -from urllib.parse import urlsplit +from typing import Any, Dict, Optional, Union from requests.auth import AuthBase -from requests.cookies import RequestsCookieJar, create_cookie +from requests.cookies import RequestsCookieJar, remove_cookie_by_name +from .context import Environment from .cli.dicts import HTTPHeadersDict from .config import BaseConfigDict, DEFAULT_CONFIG_DIR +from .utils import url_as_host from .plugins.registry import plugin_manager @@ -26,27 +28,88 @@ # SESSION_IGNORED_HEADER_PREFIXES = ['Content-', 'If-'] +# Cookie related options +KEPT_COOKIE_OPTIONS = ['name', 'expires', 'path', 'value', 'domain', 'secure'] +DEFAULT_COOKIE_PATH = '/' + +INSECURE_COOKIE_JAR_WARNING = '''\ +Outdated layout detected for the current session. Please consider updating it, +in order to not get affected by potential security problems. + +For fixing the current session: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade --bind-cookies {hostname} {session_id} + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade {hostname} {session_id} +''' + +INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS = '''\ + +For fixing all named sessions: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade-all --bind-cookies + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade-all + +See https://pie.co/docs/security for more information. +''' + + +def is_anonymous_session(session_name: str) -> bool: + return os.path.sep in session_name + + +def materialize_cookie(cookie: Cookie) -> Dict[str, Any]: + materialized_cookie = { + option: getattr(cookie, option) + for option in KEPT_COOKIE_OPTIONS + } + + if ( + cookie._rest.get('is_explicit_none') + and materialized_cookie['domain'] == '' + ): + materialized_cookie['domain'] = None + + return materialized_cookie + def get_httpie_session( + env: Environment, config_dir: Path, session_name: str, host: Optional[str], url: str, + *, + refactor_mode: bool = False ) -> 'Session': - if os.path.sep in session_name: + bound_hostname = host or url_as_host(url) + if not bound_hostname: + # HACK/FIXME: httpie-unixsocket's URLs have no hostname. + bound_hostname = 'localhost' + + # host:port => host_port + hostname = bound_hostname.replace(':', '_') + if is_anonymous_session(session_name): path = os.path.expanduser(session_name) + session_id = path else: - hostname = host or urlsplit(url).netloc.split('@')[-1] - if not hostname: - # HACK/FIXME: httpie-unixsocket's URLs have no hostname. - hostname = 'localhost' - - # host:port => host_port - hostname = hostname.replace(':', '_') path = ( config_dir / SESSIONS_DIR_NAME / hostname / f'{session_name}.json' ) - session = Session(path) + session_id = session_name + + session = Session( + path, + env=env, + session_id=session_id, + bound_host=bound_hostname.split(':')[0], + refactor_mode=refactor_mode + ) session.load() return session @@ -55,15 +118,86 @@ class Session(BaseConfigDict): helpurl = 'https://httpie.io/docs#sessions' about = 'HTTPie session file' - def __init__(self, path: Union[str, Path]): + def __init__( + self, + path: Union[str, Path], + env: Environment, + bound_host: str, + session_id: str, + refactor_mode: bool = False, + ): super().__init__(path=Path(path)) self['headers'] = {} - self['cookies'] = {} + self['cookies'] = [] self['auth'] = { 'type': None, 'username': None, 'password': None } + self.env = env + self.cookie_jar = RequestsCookieJar() + self.session_id = session_id + self.bound_host = bound_host + self.refactor_mode = refactor_mode + + def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + cookies = data.get('cookies') + if isinstance(cookies, dict): + normalized_cookies = [ + { + 'name': key, + **value + } + for key, value in cookies.items() + ] + elif isinstance(cookies, list): + normalized_cookies = cookies + else: + normalized_cookies = [] + + should_issue_warning = False + for cookie in normalized_cookies: + domain = cookie.get('domain', '') + if domain == '' and isinstance(cookies, dict): + should_issue_warning = True + elif domain is None: + # domain = None means explicitly lack of cookie, though + # requests requires domain to be string so we'll cast it + # manually. + cookie['domain'] = '' + cookie['rest'] = {'is_explicit_none': True} + + self.cookie_jar.set(**cookie) + + if should_issue_warning and not self.refactor_mode: + warning = INSECURE_COOKIE_JAR_WARNING.format(hostname=self.bound_host, session_id=self.session_id) + if not is_anonymous_session(self.session_id): + warning += INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS + + self.env.log_error( + warning, + level='warning' + ) + + return data + + def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + cookies = data.get('cookies') + # Save in the old-style fashion + + normalized_cookies = [ + materialize_cookie(cookie) + for cookie in self.cookie_jar + ] + if isinstance(cookies, dict): + data['cookies'] = { + cookie.pop('name'): cookie + for cookie in normalized_cookies + } + else: + data['cookies'] = normalized_cookies + + return data def update_headers(self, request_headers: HTTPHeadersDict): """ @@ -73,10 +207,10 @@ def update_headers(self, request_headers: HTTPHeadersDict): """ headers = self.headers for name, value in request_headers.copy().items(): - if value is None: continue # Ignore explicitly unset headers + original_value = value if type(value) is not str: value = value.decode() @@ -85,8 +219,15 @@ def update_headers(self, request_headers: HTTPHeadersDict): if name.lower() == 'cookie': for cookie_name, morsel in SimpleCookie(value).items(): - self['cookies'][cookie_name] = {'value': morsel.value} - del request_headers[name] + if not morsel['path']: + morsel['path'] = DEFAULT_COOKIE_PATH + self.cookie_jar.set(cookie_name, morsel) + + all_cookie_headers = request_headers.getall(name) + if len(all_cookie_headers) > 1: + all_cookie_headers.remove(original_value) + else: + request_headers.popall(name) continue for prefix in SESSION_IGNORED_HEADER_PREFIXES: @@ -103,23 +244,21 @@ def headers(self) -> HTTPHeadersDict: @property def cookies(self) -> RequestsCookieJar: - jar = RequestsCookieJar() - for name, cookie_dict in self['cookies'].items(): - jar.set_cookie(create_cookie( - name, cookie_dict.pop('value'), **cookie_dict)) - jar.clear_expired_cookies() - return jar + self.cookie_jar.clear_expired_cookies() + return self.cookie_jar @cookies.setter def cookies(self, jar: RequestsCookieJar): - # - stored_attrs = ['value', 'path', 'secure', 'expires'] - self['cookies'] = {} - for cookie in jar: - self['cookies'][cookie.name] = { - attname: getattr(cookie, attname) - for attname in stored_attrs - } + self.cookie_jar = jar + + def remove_cookies(self, cookies: Dict[str, str]): + for cookie in cookies: + remove_cookie_by_name( + self.cookie_jar, + cookie['name'], + domain=cookie.get('domain', None), + path=cookie.get('path', None) + ) @property def auth(self) -> Optional[AuthBase]: @@ -154,8 +293,3 @@ def auth(self) -> Optional[AuthBase]: def auth(self, auth: dict): assert {'type', 'raw_auth'} == auth.keys() self['auth'] = auth - - def remove_cookies(self, names: Iterable[str]): - for name in names: - if name in self['cookies']: - del self['cookies'][name] diff --git a/httpie/utils.py b/httpie/utils.py index fa19fa7cde..4fffb2826e 100644 --- a/httpie/utils.py +++ b/httpie/utils.py @@ -9,6 +9,7 @@ from http.cookiejar import parse_ns_headers from pathlib import Path from pprint import pformat +from urllib.parse import urlsplit from typing import Any, List, Optional, Tuple, Callable, Iterable, TypeVar import requests.auth @@ -237,3 +238,7 @@ def unwrap_context(exc: Exception) -> Optional[Exception]: return unwrap_context(context) else: return exc + + +def url_as_host(url: str) -> str: + return urlsplit(url).netloc.split('@')[-1] diff --git a/setup.py b/setup.py index 5316ff73d3..8f9a93140f 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ tests_require = [ 'pytest', 'pytest-httpbin>=0.0.6', + 'pytest-lazy-fixture>=0.0.6', 'responses', ] dev_require = [ diff --git a/tests/conftest.py b/tests/conftest.py index 5e8c511072..7ca0e60440 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,11 @@ import pytest from pytest_httpbin import certs -from .utils import HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, HTTPBIN_WITH_CHUNKED_SUPPORT +from .utils import ( # noqa + HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, + HTTPBIN_WITH_CHUNKED_SUPPORT, + mock_env +) from .utils.plugins_cli import ( # noqa broken_plugin, dummy_plugin, diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 126b13276e..6e6e73676e 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -1,6 +1,9 @@ """Test data""" +import json from pathlib import Path +from typing import Optional, Dict, Any +import httpie from httpie.encoding import UTF8 from httpie.output.formatters.xml import pretty_xml, parse_xml @@ -19,10 +22,20 @@ def patharg(path): JSON_FILE_PATH = FIXTURES_ROOT / 'test.json' JSON_WITH_DUPE_KEYS_FILE_PATH = FIXTURES_ROOT / 'test_with_dupe_keys.json' BIN_FILE_PATH = FIXTURES_ROOT / 'test.bin' + XML_FILES_PATH = FIXTURES_ROOT / 'xmldata' XML_FILES_VALID = list((XML_FILES_PATH / 'valid').glob('*_raw.xml')) XML_FILES_INVALID = list((XML_FILES_PATH / 'invalid').glob('*.xml')) +SESSION_FILES_PATH = FIXTURES_ROOT / 'session_data' +SESSION_FILES_OLD = sorted((SESSION_FILES_PATH / 'old').glob('*.json')) +SESSION_FILES_NEW = sorted((SESSION_FILES_PATH / 'new').glob('*.json')) + +SESSION_VARIABLES = { + '__version__': httpie.__version__, + '__host__': 'null', +} + FILE_PATH_ARG = patharg(FILE_PATH) BIN_FILE_PATH_ARG = patharg(BIN_FILE_PATH) JSON_FILE_PATH_ARG = patharg(JSON_FILE_PATH) @@ -40,3 +53,14 @@ def patharg(path): UNICODE = FILE_CONTENT XML_DATA_RAW = 'text' XML_DATA_FORMATTED = pretty_xml(parse_xml(XML_DATA_RAW)) + + +def read_session_file(session_file: Path, *, extra_variables: Optional[Dict[str, str]] = None) -> Any: + with open(session_file) as stream: + data = stream.read() + + session_vars = {**SESSION_VARIABLES, **(extra_variables or {})} + for variable, value in session_vars.items(): + data = data.replace(variable, value) + + return json.loads(data) diff --git a/tests/fixtures/session_data/new/cookies_dict.json b/tests/fixtures/session_data/new/cookies_dict.json new file mode 100644 index 0000000000..8a4d5f2e13 --- /dev/null +++ b/tests/fixtures/session_data/new/cookies_dict.json @@ -0,0 +1,31 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [ + { + "domain": __host__, + "expires": null, + "name": "baz", + "path": "/", + "secure": false, + "value": "quux" + }, + { + "domain": __host__, + "expires": null, + "name": "foo", + "path": "/", + "secure": false, + "value": "bar" + } + ], + "headers": {} +} diff --git a/tests/fixtures/session_data/new/cookies_dict_dev_version.json b/tests/fixtures/session_data/new/cookies_dict_dev_version.json new file mode 100644 index 0000000000..8a4d5f2e13 --- /dev/null +++ b/tests/fixtures/session_data/new/cookies_dict_dev_version.json @@ -0,0 +1,31 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [ + { + "domain": __host__, + "expires": null, + "name": "baz", + "path": "/", + "secure": false, + "value": "quux" + }, + { + "domain": __host__, + "expires": null, + "name": "foo", + "path": "/", + "secure": false, + "value": "bar" + } + ], + "headers": {} +} diff --git a/tests/fixtures/session_data/new/cookies_dict_with_extras.json b/tests/fixtures/session_data/new/cookies_dict_with_extras.json new file mode 100644 index 0000000000..9a99f15268 --- /dev/null +++ b/tests/fixtures/session_data/new/cookies_dict_with_extras.json @@ -0,0 +1,33 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "raw_auth": "foo:bar", + "type": "basic" + }, + "cookies": [ + { + "domain": __host__, + "expires": null, + "name": "baz", + "path": "/", + "secure": false, + "value": "quux" + }, + { + "domain": __host__, + "expires": null, + "name": "foo", + "path": "/", + "secure": false, + "value": "bar" + } + ], + "headers": { + "X-Data": "value", + "X-Foo": "bar" + } +} diff --git a/tests/fixtures/session_data/new/empty_cookies_dict.json b/tests/fixtures/session_data/new/empty_cookies_dict.json new file mode 100644 index 0000000000..1d01661a06 --- /dev/null +++ b/tests/fixtures/session_data/new/empty_cookies_dict.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [], + "headers": {} +} diff --git a/tests/fixtures/session_data/new/empty_cookies_list.json b/tests/fixtures/session_data/new/empty_cookies_list.json new file mode 100644 index 0000000000..1d01661a06 --- /dev/null +++ b/tests/fixtures/session_data/new/empty_cookies_list.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [], + "headers": {} +} diff --git a/tests/fixtures/session_data/old/cookies_dict.json b/tests/fixtures/session_data/old/cookies_dict.json new file mode 100644 index 0000000000..9c4fd21476 --- /dev/null +++ b/tests/fixtures/session_data/old/cookies_dict.json @@ -0,0 +1,27 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": { + "baz": { + "expires": null, + "path": "/", + "secure": false, + "value": "quux" + }, + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + }, + "headers": {} +} diff --git a/tests/fixtures/session_data/old/cookies_dict_dev_version.json b/tests/fixtures/session_data/old/cookies_dict_dev_version.json new file mode 100644 index 0000000000..935b43f083 --- /dev/null +++ b/tests/fixtures/session_data/old/cookies_dict_dev_version.json @@ -0,0 +1,27 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "2.7.0.dev0" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": { + "baz": { + "expires": null, + "path": "/", + "secure": false, + "value": "quux" + }, + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + }, + "headers": {} +} diff --git a/tests/fixtures/session_data/old/cookies_dict_with_extras.json b/tests/fixtures/session_data/old/cookies_dict_with_extras.json new file mode 100644 index 0000000000..42968e52a9 --- /dev/null +++ b/tests/fixtures/session_data/old/cookies_dict_with_extras.json @@ -0,0 +1,29 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "raw_auth": "foo:bar", + "type": "basic" + }, + "cookies": { + "baz": { + "expires": null, + "path": "/", + "secure": false, + "value": "quux" + }, + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + }, + "headers": { + "X-Data": "value", + "X-Foo": "bar" + } +} diff --git a/tests/fixtures/session_data/old/empty_cookies_dict.json b/tests/fixtures/session_data/old/empty_cookies_dict.json new file mode 100644 index 0000000000..8de1a9217c --- /dev/null +++ b/tests/fixtures/session_data/old/empty_cookies_dict.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": {}, + "headers": {} +} diff --git a/tests/fixtures/session_data/old/empty_cookies_list.json b/tests/fixtures/session_data/old/empty_cookies_list.json new file mode 100644 index 0000000000..12194f7ed2 --- /dev/null +++ b/tests/fixtures/session_data/old/empty_cookies_list.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [], + "headers": {} +} diff --git a/tests/test_cookie_on_redirects.py b/tests/test_cookie_on_redirects.py new file mode 100644 index 0000000000..e22f833048 --- /dev/null +++ b/tests/test_cookie_on_redirects.py @@ -0,0 +1,262 @@ +import pytest +from .utils import http + + +@pytest.fixture +def remote_httpbin(httpbin_with_chunked_support): + return httpbin_with_chunked_support + + +def _stringify(fixture): + return fixture + '' + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +def test_explicit_user_set_cookie(httpbin, instance): + # User set cookies ARE NOT persisted within redirects + # when there is no session, even on the same domain. + + r = http( + '--follow', + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/cookies', + 'Cookie:a=b' + ) + assert r.json == {'cookies': {}} + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +def test_explicit_user_set_cookie_in_session(tmp_path, httpbin, instance): + # User set cookies ARE persisted within redirects + # when there is A session, even on the same domain. + + r = http( + '--follow', + '--session', + str(tmp_path / 'session.json'), + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/cookies', + 'Cookie:a=b' + ) + assert r.json == {'cookies': {'a': 'b'}} + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +def test_saved_user_set_cookie_in_session(tmp_path, httpbin, instance): + # User set cookies ARE persisted within redirects + # when there is A session, even on the same domain. + + http( + '--follow', + '--session', + str(tmp_path / 'session.json'), + httpbin + '/get', + 'Cookie:a=b' + ) + r = http( + '--follow', + '--session', + str(tmp_path / 'session.json'), + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/cookies', + ) + assert r.json == {'cookies': {'a': 'b'}} + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +@pytest.mark.parametrize('session', [True, False]) +def test_explicit_user_set_headers(httpbin, tmp_path, instance, session): + # User set headers ARE persisted within redirects + # even on different domains domain with or without + # an active session. + session_args = [] + if session: + session_args.extend([ + '--session', + str(tmp_path / 'session.json') + ]) + + r = http( + '--follow', + *session_args, + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/get', + 'X-Custom-Header:value' + ) + assert 'X-Custom-Header' in r.json['headers'] + + +@pytest.mark.parametrize('session', [True, False]) +def test_server_set_cookie_on_redirect_same_domain(tmp_path, httpbin, session): + # Server set cookies ARE persisted on the same domain + # when they are forwarded. + + session_args = [] + if session: + session_args.extend([ + '--session', + str(tmp_path / 'session.json') + ]) + + r = http( + '--follow', + *session_args, + httpbin + '/cookies/set/a/b', + ) + assert r.json['cookies'] == {'a': 'b'} + + +@pytest.mark.parametrize('session', [True, False]) +def test_server_set_cookie_on_redirect_different_domain(tmp_path, http_server, httpbin, session): + # Server set cookies ARE persisted on different domains + # when they are forwarded. + + session_args = [] + if session: + session_args.extend([ + '--session', + str(tmp_path / 'session.json') + ]) + + r = http( + '--follow', + *session_args, + http_server + '/cookies/set-and-redirect', + f"X-Redirect-To:{httpbin + '/cookies'}", + 'X-Cookies:a=b' + ) + assert r.json['cookies'] == {'a': 'b'} + + +def test_saved_session_cookies_on_same_domain(tmp_path, httpbin): + # Saved session cookies ARE persisted when making a new + # request to the same domain. + http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies/set/a/b' + ) + r = http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies' + ) + assert r.json == {'cookies': {'a': 'b'}} + + +def test_saved_session_cookies_on_different_domain(tmp_path, httpbin, remote_httpbin): + # Saved session cookies ARE persisted when making a new + # request to a different domain. + http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies/set/a/b' + ) + r = http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies' + ) + assert r.json == {'cookies': {}} + + +@pytest.mark.parametrize('initial_domain, first_request_domain, second_request_domain, expect_cookies', [ + ( + # Cookies are set by Domain A + # Initial domain is Domain A + # Redirected domain is Domain A + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('httpbin'), + True, + ), + ( + # Cookies are set by Domain A + # Initial domain is Domain B + # Redirected domain is Domain B + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), + pytest.lazy_fixture('remote_httpbin'), + False, + ), + ( + # Cookies are set by Domain A + # Initial domain is Domain A + # Redirected domain is Domain B + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), + False, + ), + ( + # Cookies are set by Domain A + # Initial domain is Domain B + # Redirected domain is Domain A + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), + pytest.lazy_fixture('httpbin'), + True, + ), +]) +def test_saved_session_cookies_on_redirect(tmp_path, initial_domain, first_request_domain, second_request_domain, expect_cookies): + http( + '--session', + str(tmp_path / 'session.json'), + initial_domain + '/cookies/set/a/b' + ) + r = http( + '--session', + str(tmp_path / 'session.json'), + '--follow', + first_request_domain + '/redirect-to', + f'url=={_stringify(second_request_domain)}/cookies' + ) + if expect_cookies: + expected_data = {'cookies': {'a': 'b'}} + else: + expected_data = {'cookies': {}} + assert r.json == expected_data + + +def test_saved_session_cookie_pool(tmp_path, httpbin, remote_httpbin): + http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies/set/a/b' + ) + http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies/set/a/c' + ) + http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies/set/b/d' + ) + + response = http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies' + ) + assert response.json['cookies'] == {'a': 'b'} + + response = http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies' + ) + assert response.json['cookies'] == {'a': 'c', 'b': 'd'} diff --git a/tests/test_httpie_cli.py b/tests/test_httpie_cli.py new file mode 100644 index 0000000000..31c44d7f1f --- /dev/null +++ b/tests/test_httpie_cli.py @@ -0,0 +1,125 @@ +import pytest +import shutil +import json +from httpie.sessions import SESSIONS_DIR_NAME +from httpie.status import ExitStatus +from tests.utils import DUMMY_HOST, httpie +from tests.fixtures import SESSION_FILES_PATH, SESSION_FILES_NEW, SESSION_FILES_OLD, read_session_file + + +OLD_SESSION_FILES_PATH = SESSION_FILES_PATH / 'old' + + +@pytest.mark.requires_installation +def test_plugins_cli_error_message_without_args(): + # No arguments + result = httpie(no_debug=True) + assert result.exit_status == ExitStatus.ERROR + assert 'usage: ' in result.stderr + assert 'specify one of these' in result.stderr + assert 'please use the http/https commands:' in result.stderr + + +@pytest.mark.parametrize( + 'example', + [ + 'pie.dev/get', + 'DELETE localhost:8000/delete', + 'POST pie.dev/post header:value a=b header_2:value x:=1', + ], +) +@pytest.mark.requires_installation +def test_plugins_cli_error_messages_with_example(example): + result = httpie(*example.split(), no_debug=True) + assert result.exit_status == ExitStatus.ERROR + assert 'usage: ' in result.stderr + assert f'http {example}' in result.stderr + assert f'https {example}' in result.stderr + + +@pytest.mark.parametrize( + 'example', + [ + 'cli', + 'plugins', + 'cli foo', + 'plugins unknown', + 'plugins unknown.com A:B c=d', + 'unknown.com UNPARSABLE????SYNTAX', + ], +) +@pytest.mark.requires_installation +def test_plugins_cli_error_messages_invalid_example(example): + result = httpie(*example.split(), no_debug=True) + assert result.exit_status == ExitStatus.ERROR + assert 'usage: ' in result.stderr + assert f'http {example}' not in result.stderr + assert f'https {example}' not in result.stderr + + +HTTPIE_CLI_SESSIONS_UPGRADE_OPTIONS = [ + ( + # Default settings + [], + {'__host__': json.dumps(None)}, + ), + ( + # When --bind-cookies is applied, the __host__ becomes DUMMY_URL. + ['--bind-cookies'], + {'__host__': json.dumps(DUMMY_HOST)}, + ), +] + + +@pytest.mark.parametrize( + 'old_session_file, new_session_file', zip(SESSION_FILES_OLD, SESSION_FILES_NEW) +) +@pytest.mark.parametrize( + 'extra_args, extra_variables', + HTTPIE_CLI_SESSIONS_UPGRADE_OPTIONS, +) +def test_httpie_sessions_upgrade(tmp_path, old_session_file, new_session_file, extra_args, extra_variables): + session_path = tmp_path / 'session.json' + shutil.copyfile(old_session_file, session_path) + + result = httpie( + 'cli', 'sessions', 'upgrade', *extra_args, DUMMY_HOST, str(session_path) + ) + assert result.exit_status == ExitStatus.SUCCESS + assert read_session_file(session_path) == read_session_file( + new_session_file, extra_variables=extra_variables + ) + + +def test_httpie_sessions_upgrade_on_non_existent_file(tmp_path): + session_path = tmp_path / 'session.json' + result = httpie('cli', 'sessions', 'upgrade', DUMMY_HOST, str(session_path)) + assert result.exit_status == ExitStatus.ERROR + assert 'does not exist' in result.stderr + + +@pytest.mark.parametrize( + 'extra_args, extra_variables', + HTTPIE_CLI_SESSIONS_UPGRADE_OPTIONS, +) +def test_httpie_sessions_upgrade_all(tmp_path, mock_env, extra_args, extra_variables): + mock_env._create_temp_config_dir = False + mock_env.config_dir = tmp_path / "config" + + session_dir = mock_env.config_dir / SESSIONS_DIR_NAME / DUMMY_HOST + session_dir.mkdir(parents=True) + for original_session_file in SESSION_FILES_OLD: + shutil.copy(original_session_file, session_dir) + + result = httpie( + 'cli', 'sessions', 'upgrade-all', *extra_args, env=mock_env + ) + assert result.exit_status == ExitStatus.SUCCESS + + for refactored_session_file, expected_session_file in zip( + sorted(session_dir.glob("*.json")), + SESSION_FILES_NEW + ): + assert read_session_file(refactored_session_file) == read_session_file( + expected_session_file, extra_variables=extra_variables + ) diff --git a/tests/test_plugins_cli.py b/tests/test_plugins_cli.py index 9f94821505..70cecb1fb7 100644 --- a/tests/test_plugins_cli.py +++ b/tests/test_plugins_cli.py @@ -1,7 +1,6 @@ import pytest from httpie.status import ExitStatus -from tests.utils import httpie from tests.utils.plugins_cli import parse_listing @@ -149,45 +148,3 @@ def test_broken_plugins(httpie_plugins, httpie_plugins_success, dummy_plugin, br # No warning now, since it is uninstalled. data = parse_listing(httpie_plugins_success('list')) assert len(data) == 1 - - -@pytest.mark.requires_installation -def test_plugins_cli_error_message_without_args(): - # No arguments - result = httpie(no_debug=True) - assert result.exit_status == ExitStatus.ERROR - assert 'usage: ' in result.stderr - assert 'specify one of these' in result.stderr - assert 'please use the http/https commands:' in result.stderr - - -@pytest.mark.parametrize( - 'example', [ - 'pie.dev/get', - 'DELETE localhost:8000/delete', - 'POST pie.dev/post header:value a=b header_2:value x:=1' - ] -) -@pytest.mark.requires_installation -def test_plugins_cli_error_messages_with_example(example): - result = httpie(*example.split(), no_debug=True) - assert result.exit_status == ExitStatus.ERROR - assert 'usage: ' in result.stderr - assert f'http {example}' in result.stderr - assert f'https {example}' in result.stderr - - -@pytest.mark.parametrize( - 'example', [ - 'plugins unknown', - 'plugins unknown.com A:B c=d', - 'unknown.com UNPARSABLE????SYNTAX', - ] -) -@pytest.mark.requires_installation -def test_plugins_cli_error_messages_invalid_example(example): - result = httpie(*example.split(), no_debug=True) - assert result.exit_status == ExitStatus.ERROR - assert 'usage: ' in result.stderr - assert f'http {example}' not in result.stderr - assert f'https {example}' not in result.stderr diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 5835993605..8bcd906327 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -1,12 +1,16 @@ import json import os import shutil +from contextlib import contextmanager from datetime import datetime from unittest import mock +from pathlib import Path +from typing import Iterator import pytest from .fixtures import FILE_PATH_ARG, UNICODE +from httpie.context import Environment from httpie.encoding import UTF8 from httpie.plugins import AuthPlugin from httpie.plugins.builtin import HTTPBasicAuth @@ -14,7 +18,7 @@ from httpie.sessions import Session from httpie.utils import get_expired_cookies from .test_auth_plugins import basic_auth -from .utils import HTTP_OK, MockEnvironment, http, mk_config_dir +from .utils import DUMMY_HOST, HTTP_OK, MockEnvironment, http, mk_config_dir from base64 import b64encode @@ -203,9 +207,9 @@ def test_session_with_cookie_followed_by_another_header(self, httpbin): """ self.start_session(httpbin) session_data = { - "headers": { - "cookie": "...", - "zzz": "..." + 'headers': { + 'cookie': '...', + 'zzz': '...' } } session_path = self.config_dir / 'session-data.json' @@ -307,7 +311,7 @@ class Plugin(AuthPlugin): auth_type = 'test-prompted' def get_auth(self, username=None, password=None): - basic_auth_header = "Basic " + b64encode(self.raw_auth.encode()).strip().decode('latin1') + basic_auth_header = 'Basic ' + b64encode(self.raw_auth.encode()).strip().decode('latin1') return basic_auth(basic_auth_header) plugin_manager.register(Plugin) @@ -359,7 +363,7 @@ def get_auth(self, username=None, password=None): ) updated_session = json.loads(self.session_path.read_text(encoding=UTF8)) assert updated_session['auth']['type'] == 'test-saved' - assert updated_session['auth']['raw_auth'] == "user:password" + assert updated_session['auth']['raw_auth'] == 'user:password' plugin_manager.unregister(Plugin) @@ -368,12 +372,12 @@ class TestExpiredCookies(CookieTestBase): @pytest.mark.parametrize( 'initial_cookie, expired_cookie', [ - ({'id': {'value': 123}}, 'id'), - ({'id': {'value': 123}}, 'token') + ({'id': {'value': 123}}, {'name': 'id'}), + ({'id': {'value': 123}}, {'name': 'token'}) ] ) - def test_removes_expired_cookies_from_session_obj(self, initial_cookie, expired_cookie, httpbin): - session = Session(self.config_dir) + def test_removes_expired_cookies_from_session_obj(self, initial_cookie, expired_cookie, httpbin, mock_env): + session = Session(self.config_dir, env=mock_env, session_id=None, bound_host=None) session['cookies'] = initial_cookie session.remove_cookies([expired_cookie]) assert expired_cookie not in session.cookies @@ -524,3 +528,165 @@ def test_cookie_storage_priority(self, cli_cookie, set_cookie, expected, httpbin updated_session = json.loads(self.session_path.read_text(encoding=UTF8)) assert updated_session['cookies']['cookie1']['value'] == expected + + +@pytest.fixture +def basic_session(httpbin, tmp_path): + session_path = tmp_path / 'session.json' + http( + '--session', str(session_path), + httpbin + '/get' + ) + return session_path + + +@contextmanager +def open_session(path: Path, env: Environment, read_only: bool = False) -> Iterator[Session]: + session = Session(path, env, session_id='test', bound_host=DUMMY_HOST) + session.load() + yield session + if not read_only: + session.save() + + +@contextmanager +def open_raw_session(path: Path, read_only: bool = False) -> None: + with open(path) as stream: + raw_session = json.load(stream) + + yield raw_session + + if not read_only: + with open(path, 'w') as stream: + json.dump(raw_session, stream) + + +def read_stderr(env: Environment) -> bytes: + env.stderr.seek(0) + stderr_data = env.stderr.read() + if isinstance(stderr_data, str): + return stderr_data.encode() + else: + return stderr_data + + +def test_old_session_version_saved_as_is(basic_session, mock_env): + with open_session(basic_session, mock_env) as session: + session['__meta__'] = {'httpie': '0.0.1'} + + with open_session(basic_session, mock_env, read_only=True) as session: + assert session['__meta__']['httpie'] == '0.0.1' + + +def test_old_session_cookie_layout_warning(basic_session, mock_env): + with open_session(basic_session, mock_env) as session: + # Use the old layout & set a cookie + session['cookies'] = {} + session.cookies.set('foo', 'bar') + + assert read_stderr(mock_env) == b'' + + with open_session(basic_session, mock_env, read_only=True) as session: + assert b'Outdated layout detected' in read_stderr(mock_env) + + +@pytest.mark.parametrize('cookies, expect_warning', [ + # Old-style cookie format + ( + # Without 'domain' set + {'foo': {'value': 'bar'}}, + True + ), + ( + # With 'domain' set to empty string + {'foo': {'value': 'bar', 'domain': ''}}, + True + ), + ( + # With 'domain' set to null + {'foo': {'value': 'bar', 'domain': None}}, + False, + ), + ( + # With 'domain' set to a URL + {'foo': {'value': 'bar', 'domain': DUMMY_HOST}}, + False, + ), + # New style cookie format + ( + # Without 'domain' set + [{'name': 'foo', 'value': 'bar'}], + False + ), + ( + # With 'domain' set to empty string + [{'name': 'foo', 'value': 'bar', 'domain': ''}], + False + ), + ( + # With 'domain' set to null + [{'name': 'foo', 'value': 'bar', 'domain': None}], + False, + ), + ( + # With 'domain' set to a URL + [{'name': 'foo', 'value': 'bar', 'domain': DUMMY_HOST}], + False, + ), +]) +def test_cookie_security_warnings_on_raw_cookies(basic_session, mock_env, cookies, expect_warning): + with open_raw_session(basic_session) as raw_session: + raw_session['cookies'] = cookies + + with open_session(basic_session, mock_env, read_only=True): + warning = b'Outdated layout detected' + stderr = read_stderr(mock_env) + + if expect_warning: + assert warning in stderr + else: + assert warning not in stderr + + +def test_old_session_cookie_layout_loading(basic_session, httpbin, mock_env): + with open_session(basic_session, mock_env) as session: + # Use the old layout & set a cookie + session['cookies'] = {} + session.cookies.set('foo', 'bar') + + response = http( + '--session', str(basic_session), + httpbin + '/cookies' + ) + assert response.json['cookies'] == {'foo': 'bar'} + + +@pytest.mark.parametrize('layout_type', [ + dict, list +]) +def test_session_cookie_layout_preservance(basic_session, mock_env, layout_type): + with open_session(basic_session, mock_env) as session: + session['cookies'] = layout_type() + session.cookies.set('foo', 'bar') + session.save() + + with open_session(basic_session, mock_env, read_only=True) as session: + assert isinstance(session['cookies'], layout_type) + + +@pytest.mark.parametrize('layout_type', [ + dict, list +]) +def test_session_cookie_layout_preservance_on_new_cookies(basic_session, httpbin, mock_env, layout_type): + with open_session(basic_session, mock_env) as session: + session['cookies'] = layout_type() + session.cookies.set('foo', 'bar') + session.save() + + http( + '--session', str(basic_session), + httpbin + '/cookies/set/baz/quux' + ) + + with open_session(basic_session, mock_env, read_only=True) as session: + assert isinstance(session['cookies'], layout_type) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index cf90d684b9..d3359820c1 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -6,6 +6,8 @@ import json import tempfile import warnings +import pytest +from contextlib import suppress from io import BytesIO from pathlib import Path from typing import Any, Optional, Union, List, Iterable @@ -16,6 +18,7 @@ from httpie.status import ExitStatus from httpie.config import Config from httpie.context import Environment +from httpie.utils import url_as_host # pytest-httpbin currently does not support chunked requests: @@ -39,6 +42,7 @@ ) DUMMY_URL = 'http://this-should.never-resolve' # Note: URL never fetched +DUMMY_HOST = url_as_host(DUMMY_URL) def strip_colors(colorized_msg: str) -> str: @@ -187,6 +191,13 @@ class ExitStatusError(Exception): pass +@pytest.fixture +def mock_env() -> MockEnvironment: + env = MockEnvironment(stdout_mode='') + yield env + env.cleanup() + + def normalize_args(args: Iterable[Any]) -> List[str]: return [str(arg) for arg in args] @@ -201,7 +212,7 @@ def httpie( status. """ - env = kwargs.setdefault('env', MockEnvironment()) + env = kwargs.setdefault('env', MockEnvironment(stdout_mode='')) cli_args = ['httpie'] if not kwargs.pop('no_debug', False): cli_args.append('--debug') @@ -214,7 +225,16 @@ def httpie( env.stdout.seek(0) env.stderr.seek(0) try: - response = StrCLIResponse(env.stdout.read()) + output = env.stdout.read() + if isinstance(output, bytes): + with suppress(UnicodeDecodeError): + output = output.decode() + + if isinstance(output, bytes): + response = BytesCLIResponse(output) + else: + response = StrCLIResponse(output) + response.stderr = env.stderr.read() response.exit_status = exit_status response.args = cli_args diff --git a/tests/utils/http_server.py b/tests/utils/http_server.py index 0a96dd8b07..ecc14966b9 100644 --- a/tests/utils/http_server.py +++ b/tests/utils/http_server.py @@ -85,6 +85,19 @@ def status_custom_msg(handler): handler.end_headers() +@TestHandler.handler('GET', '/cookies/set-and-redirect') +def set_cookie_and_redirect(handler): + handler.send_response(302) + + redirect_to = handler.headers.get('X-Redirect-To', '/headers') + handler.send_header('Location', redirect_to) + + raw_cookies = handler.headers.get('X-Cookies', 'a=b') + for cookie in raw_cookies.split(', '): + handler.send_header('Set-Cookie', cookie) + handler.end_headers() + + @pytest.fixture(scope="function") def http_server(): """A custom HTTP server implementation for our tests, that is From 395914fb4d439ce5220a44af231d3e16bf3fe18d Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Fri, 4 Mar 2022 14:09:16 +0300 Subject: [PATCH 31/36] Apply suggestions from the review --- SECURITY.md | 10 ++++ docs/README.md | 17 ------ httpie/legacy/__init__.py | 0 httpie/legacy/cookie_format.py | 103 +++++++++++++++++++++++++++++++++ httpie/manager/cli.py | 6 +- httpie/manager/tasks.py | 24 +------- httpie/sessions.py | 94 ++++++++++-------------------- 7 files changed, 148 insertions(+), 106 deletions(-) create mode 100644 SECURITY.md create mode 100644 httpie/legacy/__init__.py create mode 100644 httpie/legacy/cookie_format.py diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..b10980cbb6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,10 @@ +# Security Policy + +## Reporting a Vulnerability + +To report a vulnerability, please send an email to `security@httpie.io` describing the: + +- The description of the vulnerability itself +- A short reproducer to verify it (you can submit a small HTTP server, a shell script, a docker image etc.) +- The severity level classification (`LOW`/`MEDIUM`/`HIGH`/`CRITICAL`) +- If associated with any, the [CWE](https://cwe.mitre.org/) ID. diff --git a/docs/README.md b/docs/README.md index efd579a343..30daa4b678 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2422,23 +2422,6 @@ And since there’s neither data nor `EOF`, it will get stuck. So unless you’r Also, it might be good to set a connection `--timeout` limit to prevent your program from hanging if the server never responds. -### Security - -#### Exposure of Cookies To The 3rd Party Hosts On Redirects - -*Vulnerability Type*: [CWE-200](https://cwe.mitre.org/data/definitions/200.html) -*Severity Level*: LOW -*Affected Versions*: `<3.1.0` - -The handling of [cookies](#cookies) was not compatible with the [RFC 6265](https://datatracker.ietf.org/doc/html/rfc6265) -on the point of handling the `Domain` attribute when they were saved into [session](#sessions) files. All cookies were shared -across all hosts during the runtime, including redirects to the 3rd party hosts. - -This vulnerability has been fixed in [3.1.0](https://github.com/httpie/httpie/releases/tag/3.1.0) and the -[`httpie cli sessions upgrade`](#upgrading-sessions)/[`httpie cli sessions upgrade-all`]((#upgrading-sessions) commands -have been put in place in order to allow a smooth transition to the new session layout from the existing [session](#sessions) -files. - ## Plugin manager HTTPie offers extensibility through a [plugin API](https://github.com/httpie/httpie/blob/master/httpie/plugins/base.py), diff --git a/httpie/legacy/__init__.py b/httpie/legacy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/httpie/legacy/cookie_format.py b/httpie/legacy/cookie_format.py new file mode 100644 index 0000000000..b5c6392b7c --- /dev/null +++ b/httpie/legacy/cookie_format.py @@ -0,0 +1,103 @@ +import argparse +from typing import Any, Type, List, Dict, TYPE_CHECKING + +if TYPE_CHECKING: + from httpie.sessions import Session + +INSECURE_COOKIE_JAR_WARNING = '''\ +Outdated layout detected for the current session. Please consider updating it, +in order to not get affected by potential security problems. + +For fixing the current session: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade --bind-cookies {hostname} {session_id} + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade {hostname} {session_id} +''' + + +INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS = '''\ + +For fixing all named sessions: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade-all --bind-cookies + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade-all +''' + +INSECURE_COOKIE_SECURITY_LINK = '\nSee https://pie.co/docs/security for more information.' + + +def pre_process(session: 'Session', cookies: Any) -> List[Dict[str, Any]]: + """Load the given cookies to the cookie jar while maintaining + support for the old cookie layout.""" + + is_old_style = isinstance(cookies, dict) + if is_old_style: + normalized_cookies = [ + { + 'name': key, + **value + } + for key, value in cookies.items() + ] + else: + normalized_cookies = cookies + + should_issue_warning = is_old_style and any( + cookie.get('domain', '') == '' + for cookie in normalized_cookies + ) + + if should_issue_warning and not session.refactor_mode: + warning = INSECURE_COOKIE_JAR_WARNING.format(hostname=session.bound_host, session_id=session.session_id) + if not session.is_anonymous: + warning += INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS + warning += INSECURE_COOKIE_SECURITY_LINK + + session.env.log_error( + warning, + level='warning' + ) + + return normalized_cookies + + +def post_process( + normalized_cookies: List[Dict[str, Any]], + *, + original_type: Type[Any] +) -> Any: + """Convert the cookies to their original format for + maximum compatibility.""" + + if issubclass(original_type, dict): + return { + cookie.pop('name'): cookie + for cookie in normalized_cookies + } + else: + return normalized_cookies + + +def fix_layout(session: 'Session', hostname: str, args: argparse.Namespace) -> None: + if not isinstance(session['cookies'], dict): + return None + + session['cookies'] = [ + { + 'name': key, + **value + } + for key, value in session['cookies'].items() + ] + for cookie in session.cookies: + if cookie.domain == '': + if args.bind_cookies: + cookie.domain = hostname + else: + cookie._rest['is_explicit_none'] = True diff --git a/httpie/manager/cli.py b/httpie/manager/cli.py index 9ad4eca6ba..1473ccf977 100644 --- a/httpie/manager/cli.py +++ b/httpie/manager/cli.py @@ -4,7 +4,7 @@ CLI_SESSION_UPGRADE_FLAGS = [ { - 'variadic': ['--bind-cookies'], + 'flags': ['--bind-cookies'], 'action': 'store_true', 'default': False, 'help': 'Bind domainless cookies to the host that session belongs.' @@ -102,8 +102,8 @@ def generate_subparsers(root, parent_parser, definitions): for argument in properties: argument = argument.copy() - variadic = argument.pop('variadic', []) - command_parser.add_argument(*variadic, **argument) + flags = argument.pop('flags', []) + command_parser.add_argument(*flags, **argument) parser = HTTPieManagerArgumentParser( diff --git a/httpie/manager/tasks.py b/httpie/manager/tasks.py index c04ed9bc3d..297767b025 100644 --- a/httpie/manager/tasks.py +++ b/httpie/manager/tasks.py @@ -1,9 +1,10 @@ import argparse from typing import TypeVar, Callable, Tuple -from httpie.sessions import SESSIONS_DIR_NAME, Session, get_httpie_session +from httpie.sessions import SESSIONS_DIR_NAME, get_httpie_session from httpie.status import ExitStatus from httpie.context import Environment +from httpie.legacy import cookie_format as legacy_cookies from httpie.manager.cli import missing_subcommand, parser T = TypeVar('T') @@ -51,27 +52,8 @@ def split_version(version: str) -> Tuple[int, ...]: return split_version(version_1) > split_version(version_2) -def fix_cookie_layout(session: Session, hostname: str, args: argparse.Namespace) -> None: - if not isinstance(session['cookies'], dict): - return None - - session['cookies'] = [ - { - 'name': key, - **value - } - for key, value in session['cookies'].items() - ] - for cookie in session.cookies: - if cookie.domain == '': - if args.bind_cookies: - cookie.domain = hostname - else: - cookie._rest['is_explicit_none'] = True - - FIXERS_TO_VERSIONS = { - '3.1.0': fix_cookie_layout + '3.1.0': legacy_cookies.fix_layout } diff --git a/httpie/sessions.py b/httpie/sessions.py index c23cb56852..e4a20a5344 100644 --- a/httpie/sessions.py +++ b/httpie/sessions.py @@ -8,7 +8,7 @@ from http.cookies import SimpleCookie from http.cookiejar import Cookie from pathlib import Path -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union from requests.auth import AuthBase from requests.cookies import RequestsCookieJar, remove_cookie_by_name @@ -18,6 +18,7 @@ from .config import BaseConfigDict, DEFAULT_CONFIG_DIR from .utils import url_as_host from .plugins.registry import plugin_manager +from .legacy import cookie_format as legacy_cookies SESSIONS_DIR_NAME = 'sessions' @@ -32,35 +33,23 @@ KEPT_COOKIE_OPTIONS = ['name', 'expires', 'path', 'value', 'domain', 'secure'] DEFAULT_COOKIE_PATH = '/' -INSECURE_COOKIE_JAR_WARNING = '''\ -Outdated layout detected for the current session. Please consider updating it, -in order to not get affected by potential security problems. -For fixing the current session: - - With binding all cookies to the current host (secure): - $ httpie cli sessions upgrade --bind-cookies {hostname} {session_id} - - Without binding cookies (leaving them as is) (insecure): - $ httpie cli sessions upgrade {hostname} {session_id} -''' - -INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS = '''\ - -For fixing all named sessions: - - With binding all cookies to the current host (secure): - $ httpie cli sessions upgrade-all --bind-cookies +def is_anonymous_session(session_name: str) -> bool: + return os.path.sep in session_name - Without binding cookies (leaving them as is) (insecure): - $ httpie cli sessions upgrade-all -See https://pie.co/docs/security for more information. -''' +def session_hostname_to_dirname(hostname: str, session_name: str) -> str: + # host:port => host_port + hostname = hostname.replace(':', '_') + return os.path.join( + SESSIONS_DIR_NAME, + hostname, + f'{session_name}.json' + ) -def is_anonymous_session(session_name: str) -> bool: - return os.path.sep in session_name +def strip_port(hostname: str) -> str: + return hostname.split(':')[0] def materialize_cookie(cookie: Cookie) -> Dict[str, Any]: @@ -92,22 +81,18 @@ def get_httpie_session( # HACK/FIXME: httpie-unixsocket's URLs have no hostname. bound_hostname = 'localhost' - # host:port => host_port - hostname = bound_hostname.replace(':', '_') if is_anonymous_session(session_name): path = os.path.expanduser(session_name) session_id = path else: - path = ( - config_dir / SESSIONS_DIR_NAME / hostname / f'{session_name}.json' - ) + path = config_dir / session_hostname_to_dirname(bound_hostname, session_name) session_id = session_name session = Session( path, env=env, session_id=session_id, - bound_host=bound_hostname.split(':')[0], + bound_host=strip_port(bound_hostname), refactor_mode=refactor_mode ) session.load() @@ -142,60 +127,35 @@ def __init__( def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: cookies = data.get('cookies') - if isinstance(cookies, dict): - normalized_cookies = [ - { - 'name': key, - **value - } - for key, value in cookies.items() - ] - elif isinstance(cookies, list): - normalized_cookies = cookies + if cookies: + normalized_cookies = legacy_cookies.pre_process(self, cookies) else: normalized_cookies = [] - should_issue_warning = False for cookie in normalized_cookies: domain = cookie.get('domain', '') - if domain == '' and isinstance(cookies, dict): - should_issue_warning = True - elif domain is None: + if domain is None: # domain = None means explicitly lack of cookie, though - # requests requires domain to be string so we'll cast it + # requests requires domain to be a string so we'll cast it # manually. cookie['domain'] = '' cookie['rest'] = {'is_explicit_none': True} self.cookie_jar.set(**cookie) - if should_issue_warning and not self.refactor_mode: - warning = INSECURE_COOKIE_JAR_WARNING.format(hostname=self.bound_host, session_id=self.session_id) - if not is_anonymous_session(self.session_id): - warning += INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS - - self.env.log_error( - warning, - level='warning' - ) - return data def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: cookies = data.get('cookies') - # Save in the old-style fashion normalized_cookies = [ materialize_cookie(cookie) for cookie in self.cookie_jar ] - if isinstance(cookies, dict): - data['cookies'] = { - cookie.pop('name'): cookie - for cookie in normalized_cookies - } - else: - data['cookies'] = normalized_cookies + data['cookies'] = legacy_cookies.post_process( + normalized_cookies, + original_type=type(cookies) + ) return data @@ -251,7 +211,7 @@ def cookies(self) -> RequestsCookieJar: def cookies(self, jar: RequestsCookieJar): self.cookie_jar = jar - def remove_cookies(self, cookies: Dict[str, str]): + def remove_cookies(self, cookies: List[Dict[str, str]]): for cookie in cookies: remove_cookie_by_name( self.cookie_jar, @@ -293,3 +253,7 @@ def auth(self) -> Optional[AuthBase]: def auth(self, auth: dict): assert {'type', 'raw_auth'} == auth.keys() self['auth'] = auth + + @property + def is_anonymous(self): + return is_anonymous_session(self.session_id) From 614866eeb237aebe01f467e64f58c1004ba780b2 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 7 Mar 2022 20:43:48 +0100 Subject: [PATCH 32/36] Polish sessions docs --- docs/README.md | 154 +++++++++++++++++++------------------------------ 1 file changed, 59 insertions(+), 95 deletions(-) diff --git a/docs/README.md b/docs/README.md index 30daa4b678..5091295536 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2157,30 +2157,28 @@ $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:orig- $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:new-value ``` -### Host-based Cookie Policy +### Host-based cookie policy -Cookies in stored HTTPie sessions have a `domain` field which is binding them to the -specified hostname. For example, in the following session: +Cookies persisted in sessions files have a `domain` field. This _binds_ them to a specified hostname. For example: ```json { "cookies": [ { "domain": "pie.dev", - "name": "secret_cookie", - "value": "value_1" + "name": "pie", + "value": "apple" }, { "domain": "httpbin.org", - "name": "secret_cookie", - "value": "value_2" + "name": "bin", + "value": "http" } ] } ``` -we will send `Cookie:secret_cookie=value_1` only when you are making a request against `pie.dev` (it -also includes the domains, like `api.pie.dev`), and `Cookie:secret_cookie=value_2` when you use `httpbin.org`. +Using this session file, we include `Cookie: pie=apple` only in requests against `pie.dev` and subdomains (e.g., `foo.pie.dev` or `foo.bar.pie.dev`): ```bash $ http --session=./session.json pie.dev/cookies @@ -2189,36 +2187,20 @@ $ http --session=./session.json pie.dev/cookies ```json { "cookies": { - "secret_cookie": "value_1" + "pie": "apple" } } ``` -```bash -$ http --session=./session.json httpbin.org/cookies -``` - -```json -{ - "cookies": { - "secret_cookie": "value_2" - } -} -``` - -If you want to make a cookie domain unbound, you can simply set the `domain` -field to `null` by editing the session file directly: +To make a cookie domain _unbound_ (i.e., to make it available to all hosts, including throughout a cross-domain redirect chain), you can set the `domain` field to `null` in the session file: ```json { "cookies": [ { "domain": null, - "expires": null, - "name": "generic_cookie", - "path": "/", - "secure": false, - "value": "generic_value" + "name": "unbound-cookie", + "value": "send-me-to-any-host" } ] } @@ -2231,105 +2213,87 @@ $ http --session=./session.json pie.dev/cookies ```json { "cookies": { - "generic_cookie": "generic_value" + "unbound-cookie": "send-me-to-any-host" } } ``` -### Cookie Storage Behavior -**TL;DR:** Cookie storage priority: Server response > Command line request > Session file +### Cookie storage behavior -To set a cookie within a Session there are three options: +There are three possible sources of persisted cookies within a session. They have the following storage priority: 1—response; 2—command line; 3—session file. -1. Get a `Set-Cookie` header in a response from a server +1. Receive a response with a `Set-Cookie` header: - ```bash - $ http --session=./session.json pie.dev/cookie/set?foo=bar - ``` +```bash +$ http --session=./session.json pie.dev/cookie/set?foo=bar +``` -2. Set the cookie name and value through the command line as seen in [cookies](#cookies) +2. Send a cookie specified on the command line as seen in [cookies](#cookies): - ```bash - $ http --session=./session.json pie.dev/headers Cookie:foo=bar - ``` +```bash +$ http --session=./session.json pie.dev/headers Cookie:foo=bar +``` -3. Manually set cookie parameters in the JSON file of the session +3. Manually set cookie parameters in the session file: - ```json - { - "__meta__": { - "about": "HTTPie session file", - "help": "https://httpie.org/doc#sessions", - "httpie": "2.2.0-dev" - }, - "auth": { - "password": null, - "type": null, - "username": null - }, - "cookies": { - "foo": { - "expires": null, - "path": "/", - "secure": false, - "value": "bar" - } - } +```json +{ + "cookies": { + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } } - ``` - -Cookies will be set in the session file with the priority specified above. -For example, a cookie set through the command line will overwrite a cookie of the same name stored in the session file. -If the server returns a `Set-Cookie` header with a cookie of the same name, the returned cookie will overwrite the preexisting cookie. +} +``` -Expired cookies are never stored. -If a cookie in a session file expires, it will be removed before sending a new request. -If the server expires an existing cookie, it will also be removed from the session file. +In summary: -### Upgrading Sessions +- Cookies set via the CLI overwrite cookies of the same name inside session files. +- Server-sent `Set-Cookie` header cookies overwrite any pre-existing ones with the same name. -In rare circumstances, HTTPie makes changes in it's session layout. For allowing a smoother transition of existing files -from the old layout to the new layout we offer 2 interfaces: +Cookie expiration handling: -- `httpie cli sessions upgrade` -- `httpie cli sessions upgrade-all` +- When the server expires an existing cookie, HTTPie removes it from the session file. +- When a cookie in a session file expires, HTTPie removes it before sending a new request. +### Upgrading sessions -With `httpie cli sessions upgrade`, you can upgrade a single session with it's name (or it's path, if it is an -[anonymous session](#anonymous-sessions)) and the hostname it belongs to. For example: +HTTPie may introduce changes in the session file format. When HTTPie detects an obsolete format, it shows a warning. You can upgrade your session files using the following commands: -([named session](#named-sessions)) +Upgrade all existing [named sessions](#named-sessions) inside the `sessions` subfolder of your [config directory](https://httpie.io/docs/cli/config-file-directory): ```bash -$ httpie cli sessions upgrade pie.dev api_auth -Refactored 'api_auth' (for 'pie.dev') to the version 3.1.0. +$ httpie cli sessions upgrade-all +Upgraded 'api_auth' @ 'pie.dev' to v3.1.0 +Upgraded 'login_cookies' @ 'httpie.io' to v3.1.0 ``` -([anonymous session](#anonymous-sessions)) +Upgrading individual sessions requires you to specify the session's hostname. That allows HTTPie to find the correct file in the case of name sessions. Additionally, it allows it to correctly bind cookies when upgrading with [`--bind-cookies`](#session-upgrade-options). + +Upgrade a single [named session](#named-sessions): ```bash -$ httpie cli sessions upgrade pie.dev ./session.json -Refactored 'session' (for 'pie.dev') to the version 3.1.0. +$ httpie cli sessions upgrade pie.dev api_auth +Upgraded 'api_auth' @ 'pie.dev' to v3.1.0 ``` -If you want to upgrade every existing [named session](#named-sessions), you can use `httpie cli sessions upgrade-all` (be aware -that this won't upgrade [anonymous sessions](#anonymous-sessions)): +Upgrade a single [anonymous session](#anonymous-sessions) using a file path: ```bash -$ httpie cli sessions upgrade-all -Refactored 'api_auth' (for 'pie.dev') to the version 3.1.0. -Refactored 'login_cookies' (for 'httpie.io') to the version 3.1.0. +$ httpie cli sessions upgrade pie.dev ./session.json +Upgraded 'session.json' @ 'pie.dev' to v3.1.0 ``` -#### Additional Customizations +#### Session upgrade options -| Flag | Description | -|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--bind-cookies` | Bind all the unbound cookies to the hostname that session belongs. By default, if the cookie is unbound (the `domain` attribute does not exist / set to an empty string) then it will still continue to be a generic cookie. | +These flags are available for both `sessions upgrade` and `sessions upgrade-all`: -These flags can be used to customize the defaults during an `upgrade` operation. They can -be used in both `sessions upgrade` and `sessions upgrade-all`. +------------------|------------------------------------------ +`--bind-cookies` | Bind all previously [unbound cookies](#host-based-cookie-policy) to the session’s host. ## Config @@ -2342,7 +2306,7 @@ To see the exact location for your installation, run `http --debug` and look for The default location of the configuration file on most platforms is `$XDG_CONFIG_HOME/httpie/config.json` (defaulting to `~/.config/httpie/config.json`). -For backwards compatibility, if the directory `~/.httpie` exists, the configuration file there will be used instead. +For backward compatibility, if the directory `~/.httpie` exists, the configuration file there will be used instead. On Windows, the config file is located at `%APPDATA%\httpie\config.json`. From 0a873172c95404b43387c1a4302eecc1cdb8379e Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 7 Mar 2022 20:55:51 +0100 Subject: [PATCH 33/36] Tweak SECURITY and add a Security policy section to docs --- SECURITY.md | 18 +++++++++++------- docs/README.md | 8 ++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index b10980cbb6..6d1b95da54 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,10 +1,14 @@ -# Security Policy +# Security policy -## Reporting a Vulnerability +## Reporting a vulnerability -To report a vulnerability, please send an email to `security@httpie.io` describing the: +When you identify a vulnerability in HTTPie, please report it privately using one of the following channels: -- The description of the vulnerability itself -- A short reproducer to verify it (you can submit a small HTTP server, a shell script, a docker image etc.) -- The severity level classification (`LOW`/`MEDIUM`/`HIGH`/`CRITICAL`) -- If associated with any, the [CWE](https://cwe.mitre.org/) ID. +- Email to [`security@httpie.io`](mailto:security@httpie.io) +- Report on [huntr.dev](https://huntr.dev/) + +In addition to the description of the vulnerability, please include also: + +- A short reproducer to verify it (it can be a small HTTP server, shell script, docker image, etc.) +- Your deemed severity level of the vulnerability (`LOW`/`MEDIUM`/`HIGH`/`CRITICAL`) +- [CWE](https://cwe.mitre.org/) ID, if available. diff --git a/docs/README.md b/docs/README.md index 5091295536..836c478dee 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2252,7 +2252,7 @@ $ http --session=./session.json pie.dev/headers Cookie:foo=bar In summary: -- Cookies set via the CLI overwrite cookies of the same name inside session files. +- Cookies set via the CLI overwrite cookies of the same name inside session files. - Server-sent `Set-Cookie` header cookies overwrite any pre-existing ones with the same name. Cookie expiration handling: @@ -2293,7 +2293,7 @@ Upgraded 'session.json' @ 'pie.dev' to v3.1.0 These flags are available for both `sessions upgrade` and `sessions upgrade-all`: ------------------|------------------------------------------ -`--bind-cookies` | Bind all previously [unbound cookies](#host-based-cookie-policy) to the session’s host. +`--bind-cookies` | Bind all previously [unbound cookies](#host-based-cookie-policy) to the session’s host. ## Config @@ -2532,6 +2532,10 @@ Helpers to convert from other client tools: See [CONTRIBUTING](https://github.com/httpie/httpie/blob/master/CONTRIBUTING.md). +### Security policy + +See [github.com/httpie/httpie/security/policy](https://github.com/httpie/httpie/security/policy). + ### Change log See [CHANGELOG](https://github.com/httpie/httpie/blob/master/CHANGELOG.md). From 59d9e928f8a6de4bd78e9e11adbabf9de9207d45 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 7 Mar 2022 20:57:03 +0100 Subject: [PATCH 34/36] Tweak --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 6d1b95da54..542bcd7854 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,7 +7,7 @@ When you identify a vulnerability in HTTPie, please report it privately using on - Email to [`security@httpie.io`](mailto:security@httpie.io) - Report on [huntr.dev](https://huntr.dev/) -In addition to the description of the vulnerability, please include also: +In addition to the description of the vulnerability, include the following information: - A short reproducer to verify it (it can be a small HTTP server, shell script, docker image, etc.) - Your deemed severity level of the vulnerability (`LOW`/`MEDIUM`/`HIGH`/`CRITICAL`) From f08c1bee178bade4a1a67213580dab2f6b00f6d5 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 7 Mar 2022 23:10:50 +0300 Subject: [PATCH 35/36] Change error messages to use a better format. --- httpie/manager/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/httpie/manager/tasks.py b/httpie/manager/tasks.py index 297767b025..f039a142f1 100644 --- a/httpie/manager/tasks.py +++ b/httpie/manager/tasks.py @@ -69,7 +69,7 @@ def upgrade_session(env: Environment, args: argparse.Namespace, hostname: str, s session_name = session.path.stem if session.is_new(): - env.log_error(f'{session_name!r} (for {hostname!r}) does not exist.') + env.log_error(f'{session_name!r} @ {hostname!r} does not exist.') return ExitStatus.ERROR fixers = [ @@ -79,14 +79,14 @@ def upgrade_session(env: Environment, args: argparse.Namespace, hostname: str, s ] if len(fixers) == 0: - env.stdout.write(f'{session_name!r} (for {hostname!r}) is already up-to-date.\n') + env.stdout.write(f'{session_name!r} @ {hostname!r} is already up to date.\n') return ExitStatus.SUCCESS for fixer in fixers: fixer(session, hostname, args) session.save(bump_version=True) - env.stdout.write(f'Refactored {session_name!r} (for {hostname!r}) to the version {session.version}.\n') + env.stdout.write(f'Upgraded {session_name!r} @ {hostname!r} to v{session.version}\n') return ExitStatus.SUCCESS From 7509dd4e6cd8caae20b0ce50a855ff1ebfc48f62 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 7 Mar 2022 23:22:28 +0300 Subject: [PATCH 36/36] Fix documentation styling errors. --- docs/README.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/README.md b/docs/README.md index 836c478dee..81b0aa7643 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2225,30 +2225,30 @@ There are three possible sources of persisted cookies within a session. They hav 1. Receive a response with a `Set-Cookie` header: -```bash -$ http --session=./session.json pie.dev/cookie/set?foo=bar -``` + ```bash + $ http --session=./session.json pie.dev/cookie/set?foo=bar + ``` 2. Send a cookie specified on the command line as seen in [cookies](#cookies): -```bash -$ http --session=./session.json pie.dev/headers Cookie:foo=bar -``` + ```bash + $ http --session=./session.json pie.dev/headers Cookie:foo=bar + ``` 3. Manually set cookie parameters in the session file: -```json -{ - "cookies": { - "foo": { - "expires": null, - "path": "/", - "secure": false, - "value": "bar" - } - } -} -``` + ```json + { + "cookies": { + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + } + } + ``` In summary: