From b653a8eb932f5e58cb6555da078f25cd5a4c5801 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 1 Feb 2022 12:14:24 +0300 Subject: [PATCH 1/7] Implement new style cookies --- docs/README.md | 140 ++++++++++ httpie/client.py | 6 +- httpie/config.py | 60 ++-- httpie/manager/cli.py | 43 ++- httpie/manager/core.py | 11 + httpie/manager/tasks.py | 134 +++++++++ httpie/sessions.py | 206 +++++++++++--- httpie/utils.py | 5 + setup.py | 1 + tests/conftest.py | 6 +- tests/fixtures/__init__.py | 24 ++ .../session_data/new/cookies_dict.json | 31 +++ .../new/cookies_dict_dev_version.json | 31 +++ .../new/cookies_dict_with_extras.json | 33 +++ .../session_data/new/empty_cookies_dict.json | 14 + .../session_data/new/empty_cookies_list.json | 14 + .../session_data/old/cookies_dict.json | 27 ++ .../old/cookies_dict_dev_version.json | 27 ++ .../old/cookies_dict_with_extras.json | 29 ++ .../session_data/old/empty_cookies_dict.json | 14 + .../session_data/old/empty_cookies_list.json | 14 + tests/test_cookie_on_redirects.py | 262 ++++++++++++++++++ tests/test_httpie_cli.py | 125 +++++++++ tests/test_plugins_cli.py | 43 --- tests/test_sessions.py | 186 ++++++++++++- tests/utils/__init__.py | 24 +- tests/utils/http_server.py | 13 + 27 files changed, 1406 insertions(+), 117 deletions(-) create mode 100644 httpie/manager/tasks.py create mode 100644 tests/fixtures/session_data/new/cookies_dict.json create mode 100644 tests/fixtures/session_data/new/cookies_dict_dev_version.json create mode 100644 tests/fixtures/session_data/new/cookies_dict_with_extras.json create mode 100644 tests/fixtures/session_data/new/empty_cookies_dict.json create mode 100644 tests/fixtures/session_data/new/empty_cookies_list.json create mode 100644 tests/fixtures/session_data/old/cookies_dict.json create mode 100644 tests/fixtures/session_data/old/cookies_dict_dev_version.json create mode 100644 tests/fixtures/session_data/old/cookies_dict_with_extras.json create mode 100644 tests/fixtures/session_data/old/empty_cookies_dict.json create mode 100644 tests/fixtures/session_data/old/empty_cookies_list.json create mode 100644 tests/test_cookie_on_redirects.py create mode 100644 tests/test_httpie_cli.py diff --git a/docs/README.md b/docs/README.md index 00aff4b842..efd579a343 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2157,6 +2157,85 @@ $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:orig- $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:new-value ``` +### Host-based Cookie Policy + +Cookies in stored HTTPie sessions have a `domain` field which is binding them to the +specified hostname. For example, in the following session: + +```json +{ + "cookies": [ + { + "domain": "pie.dev", + "name": "secret_cookie", + "value": "value_1" + }, + { + "domain": "httpbin.org", + "name": "secret_cookie", + "value": "value_2" + } + ] +} +``` + +we will send `Cookie:secret_cookie=value_1` only when you are making a request against `pie.dev` (it +also includes the domains, like `api.pie.dev`), and `Cookie:secret_cookie=value_2` when you use `httpbin.org`. + +```bash +$ http --session=./session.json pie.dev/cookies +``` + +```json +{ + "cookies": { + "secret_cookie": "value_1" + } +} +``` + +```bash +$ http --session=./session.json httpbin.org/cookies +``` + +```json +{ + "cookies": { + "secret_cookie": "value_2" + } +} +``` + +If you want to make a cookie domain unbound, you can simply set the `domain` +field to `null` by editing the session file directly: + +```json +{ + "cookies": [ + { + "domain": null, + "expires": null, + "name": "generic_cookie", + "path": "/", + "secure": false, + "value": "generic_value" + } + ] +} +``` + +```bash +$ http --session=./session.json pie.dev/cookies +``` + +```json +{ + "cookies": { + "generic_cookie": "generic_value" + } +} +``` + ### Cookie Storage Behavior **TL;DR:** Cookie storage priority: Server response > Command line request > Session file @@ -2208,6 +2287,50 @@ Expired cookies are never stored. If a cookie in a session file expires, it will be removed before sending a new request. If the server expires an existing cookie, it will also be removed from the session file. +### Upgrading Sessions + +In rare circumstances, HTTPie makes changes in it's session layout. For allowing a smoother transition of existing files +from the old layout to the new layout we offer 2 interfaces: + +- `httpie cli sessions upgrade` +- `httpie cli sessions upgrade-all` + + +With `httpie cli sessions upgrade`, you can upgrade a single session with it's name (or it's path, if it is an +[anonymous session](#anonymous-sessions)) and the hostname it belongs to. For example: + +([named session](#named-sessions)) + +```bash +$ httpie cli sessions upgrade pie.dev api_auth +Refactored 'api_auth' (for 'pie.dev') to the version 3.1.0. +``` + +([anonymous session](#anonymous-sessions)) + +```bash +$ httpie cli sessions upgrade pie.dev ./session.json +Refactored 'session' (for 'pie.dev') to the version 3.1.0. +``` + +If you want to upgrade every existing [named session](#named-sessions), you can use `httpie cli sessions upgrade-all` (be aware +that this won't upgrade [anonymous sessions](#anonymous-sessions)): + +```bash +$ httpie cli sessions upgrade-all +Refactored 'api_auth' (for 'pie.dev') to the version 3.1.0. +Refactored 'login_cookies' (for 'httpie.io') to the version 3.1.0. +``` + +#### Additional Customizations + +| Flag | Description | +|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--bind-cookies` | Bind all the unbound cookies to the hostname that session belongs. By default, if the cookie is unbound (the `domain` attribute does not exist / set to an empty string) then it will still continue to be a generic cookie. | + +These flags can be used to customize the defaults during an `upgrade` operation. They can +be used in both `sessions upgrade` and `sessions upgrade-all`. + ## Config HTTPie uses a simple `config.json` file. @@ -2299,6 +2422,23 @@ And since there’s neither data nor `EOF`, it will get stuck. So unless you’r Also, it might be good to set a connection `--timeout` limit to prevent your program from hanging if the server never responds. +### Security + +#### Exposure of Cookies To The 3rd Party Hosts On Redirects + +*Vulnerability Type*: [CWE-200](https://cwe.mitre.org/data/definitions/200.html) +*Severity Level*: LOW +*Affected Versions*: `<3.1.0` + +The handling of [cookies](#cookies) was not compatible with the [RFC 6265](https://datatracker.ietf.org/doc/html/rfc6265) +on the point of handling the `Domain` attribute when they were saved into [session](#sessions) files. All cookies were shared +across all hosts during the runtime, including redirects to the 3rd party hosts. + +This vulnerability has been fixed in [3.1.0](https://github.com/httpie/httpie/releases/tag/3.1.0) and the +[`httpie cli sessions upgrade`](#upgrading-sessions)/[`httpie cli sessions upgrade-all`]((#upgrading-sessions) commands +have been put in place in order to allow a smooth transition to the new session layout from the existing [session](#sessions) +files. + ## Plugin manager HTTPie offers extensibility through a [plugin API](https://github.com/httpie/httpie/blob/master/httpie/plugins/base.py), diff --git a/httpie/client.py b/httpie/client.py index 1984537c2b..530d589cae 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -44,6 +44,7 @@ def collect_messages( httpie_session_headers = None if args.session or args.session_read_only: httpie_session = get_httpie_session( + env=env, config_dir=env.config.directory, session_name=args.session or args.session_read_only, host=args.headers.get('Host'), @@ -130,10 +131,7 @@ def collect_messages( if httpie_session: if httpie_session.is_new() or not args.session_read_only: httpie_session.cookies = requests_session.cookies - httpie_session.remove_cookies( - # TODO: take path & domain into account? - cookie['name'] for cookie in expired_cookies - ) + httpie_session.remove_cookies(expired_cookies) httpie_session.save() diff --git a/httpie/config.py b/httpie/config.py index 28574e4ae7..f7fee5bdab 100644 --- a/httpie/config.py +++ b/httpie/config.py @@ -1,7 +1,7 @@ import json import os from pathlib import Path -from typing import Union +from typing import Any, Dict, Union from . import __version__ from .compat import is_windows @@ -62,6 +62,21 @@ class ConfigFileError(Exception): pass +def read_raw_config(config_type: str, path: Path) -> Dict[str, Any]: + try: + with path.open(encoding=UTF8) as f: + try: + return json.load(f) + except ValueError as e: + raise ConfigFileError( + f'invalid {config_type} file: {e} [{path}]' + ) + except FileNotFoundError: + pass + except OSError as e: + raise ConfigFileError(f'cannot read {config_type} file: {e}') + + class BaseConfigDict(dict): name = None helpurl = None @@ -77,26 +92,25 @@ def ensure_directory(self): def is_new(self) -> bool: return not self.path.exists() + def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Hook for processing the incoming config data.""" + return data + + def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Hook for processing the outgoing config data.""" + return data + def load(self): config_type = type(self).__name__.lower() - try: - with self.path.open(encoding=UTF8) as f: - try: - data = json.load(f) - except ValueError as e: - raise ConfigFileError( - f'invalid {config_type} file: {e} [{self.path}]' - ) - self.update(data) - except FileNotFoundError: - pass - except OSError as e: - raise ConfigFileError(f'cannot read {config_type} file: {e}') - - def save(self): - self['__meta__'] = { - 'httpie': __version__ - } + data = read_raw_config(config_type, self.path) + if data is not None: + data = self.pre_process_data(data) + self.update(data) + + def save(self, *, bump_version: bool = False): + self.setdefault('__meta__', {}) + if bump_version or 'httpie' not in self['__meta__']: + self['__meta__']['httpie'] = __version__ if self.helpurl: self['__meta__']['help'] = self.helpurl @@ -106,13 +120,19 @@ def save(self): self.ensure_directory() json_string = json.dumps( - obj=self, + obj=self.post_process_data(self), indent=4, sort_keys=True, ensure_ascii=True, ) self.path.write_text(json_string + '\n', encoding=UTF8) + @property + def version(self): + return self.get( + '__meta__', {} + ).get('httpie', __version__) + class Config(BaseConfigDict): FILENAME = 'config.json' diff --git a/httpie/manager/cli.py b/httpie/manager/cli.py index 11c63d0a31..9ad4eca6ba 100644 --- a/httpie/manager/cli.py +++ b/httpie/manager/cli.py @@ -2,6 +2,15 @@ from httpie.cli.argparser import HTTPieManagerArgumentParser from httpie import __version__ +CLI_SESSION_UPGRADE_FLAGS = [ + { + 'variadic': ['--bind-cookies'], + 'action': 'store_true', + 'default': False, + 'help': 'Bind domainless cookies to the host that session belongs.' + } +] + COMMANDS = { 'plugins': { 'help': 'Manage HTTPie plugins.', @@ -34,6 +43,34 @@ 'List all installed HTTPie plugins.' ], }, + 'cli': { + 'help': 'Manage HTTPie for Terminal', + 'sessions': { + 'help': 'Manage HTTPie sessions', + 'upgrade': [ + 'Upgrade the given HTTPie session with the latest ' + 'layout. A list of changes between different session versions ' + 'can be found in the official documentation.', + { + 'dest': 'hostname', + 'metavar': 'HOSTNAME', + 'help': 'The host this session belongs.' + }, + { + 'dest': 'session', + 'metavar': 'SESSION_NAME_OR_PATH', + 'help': 'The name or the path for the session that will be upgraded.' + }, + *CLI_SESSION_UPGRADE_FLAGS + ], + 'upgrade-all': [ + 'Upgrade all named sessions with the latest layout. A list of ' + 'changes between different session versions can be found in the official ' + 'documentation.', + *CLI_SESSION_UPGRADE_FLAGS + ], + } + } } @@ -54,6 +91,8 @@ def generate_subparsers(root, parent_parser, definitions): ) for command, properties in definitions.items(): is_subparser = isinstance(properties, dict) + properties = properties.copy() + descr = properties.pop('help', None) if is_subparser else properties.pop(0) command_parser = actions.add_parser(command, description=descr) command_parser.root = root @@ -62,7 +101,9 @@ def generate_subparsers(root, parent_parser, definitions): continue for argument in properties: - command_parser.add_argument(**argument) + argument = argument.copy() + variadic = argument.pop('variadic', []) + command_parser.add_argument(*variadic, **argument) parser = HTTPieManagerArgumentParser( diff --git a/httpie/manager/core.py b/httpie/manager/core.py index e2134b5527..1289fef1a4 100644 --- a/httpie/manager/core.py +++ b/httpie/manager/core.py @@ -1,9 +1,11 @@ import argparse +from typing import Optional from httpie.context import Environment from httpie.manager.plugins import PluginInstaller from httpie.status import ExitStatus from httpie.manager.cli import missing_subcommand, parser +from httpie.manager.tasks import CLI_TASKS MSG_COMMAND_CONFUSION = '''\ This command is only for managing HTTPie plugins. @@ -22,6 +24,13 @@ '''.rstrip("\n").format(args='POST pie.dev/post hello=world') +def dispatch_cli_task(env: Environment, action: Optional[str], args: argparse.Namespace) -> ExitStatus: + if action is None: + parser.error(missing_subcommand('cli')) + + return CLI_TASKS[action](env, args) + + def program(args: argparse.Namespace, env: Environment) -> ExitStatus: if args.action is None: parser.error(MSG_NAKED_INVOCATION) @@ -29,5 +38,7 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus: if args.action == 'plugins': plugins = PluginInstaller(env, debug=args.debug) return plugins.run(args.plugins_action, args) + elif args.action == 'cli': + return dispatch_cli_task(env, args.cli_action, args) return ExitStatus.SUCCESS diff --git a/httpie/manager/tasks.py b/httpie/manager/tasks.py new file mode 100644 index 0000000000..c04ed9bc3d --- /dev/null +++ b/httpie/manager/tasks.py @@ -0,0 +1,134 @@ +import argparse +from typing import TypeVar, Callable, Tuple + +from httpie.sessions import SESSIONS_DIR_NAME, Session, get_httpie_session +from httpie.status import ExitStatus +from httpie.context import Environment +from httpie.manager.cli import missing_subcommand, parser + +T = TypeVar('T') + +CLI_TASKS = {} + + +def task(name: str) -> Callable[[T], T]: + def wrapper(func: T) -> T: + CLI_TASKS[name] = func + return func + return wrapper + + +@task('sessions') +def cli_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus: + action = args.cli_sessions_action + if action is None: + parser.error(missing_subcommand('cli', 'sessions')) + + if action == 'upgrade': + return cli_upgrade_session(env, args) + elif action == 'upgrade-all': + return cli_upgrade_all_sessions(env, args) + else: + raise ValueError(f'Unexpected action: {action}') + + +def is_version_greater(version_1: str, version_2: str) -> bool: + # In an ideal scenerio, we would depend on `packaging` in order + # to offer PEP 440 compatible parsing. But since it might not be + # commonly available for outside packages, and since we are only + # going to parse HTTPie's own version it should be fine to compare + # this in a SemVer subset fashion. + + def split_version(version: str) -> Tuple[int, ...]: + parts = [] + for part in version.split('.')[:3]: + try: + parts.append(int(part)) + except ValueError: + break + return tuple(parts) + + return split_version(version_1) > split_version(version_2) + + +def fix_cookie_layout(session: Session, hostname: str, args: argparse.Namespace) -> None: + if not isinstance(session['cookies'], dict): + return None + + session['cookies'] = [ + { + 'name': key, + **value + } + for key, value in session['cookies'].items() + ] + for cookie in session.cookies: + if cookie.domain == '': + if args.bind_cookies: + cookie.domain = hostname + else: + cookie._rest['is_explicit_none'] = True + + +FIXERS_TO_VERSIONS = { + '3.1.0': fix_cookie_layout +} + + +def upgrade_session(env: Environment, args: argparse.Namespace, hostname: str, session_name: str): + session = get_httpie_session( + env=env, + config_dir=env.config.directory, + session_name=session_name, + host=hostname, + url=hostname, + refactor_mode=True + ) + + session_name = session.path.stem + if session.is_new(): + env.log_error(f'{session_name!r} (for {hostname!r}) does not exist.') + return ExitStatus.ERROR + + fixers = [ + fixer + for version, fixer in FIXERS_TO_VERSIONS.items() + if is_version_greater(version, session.version) + ] + + if len(fixers) == 0: + env.stdout.write(f'{session_name!r} (for {hostname!r}) is already up-to-date.\n') + return ExitStatus.SUCCESS + + for fixer in fixers: + fixer(session, hostname, args) + + session.save(bump_version=True) + env.stdout.write(f'Refactored {session_name!r} (for {hostname!r}) to the version {session.version}.\n') + return ExitStatus.SUCCESS + + +def cli_upgrade_session(env: Environment, args: argparse.Namespace) -> ExitStatus: + return upgrade_session( + env, + args=args, + hostname=args.hostname, + session_name=args.session + ) + + +def cli_upgrade_all_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus: + session_dir_path = env.config_dir / SESSIONS_DIR_NAME + + status = ExitStatus.SUCCESS + for host_path in session_dir_path.iterdir(): + hostname = host_path.name + for session_path in host_path.glob("*.json"): + session_name = session_path.stem + status |= upgrade_session( + env, + args=args, + hostname=hostname, + session_name=session_name + ) + return status diff --git a/httpie/sessions.py b/httpie/sessions.py index 176c03e76d..c23cb56852 100644 --- a/httpie/sessions.py +++ b/httpie/sessions.py @@ -6,15 +6,17 @@ import re from http.cookies import SimpleCookie +from http.cookiejar import Cookie from pathlib import Path -from typing import Iterable, Optional, Union -from urllib.parse import urlsplit +from typing import Any, Dict, Optional, Union from requests.auth import AuthBase -from requests.cookies import RequestsCookieJar, create_cookie +from requests.cookies import RequestsCookieJar, remove_cookie_by_name +from .context import Environment from .cli.dicts import HTTPHeadersDict from .config import BaseConfigDict, DEFAULT_CONFIG_DIR +from .utils import url_as_host from .plugins.registry import plugin_manager @@ -26,27 +28,88 @@ # SESSION_IGNORED_HEADER_PREFIXES = ['Content-', 'If-'] +# Cookie related options +KEPT_COOKIE_OPTIONS = ['name', 'expires', 'path', 'value', 'domain', 'secure'] +DEFAULT_COOKIE_PATH = '/' + +INSECURE_COOKIE_JAR_WARNING = '''\ +Outdated layout detected for the current session. Please consider updating it, +in order to not get affected by potential security problems. + +For fixing the current session: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade --bind-cookies {hostname} {session_id} + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade {hostname} {session_id} +''' + +INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS = '''\ + +For fixing all named sessions: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade-all --bind-cookies + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade-all + +See https://pie.co/docs/security for more information. +''' + + +def is_anonymous_session(session_name: str) -> bool: + return os.path.sep in session_name + + +def materialize_cookie(cookie: Cookie) -> Dict[str, Any]: + materialized_cookie = { + option: getattr(cookie, option) + for option in KEPT_COOKIE_OPTIONS + } + + if ( + cookie._rest.get('is_explicit_none') + and materialized_cookie['domain'] == '' + ): + materialized_cookie['domain'] = None + + return materialized_cookie + def get_httpie_session( + env: Environment, config_dir: Path, session_name: str, host: Optional[str], url: str, + *, + refactor_mode: bool = False ) -> 'Session': - if os.path.sep in session_name: + bound_hostname = host or url_as_host(url) + if not bound_hostname: + # HACK/FIXME: httpie-unixsocket's URLs have no hostname. + bound_hostname = 'localhost' + + # host:port => host_port + hostname = bound_hostname.replace(':', '_') + if is_anonymous_session(session_name): path = os.path.expanduser(session_name) + session_id = path else: - hostname = host or urlsplit(url).netloc.split('@')[-1] - if not hostname: - # HACK/FIXME: httpie-unixsocket's URLs have no hostname. - hostname = 'localhost' - - # host:port => host_port - hostname = hostname.replace(':', '_') path = ( config_dir / SESSIONS_DIR_NAME / hostname / f'{session_name}.json' ) - session = Session(path) + session_id = session_name + + session = Session( + path, + env=env, + session_id=session_id, + bound_host=bound_hostname.split(':')[0], + refactor_mode=refactor_mode + ) session.load() return session @@ -55,15 +118,86 @@ class Session(BaseConfigDict): helpurl = 'https://httpie.io/docs#sessions' about = 'HTTPie session file' - def __init__(self, path: Union[str, Path]): + def __init__( + self, + path: Union[str, Path], + env: Environment, + bound_host: str, + session_id: str, + refactor_mode: bool = False, + ): super().__init__(path=Path(path)) self['headers'] = {} - self['cookies'] = {} + self['cookies'] = [] self['auth'] = { 'type': None, 'username': None, 'password': None } + self.env = env + self.cookie_jar = RequestsCookieJar() + self.session_id = session_id + self.bound_host = bound_host + self.refactor_mode = refactor_mode + + def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + cookies = data.get('cookies') + if isinstance(cookies, dict): + normalized_cookies = [ + { + 'name': key, + **value + } + for key, value in cookies.items() + ] + elif isinstance(cookies, list): + normalized_cookies = cookies + else: + normalized_cookies = [] + + should_issue_warning = False + for cookie in normalized_cookies: + domain = cookie.get('domain', '') + if domain == '' and isinstance(cookies, dict): + should_issue_warning = True + elif domain is None: + # domain = None means explicitly lack of cookie, though + # requests requires domain to be string so we'll cast it + # manually. + cookie['domain'] = '' + cookie['rest'] = {'is_explicit_none': True} + + self.cookie_jar.set(**cookie) + + if should_issue_warning and not self.refactor_mode: + warning = INSECURE_COOKIE_JAR_WARNING.format(hostname=self.bound_host, session_id=self.session_id) + if not is_anonymous_session(self.session_id): + warning += INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS + + self.env.log_error( + warning, + level='warning' + ) + + return data + + def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + cookies = data.get('cookies') + # Save in the old-style fashion + + normalized_cookies = [ + materialize_cookie(cookie) + for cookie in self.cookie_jar + ] + if isinstance(cookies, dict): + data['cookies'] = { + cookie.pop('name'): cookie + for cookie in normalized_cookies + } + else: + data['cookies'] = normalized_cookies + + return data def update_headers(self, request_headers: HTTPHeadersDict): """ @@ -73,10 +207,10 @@ def update_headers(self, request_headers: HTTPHeadersDict): """ headers = self.headers for name, value in request_headers.copy().items(): - if value is None: continue # Ignore explicitly unset headers + original_value = value if type(value) is not str: value = value.decode() @@ -85,8 +219,15 @@ def update_headers(self, request_headers: HTTPHeadersDict): if name.lower() == 'cookie': for cookie_name, morsel in SimpleCookie(value).items(): - self['cookies'][cookie_name] = {'value': morsel.value} - del request_headers[name] + if not morsel['path']: + morsel['path'] = DEFAULT_COOKIE_PATH + self.cookie_jar.set(cookie_name, morsel) + + all_cookie_headers = request_headers.getall(name) + if len(all_cookie_headers) > 1: + all_cookie_headers.remove(original_value) + else: + request_headers.popall(name) continue for prefix in SESSION_IGNORED_HEADER_PREFIXES: @@ -103,23 +244,21 @@ def headers(self) -> HTTPHeadersDict: @property def cookies(self) -> RequestsCookieJar: - jar = RequestsCookieJar() - for name, cookie_dict in self['cookies'].items(): - jar.set_cookie(create_cookie( - name, cookie_dict.pop('value'), **cookie_dict)) - jar.clear_expired_cookies() - return jar + self.cookie_jar.clear_expired_cookies() + return self.cookie_jar @cookies.setter def cookies(self, jar: RequestsCookieJar): - # - stored_attrs = ['value', 'path', 'secure', 'expires'] - self['cookies'] = {} - for cookie in jar: - self['cookies'][cookie.name] = { - attname: getattr(cookie, attname) - for attname in stored_attrs - } + self.cookie_jar = jar + + def remove_cookies(self, cookies: Dict[str, str]): + for cookie in cookies: + remove_cookie_by_name( + self.cookie_jar, + cookie['name'], + domain=cookie.get('domain', None), + path=cookie.get('path', None) + ) @property def auth(self) -> Optional[AuthBase]: @@ -154,8 +293,3 @@ def auth(self) -> Optional[AuthBase]: def auth(self, auth: dict): assert {'type', 'raw_auth'} == auth.keys() self['auth'] = auth - - def remove_cookies(self, names: Iterable[str]): - for name in names: - if name in self['cookies']: - del self['cookies'][name] diff --git a/httpie/utils.py b/httpie/utils.py index fa19fa7cde..4fffb2826e 100644 --- a/httpie/utils.py +++ b/httpie/utils.py @@ -9,6 +9,7 @@ from http.cookiejar import parse_ns_headers from pathlib import Path from pprint import pformat +from urllib.parse import urlsplit from typing import Any, List, Optional, Tuple, Callable, Iterable, TypeVar import requests.auth @@ -237,3 +238,7 @@ def unwrap_context(exc: Exception) -> Optional[Exception]: return unwrap_context(context) else: return exc + + +def url_as_host(url: str) -> str: + return urlsplit(url).netloc.split('@')[-1] diff --git a/setup.py b/setup.py index 5316ff73d3..8f9a93140f 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ tests_require = [ 'pytest', 'pytest-httpbin>=0.0.6', + 'pytest-lazy-fixture>=0.0.6', 'responses', ] dev_require = [ diff --git a/tests/conftest.py b/tests/conftest.py index 5e8c511072..7ca0e60440 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,11 @@ import pytest from pytest_httpbin import certs -from .utils import HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, HTTPBIN_WITH_CHUNKED_SUPPORT +from .utils import ( # noqa + HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, + HTTPBIN_WITH_CHUNKED_SUPPORT, + mock_env +) from .utils.plugins_cli import ( # noqa broken_plugin, dummy_plugin, diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 126b13276e..6e6e73676e 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -1,6 +1,9 @@ """Test data""" +import json from pathlib import Path +from typing import Optional, Dict, Any +import httpie from httpie.encoding import UTF8 from httpie.output.formatters.xml import pretty_xml, parse_xml @@ -19,10 +22,20 @@ def patharg(path): JSON_FILE_PATH = FIXTURES_ROOT / 'test.json' JSON_WITH_DUPE_KEYS_FILE_PATH = FIXTURES_ROOT / 'test_with_dupe_keys.json' BIN_FILE_PATH = FIXTURES_ROOT / 'test.bin' + XML_FILES_PATH = FIXTURES_ROOT / 'xmldata' XML_FILES_VALID = list((XML_FILES_PATH / 'valid').glob('*_raw.xml')) XML_FILES_INVALID = list((XML_FILES_PATH / 'invalid').glob('*.xml')) +SESSION_FILES_PATH = FIXTURES_ROOT / 'session_data' +SESSION_FILES_OLD = sorted((SESSION_FILES_PATH / 'old').glob('*.json')) +SESSION_FILES_NEW = sorted((SESSION_FILES_PATH / 'new').glob('*.json')) + +SESSION_VARIABLES = { + '__version__': httpie.__version__, + '__host__': 'null', +} + FILE_PATH_ARG = patharg(FILE_PATH) BIN_FILE_PATH_ARG = patharg(BIN_FILE_PATH) JSON_FILE_PATH_ARG = patharg(JSON_FILE_PATH) @@ -40,3 +53,14 @@ def patharg(path): UNICODE = FILE_CONTENT XML_DATA_RAW = 'text' XML_DATA_FORMATTED = pretty_xml(parse_xml(XML_DATA_RAW)) + + +def read_session_file(session_file: Path, *, extra_variables: Optional[Dict[str, str]] = None) -> Any: + with open(session_file) as stream: + data = stream.read() + + session_vars = {**SESSION_VARIABLES, **(extra_variables or {})} + for variable, value in session_vars.items(): + data = data.replace(variable, value) + + return json.loads(data) diff --git a/tests/fixtures/session_data/new/cookies_dict.json b/tests/fixtures/session_data/new/cookies_dict.json new file mode 100644 index 0000000000..8a4d5f2e13 --- /dev/null +++ b/tests/fixtures/session_data/new/cookies_dict.json @@ -0,0 +1,31 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [ + { + "domain": __host__, + "expires": null, + "name": "baz", + "path": "/", + "secure": false, + "value": "quux" + }, + { + "domain": __host__, + "expires": null, + "name": "foo", + "path": "/", + "secure": false, + "value": "bar" + } + ], + "headers": {} +} diff --git a/tests/fixtures/session_data/new/cookies_dict_dev_version.json b/tests/fixtures/session_data/new/cookies_dict_dev_version.json new file mode 100644 index 0000000000..8a4d5f2e13 --- /dev/null +++ b/tests/fixtures/session_data/new/cookies_dict_dev_version.json @@ -0,0 +1,31 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [ + { + "domain": __host__, + "expires": null, + "name": "baz", + "path": "/", + "secure": false, + "value": "quux" + }, + { + "domain": __host__, + "expires": null, + "name": "foo", + "path": "/", + "secure": false, + "value": "bar" + } + ], + "headers": {} +} diff --git a/tests/fixtures/session_data/new/cookies_dict_with_extras.json b/tests/fixtures/session_data/new/cookies_dict_with_extras.json new file mode 100644 index 0000000000..9a99f15268 --- /dev/null +++ b/tests/fixtures/session_data/new/cookies_dict_with_extras.json @@ -0,0 +1,33 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "raw_auth": "foo:bar", + "type": "basic" + }, + "cookies": [ + { + "domain": __host__, + "expires": null, + "name": "baz", + "path": "/", + "secure": false, + "value": "quux" + }, + { + "domain": __host__, + "expires": null, + "name": "foo", + "path": "/", + "secure": false, + "value": "bar" + } + ], + "headers": { + "X-Data": "value", + "X-Foo": "bar" + } +} diff --git a/tests/fixtures/session_data/new/empty_cookies_dict.json b/tests/fixtures/session_data/new/empty_cookies_dict.json new file mode 100644 index 0000000000..1d01661a06 --- /dev/null +++ b/tests/fixtures/session_data/new/empty_cookies_dict.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [], + "headers": {} +} diff --git a/tests/fixtures/session_data/new/empty_cookies_list.json b/tests/fixtures/session_data/new/empty_cookies_list.json new file mode 100644 index 0000000000..1d01661a06 --- /dev/null +++ b/tests/fixtures/session_data/new/empty_cookies_list.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "__version__" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [], + "headers": {} +} diff --git a/tests/fixtures/session_data/old/cookies_dict.json b/tests/fixtures/session_data/old/cookies_dict.json new file mode 100644 index 0000000000..9c4fd21476 --- /dev/null +++ b/tests/fixtures/session_data/old/cookies_dict.json @@ -0,0 +1,27 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": { + "baz": { + "expires": null, + "path": "/", + "secure": false, + "value": "quux" + }, + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + }, + "headers": {} +} diff --git a/tests/fixtures/session_data/old/cookies_dict_dev_version.json b/tests/fixtures/session_data/old/cookies_dict_dev_version.json new file mode 100644 index 0000000000..935b43f083 --- /dev/null +++ b/tests/fixtures/session_data/old/cookies_dict_dev_version.json @@ -0,0 +1,27 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "2.7.0.dev0" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": { + "baz": { + "expires": null, + "path": "/", + "secure": false, + "value": "quux" + }, + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + }, + "headers": {} +} diff --git a/tests/fixtures/session_data/old/cookies_dict_with_extras.json b/tests/fixtures/session_data/old/cookies_dict_with_extras.json new file mode 100644 index 0000000000..42968e52a9 --- /dev/null +++ b/tests/fixtures/session_data/old/cookies_dict_with_extras.json @@ -0,0 +1,29 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "raw_auth": "foo:bar", + "type": "basic" + }, + "cookies": { + "baz": { + "expires": null, + "path": "/", + "secure": false, + "value": "quux" + }, + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + }, + "headers": { + "X-Data": "value", + "X-Foo": "bar" + } +} diff --git a/tests/fixtures/session_data/old/empty_cookies_dict.json b/tests/fixtures/session_data/old/empty_cookies_dict.json new file mode 100644 index 0000000000..8de1a9217c --- /dev/null +++ b/tests/fixtures/session_data/old/empty_cookies_dict.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": {}, + "headers": {} +} diff --git a/tests/fixtures/session_data/old/empty_cookies_list.json b/tests/fixtures/session_data/old/empty_cookies_list.json new file mode 100644 index 0000000000..12194f7ed2 --- /dev/null +++ b/tests/fixtures/session_data/old/empty_cookies_list.json @@ -0,0 +1,14 @@ +{ + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": { + "password": null, + "type": null, + "username": null + }, + "cookies": [], + "headers": {} +} diff --git a/tests/test_cookie_on_redirects.py b/tests/test_cookie_on_redirects.py new file mode 100644 index 0000000000..e22f833048 --- /dev/null +++ b/tests/test_cookie_on_redirects.py @@ -0,0 +1,262 @@ +import pytest +from .utils import http + + +@pytest.fixture +def remote_httpbin(httpbin_with_chunked_support): + return httpbin_with_chunked_support + + +def _stringify(fixture): + return fixture + '' + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +def test_explicit_user_set_cookie(httpbin, instance): + # User set cookies ARE NOT persisted within redirects + # when there is no session, even on the same domain. + + r = http( + '--follow', + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/cookies', + 'Cookie:a=b' + ) + assert r.json == {'cookies': {}} + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +def test_explicit_user_set_cookie_in_session(tmp_path, httpbin, instance): + # User set cookies ARE persisted within redirects + # when there is A session, even on the same domain. + + r = http( + '--follow', + '--session', + str(tmp_path / 'session.json'), + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/cookies', + 'Cookie:a=b' + ) + assert r.json == {'cookies': {'a': 'b'}} + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +def test_saved_user_set_cookie_in_session(tmp_path, httpbin, instance): + # User set cookies ARE persisted within redirects + # when there is A session, even on the same domain. + + http( + '--follow', + '--session', + str(tmp_path / 'session.json'), + httpbin + '/get', + 'Cookie:a=b' + ) + r = http( + '--follow', + '--session', + str(tmp_path / 'session.json'), + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/cookies', + ) + assert r.json == {'cookies': {'a': 'b'}} + + +@pytest.mark.parametrize('instance', [ + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), +]) +@pytest.mark.parametrize('session', [True, False]) +def test_explicit_user_set_headers(httpbin, tmp_path, instance, session): + # User set headers ARE persisted within redirects + # even on different domains domain with or without + # an active session. + session_args = [] + if session: + session_args.extend([ + '--session', + str(tmp_path / 'session.json') + ]) + + r = http( + '--follow', + *session_args, + httpbin + '/redirect-to', + f'url=={_stringify(instance)}/get', + 'X-Custom-Header:value' + ) + assert 'X-Custom-Header' in r.json['headers'] + + +@pytest.mark.parametrize('session', [True, False]) +def test_server_set_cookie_on_redirect_same_domain(tmp_path, httpbin, session): + # Server set cookies ARE persisted on the same domain + # when they are forwarded. + + session_args = [] + if session: + session_args.extend([ + '--session', + str(tmp_path / 'session.json') + ]) + + r = http( + '--follow', + *session_args, + httpbin + '/cookies/set/a/b', + ) + assert r.json['cookies'] == {'a': 'b'} + + +@pytest.mark.parametrize('session', [True, False]) +def test_server_set_cookie_on_redirect_different_domain(tmp_path, http_server, httpbin, session): + # Server set cookies ARE persisted on different domains + # when they are forwarded. + + session_args = [] + if session: + session_args.extend([ + '--session', + str(tmp_path / 'session.json') + ]) + + r = http( + '--follow', + *session_args, + http_server + '/cookies/set-and-redirect', + f"X-Redirect-To:{httpbin + '/cookies'}", + 'X-Cookies:a=b' + ) + assert r.json['cookies'] == {'a': 'b'} + + +def test_saved_session_cookies_on_same_domain(tmp_path, httpbin): + # Saved session cookies ARE persisted when making a new + # request to the same domain. + http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies/set/a/b' + ) + r = http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies' + ) + assert r.json == {'cookies': {'a': 'b'}} + + +def test_saved_session_cookies_on_different_domain(tmp_path, httpbin, remote_httpbin): + # Saved session cookies ARE persisted when making a new + # request to a different domain. + http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies/set/a/b' + ) + r = http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies' + ) + assert r.json == {'cookies': {}} + + +@pytest.mark.parametrize('initial_domain, first_request_domain, second_request_domain, expect_cookies', [ + ( + # Cookies are set by Domain A + # Initial domain is Domain A + # Redirected domain is Domain A + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('httpbin'), + True, + ), + ( + # Cookies are set by Domain A + # Initial domain is Domain B + # Redirected domain is Domain B + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), + pytest.lazy_fixture('remote_httpbin'), + False, + ), + ( + # Cookies are set by Domain A + # Initial domain is Domain A + # Redirected domain is Domain B + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), + False, + ), + ( + # Cookies are set by Domain A + # Initial domain is Domain B + # Redirected domain is Domain A + pytest.lazy_fixture('httpbin'), + pytest.lazy_fixture('remote_httpbin'), + pytest.lazy_fixture('httpbin'), + True, + ), +]) +def test_saved_session_cookies_on_redirect(tmp_path, initial_domain, first_request_domain, second_request_domain, expect_cookies): + http( + '--session', + str(tmp_path / 'session.json'), + initial_domain + '/cookies/set/a/b' + ) + r = http( + '--session', + str(tmp_path / 'session.json'), + '--follow', + first_request_domain + '/redirect-to', + f'url=={_stringify(second_request_domain)}/cookies' + ) + if expect_cookies: + expected_data = {'cookies': {'a': 'b'}} + else: + expected_data = {'cookies': {}} + assert r.json == expected_data + + +def test_saved_session_cookie_pool(tmp_path, httpbin, remote_httpbin): + http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies/set/a/b' + ) + http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies/set/a/c' + ) + http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies/set/b/d' + ) + + response = http( + '--session', + str(tmp_path / 'session.json'), + httpbin + '/cookies' + ) + assert response.json['cookies'] == {'a': 'b'} + + response = http( + '--session', + str(tmp_path / 'session.json'), + remote_httpbin + '/cookies' + ) + assert response.json['cookies'] == {'a': 'c', 'b': 'd'} diff --git a/tests/test_httpie_cli.py b/tests/test_httpie_cli.py new file mode 100644 index 0000000000..31c44d7f1f --- /dev/null +++ b/tests/test_httpie_cli.py @@ -0,0 +1,125 @@ +import pytest +import shutil +import json +from httpie.sessions import SESSIONS_DIR_NAME +from httpie.status import ExitStatus +from tests.utils import DUMMY_HOST, httpie +from tests.fixtures import SESSION_FILES_PATH, SESSION_FILES_NEW, SESSION_FILES_OLD, read_session_file + + +OLD_SESSION_FILES_PATH = SESSION_FILES_PATH / 'old' + + +@pytest.mark.requires_installation +def test_plugins_cli_error_message_without_args(): + # No arguments + result = httpie(no_debug=True) + assert result.exit_status == ExitStatus.ERROR + assert 'usage: ' in result.stderr + assert 'specify one of these' in result.stderr + assert 'please use the http/https commands:' in result.stderr + + +@pytest.mark.parametrize( + 'example', + [ + 'pie.dev/get', + 'DELETE localhost:8000/delete', + 'POST pie.dev/post header:value a=b header_2:value x:=1', + ], +) +@pytest.mark.requires_installation +def test_plugins_cli_error_messages_with_example(example): + result = httpie(*example.split(), no_debug=True) + assert result.exit_status == ExitStatus.ERROR + assert 'usage: ' in result.stderr + assert f'http {example}' in result.stderr + assert f'https {example}' in result.stderr + + +@pytest.mark.parametrize( + 'example', + [ + 'cli', + 'plugins', + 'cli foo', + 'plugins unknown', + 'plugins unknown.com A:B c=d', + 'unknown.com UNPARSABLE????SYNTAX', + ], +) +@pytest.mark.requires_installation +def test_plugins_cli_error_messages_invalid_example(example): + result = httpie(*example.split(), no_debug=True) + assert result.exit_status == ExitStatus.ERROR + assert 'usage: ' in result.stderr + assert f'http {example}' not in result.stderr + assert f'https {example}' not in result.stderr + + +HTTPIE_CLI_SESSIONS_UPGRADE_OPTIONS = [ + ( + # Default settings + [], + {'__host__': json.dumps(None)}, + ), + ( + # When --bind-cookies is applied, the __host__ becomes DUMMY_URL. + ['--bind-cookies'], + {'__host__': json.dumps(DUMMY_HOST)}, + ), +] + + +@pytest.mark.parametrize( + 'old_session_file, new_session_file', zip(SESSION_FILES_OLD, SESSION_FILES_NEW) +) +@pytest.mark.parametrize( + 'extra_args, extra_variables', + HTTPIE_CLI_SESSIONS_UPGRADE_OPTIONS, +) +def test_httpie_sessions_upgrade(tmp_path, old_session_file, new_session_file, extra_args, extra_variables): + session_path = tmp_path / 'session.json' + shutil.copyfile(old_session_file, session_path) + + result = httpie( + 'cli', 'sessions', 'upgrade', *extra_args, DUMMY_HOST, str(session_path) + ) + assert result.exit_status == ExitStatus.SUCCESS + assert read_session_file(session_path) == read_session_file( + new_session_file, extra_variables=extra_variables + ) + + +def test_httpie_sessions_upgrade_on_non_existent_file(tmp_path): + session_path = tmp_path / 'session.json' + result = httpie('cli', 'sessions', 'upgrade', DUMMY_HOST, str(session_path)) + assert result.exit_status == ExitStatus.ERROR + assert 'does not exist' in result.stderr + + +@pytest.mark.parametrize( + 'extra_args, extra_variables', + HTTPIE_CLI_SESSIONS_UPGRADE_OPTIONS, +) +def test_httpie_sessions_upgrade_all(tmp_path, mock_env, extra_args, extra_variables): + mock_env._create_temp_config_dir = False + mock_env.config_dir = tmp_path / "config" + + session_dir = mock_env.config_dir / SESSIONS_DIR_NAME / DUMMY_HOST + session_dir.mkdir(parents=True) + for original_session_file in SESSION_FILES_OLD: + shutil.copy(original_session_file, session_dir) + + result = httpie( + 'cli', 'sessions', 'upgrade-all', *extra_args, env=mock_env + ) + assert result.exit_status == ExitStatus.SUCCESS + + for refactored_session_file, expected_session_file in zip( + sorted(session_dir.glob("*.json")), + SESSION_FILES_NEW + ): + assert read_session_file(refactored_session_file) == read_session_file( + expected_session_file, extra_variables=extra_variables + ) diff --git a/tests/test_plugins_cli.py b/tests/test_plugins_cli.py index 9f94821505..70cecb1fb7 100644 --- a/tests/test_plugins_cli.py +++ b/tests/test_plugins_cli.py @@ -1,7 +1,6 @@ import pytest from httpie.status import ExitStatus -from tests.utils import httpie from tests.utils.plugins_cli import parse_listing @@ -149,45 +148,3 @@ def test_broken_plugins(httpie_plugins, httpie_plugins_success, dummy_plugin, br # No warning now, since it is uninstalled. data = parse_listing(httpie_plugins_success('list')) assert len(data) == 1 - - -@pytest.mark.requires_installation -def test_plugins_cli_error_message_without_args(): - # No arguments - result = httpie(no_debug=True) - assert result.exit_status == ExitStatus.ERROR - assert 'usage: ' in result.stderr - assert 'specify one of these' in result.stderr - assert 'please use the http/https commands:' in result.stderr - - -@pytest.mark.parametrize( - 'example', [ - 'pie.dev/get', - 'DELETE localhost:8000/delete', - 'POST pie.dev/post header:value a=b header_2:value x:=1' - ] -) -@pytest.mark.requires_installation -def test_plugins_cli_error_messages_with_example(example): - result = httpie(*example.split(), no_debug=True) - assert result.exit_status == ExitStatus.ERROR - assert 'usage: ' in result.stderr - assert f'http {example}' in result.stderr - assert f'https {example}' in result.stderr - - -@pytest.mark.parametrize( - 'example', [ - 'plugins unknown', - 'plugins unknown.com A:B c=d', - 'unknown.com UNPARSABLE????SYNTAX', - ] -) -@pytest.mark.requires_installation -def test_plugins_cli_error_messages_invalid_example(example): - result = httpie(*example.split(), no_debug=True) - assert result.exit_status == ExitStatus.ERROR - assert 'usage: ' in result.stderr - assert f'http {example}' not in result.stderr - assert f'https {example}' not in result.stderr diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 5835993605..8bcd906327 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -1,12 +1,16 @@ import json import os import shutil +from contextlib import contextmanager from datetime import datetime from unittest import mock +from pathlib import Path +from typing import Iterator import pytest from .fixtures import FILE_PATH_ARG, UNICODE +from httpie.context import Environment from httpie.encoding import UTF8 from httpie.plugins import AuthPlugin from httpie.plugins.builtin import HTTPBasicAuth @@ -14,7 +18,7 @@ from httpie.sessions import Session from httpie.utils import get_expired_cookies from .test_auth_plugins import basic_auth -from .utils import HTTP_OK, MockEnvironment, http, mk_config_dir +from .utils import DUMMY_HOST, HTTP_OK, MockEnvironment, http, mk_config_dir from base64 import b64encode @@ -203,9 +207,9 @@ def test_session_with_cookie_followed_by_another_header(self, httpbin): """ self.start_session(httpbin) session_data = { - "headers": { - "cookie": "...", - "zzz": "..." + 'headers': { + 'cookie': '...', + 'zzz': '...' } } session_path = self.config_dir / 'session-data.json' @@ -307,7 +311,7 @@ class Plugin(AuthPlugin): auth_type = 'test-prompted' def get_auth(self, username=None, password=None): - basic_auth_header = "Basic " + b64encode(self.raw_auth.encode()).strip().decode('latin1') + basic_auth_header = 'Basic ' + b64encode(self.raw_auth.encode()).strip().decode('latin1') return basic_auth(basic_auth_header) plugin_manager.register(Plugin) @@ -359,7 +363,7 @@ def get_auth(self, username=None, password=None): ) updated_session = json.loads(self.session_path.read_text(encoding=UTF8)) assert updated_session['auth']['type'] == 'test-saved' - assert updated_session['auth']['raw_auth'] == "user:password" + assert updated_session['auth']['raw_auth'] == 'user:password' plugin_manager.unregister(Plugin) @@ -368,12 +372,12 @@ class TestExpiredCookies(CookieTestBase): @pytest.mark.parametrize( 'initial_cookie, expired_cookie', [ - ({'id': {'value': 123}}, 'id'), - ({'id': {'value': 123}}, 'token') + ({'id': {'value': 123}}, {'name': 'id'}), + ({'id': {'value': 123}}, {'name': 'token'}) ] ) - def test_removes_expired_cookies_from_session_obj(self, initial_cookie, expired_cookie, httpbin): - session = Session(self.config_dir) + def test_removes_expired_cookies_from_session_obj(self, initial_cookie, expired_cookie, httpbin, mock_env): + session = Session(self.config_dir, env=mock_env, session_id=None, bound_host=None) session['cookies'] = initial_cookie session.remove_cookies([expired_cookie]) assert expired_cookie not in session.cookies @@ -524,3 +528,165 @@ def test_cookie_storage_priority(self, cli_cookie, set_cookie, expected, httpbin updated_session = json.loads(self.session_path.read_text(encoding=UTF8)) assert updated_session['cookies']['cookie1']['value'] == expected + + +@pytest.fixture +def basic_session(httpbin, tmp_path): + session_path = tmp_path / 'session.json' + http( + '--session', str(session_path), + httpbin + '/get' + ) + return session_path + + +@contextmanager +def open_session(path: Path, env: Environment, read_only: bool = False) -> Iterator[Session]: + session = Session(path, env, session_id='test', bound_host=DUMMY_HOST) + session.load() + yield session + if not read_only: + session.save() + + +@contextmanager +def open_raw_session(path: Path, read_only: bool = False) -> None: + with open(path) as stream: + raw_session = json.load(stream) + + yield raw_session + + if not read_only: + with open(path, 'w') as stream: + json.dump(raw_session, stream) + + +def read_stderr(env: Environment) -> bytes: + env.stderr.seek(0) + stderr_data = env.stderr.read() + if isinstance(stderr_data, str): + return stderr_data.encode() + else: + return stderr_data + + +def test_old_session_version_saved_as_is(basic_session, mock_env): + with open_session(basic_session, mock_env) as session: + session['__meta__'] = {'httpie': '0.0.1'} + + with open_session(basic_session, mock_env, read_only=True) as session: + assert session['__meta__']['httpie'] == '0.0.1' + + +def test_old_session_cookie_layout_warning(basic_session, mock_env): + with open_session(basic_session, mock_env) as session: + # Use the old layout & set a cookie + session['cookies'] = {} + session.cookies.set('foo', 'bar') + + assert read_stderr(mock_env) == b'' + + with open_session(basic_session, mock_env, read_only=True) as session: + assert b'Outdated layout detected' in read_stderr(mock_env) + + +@pytest.mark.parametrize('cookies, expect_warning', [ + # Old-style cookie format + ( + # Without 'domain' set + {'foo': {'value': 'bar'}}, + True + ), + ( + # With 'domain' set to empty string + {'foo': {'value': 'bar', 'domain': ''}}, + True + ), + ( + # With 'domain' set to null + {'foo': {'value': 'bar', 'domain': None}}, + False, + ), + ( + # With 'domain' set to a URL + {'foo': {'value': 'bar', 'domain': DUMMY_HOST}}, + False, + ), + # New style cookie format + ( + # Without 'domain' set + [{'name': 'foo', 'value': 'bar'}], + False + ), + ( + # With 'domain' set to empty string + [{'name': 'foo', 'value': 'bar', 'domain': ''}], + False + ), + ( + # With 'domain' set to null + [{'name': 'foo', 'value': 'bar', 'domain': None}], + False, + ), + ( + # With 'domain' set to a URL + [{'name': 'foo', 'value': 'bar', 'domain': DUMMY_HOST}], + False, + ), +]) +def test_cookie_security_warnings_on_raw_cookies(basic_session, mock_env, cookies, expect_warning): + with open_raw_session(basic_session) as raw_session: + raw_session['cookies'] = cookies + + with open_session(basic_session, mock_env, read_only=True): + warning = b'Outdated layout detected' + stderr = read_stderr(mock_env) + + if expect_warning: + assert warning in stderr + else: + assert warning not in stderr + + +def test_old_session_cookie_layout_loading(basic_session, httpbin, mock_env): + with open_session(basic_session, mock_env) as session: + # Use the old layout & set a cookie + session['cookies'] = {} + session.cookies.set('foo', 'bar') + + response = http( + '--session', str(basic_session), + httpbin + '/cookies' + ) + assert response.json['cookies'] == {'foo': 'bar'} + + +@pytest.mark.parametrize('layout_type', [ + dict, list +]) +def test_session_cookie_layout_preservance(basic_session, mock_env, layout_type): + with open_session(basic_session, mock_env) as session: + session['cookies'] = layout_type() + session.cookies.set('foo', 'bar') + session.save() + + with open_session(basic_session, mock_env, read_only=True) as session: + assert isinstance(session['cookies'], layout_type) + + +@pytest.mark.parametrize('layout_type', [ + dict, list +]) +def test_session_cookie_layout_preservance_on_new_cookies(basic_session, httpbin, mock_env, layout_type): + with open_session(basic_session, mock_env) as session: + session['cookies'] = layout_type() + session.cookies.set('foo', 'bar') + session.save() + + http( + '--session', str(basic_session), + httpbin + '/cookies/set/baz/quux' + ) + + with open_session(basic_session, mock_env, read_only=True) as session: + assert isinstance(session['cookies'], layout_type) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index cf90d684b9..d3359820c1 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -6,6 +6,8 @@ import json import tempfile import warnings +import pytest +from contextlib import suppress from io import BytesIO from pathlib import Path from typing import Any, Optional, Union, List, Iterable @@ -16,6 +18,7 @@ from httpie.status import ExitStatus from httpie.config import Config from httpie.context import Environment +from httpie.utils import url_as_host # pytest-httpbin currently does not support chunked requests: @@ -39,6 +42,7 @@ ) DUMMY_URL = 'http://this-should.never-resolve' # Note: URL never fetched +DUMMY_HOST = url_as_host(DUMMY_URL) def strip_colors(colorized_msg: str) -> str: @@ -187,6 +191,13 @@ class ExitStatusError(Exception): pass +@pytest.fixture +def mock_env() -> MockEnvironment: + env = MockEnvironment(stdout_mode='') + yield env + env.cleanup() + + def normalize_args(args: Iterable[Any]) -> List[str]: return [str(arg) for arg in args] @@ -201,7 +212,7 @@ def httpie( status. """ - env = kwargs.setdefault('env', MockEnvironment()) + env = kwargs.setdefault('env', MockEnvironment(stdout_mode='')) cli_args = ['httpie'] if not kwargs.pop('no_debug', False): cli_args.append('--debug') @@ -214,7 +225,16 @@ def httpie( env.stdout.seek(0) env.stderr.seek(0) try: - response = StrCLIResponse(env.stdout.read()) + output = env.stdout.read() + if isinstance(output, bytes): + with suppress(UnicodeDecodeError): + output = output.decode() + + if isinstance(output, bytes): + response = BytesCLIResponse(output) + else: + response = StrCLIResponse(output) + response.stderr = env.stderr.read() response.exit_status = exit_status response.args = cli_args diff --git a/tests/utils/http_server.py b/tests/utils/http_server.py index 0a96dd8b07..ecc14966b9 100644 --- a/tests/utils/http_server.py +++ b/tests/utils/http_server.py @@ -85,6 +85,19 @@ def status_custom_msg(handler): handler.end_headers() +@TestHandler.handler('GET', '/cookies/set-and-redirect') +def set_cookie_and_redirect(handler): + handler.send_response(302) + + redirect_to = handler.headers.get('X-Redirect-To', '/headers') + handler.send_header('Location', redirect_to) + + raw_cookies = handler.headers.get('X-Cookies', 'a=b') + for cookie in raw_cookies.split(', '): + handler.send_header('Set-Cookie', cookie) + handler.end_headers() + + @pytest.fixture(scope="function") def http_server(): """A custom HTTP server implementation for our tests, that is From b97e49277a9abfdfde62770b1e17ec811f2f7daa Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Fri, 4 Mar 2022 14:09:16 +0300 Subject: [PATCH 2/7] Apply suggestions from the review --- SECURITY.md | 10 ++++ docs/README.md | 17 ------ httpie/legacy/__init__.py | 0 httpie/legacy/cookie_format.py | 103 +++++++++++++++++++++++++++++++++ httpie/manager/cli.py | 6 +- httpie/manager/tasks.py | 24 +------- httpie/sessions.py | 94 ++++++++++-------------------- 7 files changed, 148 insertions(+), 106 deletions(-) create mode 100644 SECURITY.md create mode 100644 httpie/legacy/__init__.py create mode 100644 httpie/legacy/cookie_format.py diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..b10980cbb6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,10 @@ +# Security Policy + +## Reporting a Vulnerability + +To report a vulnerability, please send an email to `security@httpie.io` describing the: + +- The description of the vulnerability itself +- A short reproducer to verify it (you can submit a small HTTP server, a shell script, a docker image etc.) +- The severity level classification (`LOW`/`MEDIUM`/`HIGH`/`CRITICAL`) +- If associated with any, the [CWE](https://cwe.mitre.org/) ID. diff --git a/docs/README.md b/docs/README.md index efd579a343..30daa4b678 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2422,23 +2422,6 @@ And since there’s neither data nor `EOF`, it will get stuck. So unless you’r Also, it might be good to set a connection `--timeout` limit to prevent your program from hanging if the server never responds. -### Security - -#### Exposure of Cookies To The 3rd Party Hosts On Redirects - -*Vulnerability Type*: [CWE-200](https://cwe.mitre.org/data/definitions/200.html) -*Severity Level*: LOW -*Affected Versions*: `<3.1.0` - -The handling of [cookies](#cookies) was not compatible with the [RFC 6265](https://datatracker.ietf.org/doc/html/rfc6265) -on the point of handling the `Domain` attribute when they were saved into [session](#sessions) files. All cookies were shared -across all hosts during the runtime, including redirects to the 3rd party hosts. - -This vulnerability has been fixed in [3.1.0](https://github.com/httpie/httpie/releases/tag/3.1.0) and the -[`httpie cli sessions upgrade`](#upgrading-sessions)/[`httpie cli sessions upgrade-all`]((#upgrading-sessions) commands -have been put in place in order to allow a smooth transition to the new session layout from the existing [session](#sessions) -files. - ## Plugin manager HTTPie offers extensibility through a [plugin API](https://github.com/httpie/httpie/blob/master/httpie/plugins/base.py), diff --git a/httpie/legacy/__init__.py b/httpie/legacy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/httpie/legacy/cookie_format.py b/httpie/legacy/cookie_format.py new file mode 100644 index 0000000000..b5c6392b7c --- /dev/null +++ b/httpie/legacy/cookie_format.py @@ -0,0 +1,103 @@ +import argparse +from typing import Any, Type, List, Dict, TYPE_CHECKING + +if TYPE_CHECKING: + from httpie.sessions import Session + +INSECURE_COOKIE_JAR_WARNING = '''\ +Outdated layout detected for the current session. Please consider updating it, +in order to not get affected by potential security problems. + +For fixing the current session: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade --bind-cookies {hostname} {session_id} + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade {hostname} {session_id} +''' + + +INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS = '''\ + +For fixing all named sessions: + + With binding all cookies to the current host (secure): + $ httpie cli sessions upgrade-all --bind-cookies + + Without binding cookies (leaving them as is) (insecure): + $ httpie cli sessions upgrade-all +''' + +INSECURE_COOKIE_SECURITY_LINK = '\nSee https://pie.co/docs/security for more information.' + + +def pre_process(session: 'Session', cookies: Any) -> List[Dict[str, Any]]: + """Load the given cookies to the cookie jar while maintaining + support for the old cookie layout.""" + + is_old_style = isinstance(cookies, dict) + if is_old_style: + normalized_cookies = [ + { + 'name': key, + **value + } + for key, value in cookies.items() + ] + else: + normalized_cookies = cookies + + should_issue_warning = is_old_style and any( + cookie.get('domain', '') == '' + for cookie in normalized_cookies + ) + + if should_issue_warning and not session.refactor_mode: + warning = INSECURE_COOKIE_JAR_WARNING.format(hostname=session.bound_host, session_id=session.session_id) + if not session.is_anonymous: + warning += INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS + warning += INSECURE_COOKIE_SECURITY_LINK + + session.env.log_error( + warning, + level='warning' + ) + + return normalized_cookies + + +def post_process( + normalized_cookies: List[Dict[str, Any]], + *, + original_type: Type[Any] +) -> Any: + """Convert the cookies to their original format for + maximum compatibility.""" + + if issubclass(original_type, dict): + return { + cookie.pop('name'): cookie + for cookie in normalized_cookies + } + else: + return normalized_cookies + + +def fix_layout(session: 'Session', hostname: str, args: argparse.Namespace) -> None: + if not isinstance(session['cookies'], dict): + return None + + session['cookies'] = [ + { + 'name': key, + **value + } + for key, value in session['cookies'].items() + ] + for cookie in session.cookies: + if cookie.domain == '': + if args.bind_cookies: + cookie.domain = hostname + else: + cookie._rest['is_explicit_none'] = True diff --git a/httpie/manager/cli.py b/httpie/manager/cli.py index 9ad4eca6ba..1473ccf977 100644 --- a/httpie/manager/cli.py +++ b/httpie/manager/cli.py @@ -4,7 +4,7 @@ CLI_SESSION_UPGRADE_FLAGS = [ { - 'variadic': ['--bind-cookies'], + 'flags': ['--bind-cookies'], 'action': 'store_true', 'default': False, 'help': 'Bind domainless cookies to the host that session belongs.' @@ -102,8 +102,8 @@ def generate_subparsers(root, parent_parser, definitions): for argument in properties: argument = argument.copy() - variadic = argument.pop('variadic', []) - command_parser.add_argument(*variadic, **argument) + flags = argument.pop('flags', []) + command_parser.add_argument(*flags, **argument) parser = HTTPieManagerArgumentParser( diff --git a/httpie/manager/tasks.py b/httpie/manager/tasks.py index c04ed9bc3d..297767b025 100644 --- a/httpie/manager/tasks.py +++ b/httpie/manager/tasks.py @@ -1,9 +1,10 @@ import argparse from typing import TypeVar, Callable, Tuple -from httpie.sessions import SESSIONS_DIR_NAME, Session, get_httpie_session +from httpie.sessions import SESSIONS_DIR_NAME, get_httpie_session from httpie.status import ExitStatus from httpie.context import Environment +from httpie.legacy import cookie_format as legacy_cookies from httpie.manager.cli import missing_subcommand, parser T = TypeVar('T') @@ -51,27 +52,8 @@ def split_version(version: str) -> Tuple[int, ...]: return split_version(version_1) > split_version(version_2) -def fix_cookie_layout(session: Session, hostname: str, args: argparse.Namespace) -> None: - if not isinstance(session['cookies'], dict): - return None - - session['cookies'] = [ - { - 'name': key, - **value - } - for key, value in session['cookies'].items() - ] - for cookie in session.cookies: - if cookie.domain == '': - if args.bind_cookies: - cookie.domain = hostname - else: - cookie._rest['is_explicit_none'] = True - - FIXERS_TO_VERSIONS = { - '3.1.0': fix_cookie_layout + '3.1.0': legacy_cookies.fix_layout } diff --git a/httpie/sessions.py b/httpie/sessions.py index c23cb56852..e4a20a5344 100644 --- a/httpie/sessions.py +++ b/httpie/sessions.py @@ -8,7 +8,7 @@ from http.cookies import SimpleCookie from http.cookiejar import Cookie from pathlib import Path -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union from requests.auth import AuthBase from requests.cookies import RequestsCookieJar, remove_cookie_by_name @@ -18,6 +18,7 @@ from .config import BaseConfigDict, DEFAULT_CONFIG_DIR from .utils import url_as_host from .plugins.registry import plugin_manager +from .legacy import cookie_format as legacy_cookies SESSIONS_DIR_NAME = 'sessions' @@ -32,35 +33,23 @@ KEPT_COOKIE_OPTIONS = ['name', 'expires', 'path', 'value', 'domain', 'secure'] DEFAULT_COOKIE_PATH = '/' -INSECURE_COOKIE_JAR_WARNING = '''\ -Outdated layout detected for the current session. Please consider updating it, -in order to not get affected by potential security problems. -For fixing the current session: - - With binding all cookies to the current host (secure): - $ httpie cli sessions upgrade --bind-cookies {hostname} {session_id} - - Without binding cookies (leaving them as is) (insecure): - $ httpie cli sessions upgrade {hostname} {session_id} -''' - -INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS = '''\ - -For fixing all named sessions: - - With binding all cookies to the current host (secure): - $ httpie cli sessions upgrade-all --bind-cookies +def is_anonymous_session(session_name: str) -> bool: + return os.path.sep in session_name - Without binding cookies (leaving them as is) (insecure): - $ httpie cli sessions upgrade-all -See https://pie.co/docs/security for more information. -''' +def session_hostname_to_dirname(hostname: str, session_name: str) -> str: + # host:port => host_port + hostname = hostname.replace(':', '_') + return os.path.join( + SESSIONS_DIR_NAME, + hostname, + f'{session_name}.json' + ) -def is_anonymous_session(session_name: str) -> bool: - return os.path.sep in session_name +def strip_port(hostname: str) -> str: + return hostname.split(':')[0] def materialize_cookie(cookie: Cookie) -> Dict[str, Any]: @@ -92,22 +81,18 @@ def get_httpie_session( # HACK/FIXME: httpie-unixsocket's URLs have no hostname. bound_hostname = 'localhost' - # host:port => host_port - hostname = bound_hostname.replace(':', '_') if is_anonymous_session(session_name): path = os.path.expanduser(session_name) session_id = path else: - path = ( - config_dir / SESSIONS_DIR_NAME / hostname / f'{session_name}.json' - ) + path = config_dir / session_hostname_to_dirname(bound_hostname, session_name) session_id = session_name session = Session( path, env=env, session_id=session_id, - bound_host=bound_hostname.split(':')[0], + bound_host=strip_port(bound_hostname), refactor_mode=refactor_mode ) session.load() @@ -142,60 +127,35 @@ def __init__( def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: cookies = data.get('cookies') - if isinstance(cookies, dict): - normalized_cookies = [ - { - 'name': key, - **value - } - for key, value in cookies.items() - ] - elif isinstance(cookies, list): - normalized_cookies = cookies + if cookies: + normalized_cookies = legacy_cookies.pre_process(self, cookies) else: normalized_cookies = [] - should_issue_warning = False for cookie in normalized_cookies: domain = cookie.get('domain', '') - if domain == '' and isinstance(cookies, dict): - should_issue_warning = True - elif domain is None: + if domain is None: # domain = None means explicitly lack of cookie, though - # requests requires domain to be string so we'll cast it + # requests requires domain to be a string so we'll cast it # manually. cookie['domain'] = '' cookie['rest'] = {'is_explicit_none': True} self.cookie_jar.set(**cookie) - if should_issue_warning and not self.refactor_mode: - warning = INSECURE_COOKIE_JAR_WARNING.format(hostname=self.bound_host, session_id=self.session_id) - if not is_anonymous_session(self.session_id): - warning += INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS - - self.env.log_error( - warning, - level='warning' - ) - return data def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: cookies = data.get('cookies') - # Save in the old-style fashion normalized_cookies = [ materialize_cookie(cookie) for cookie in self.cookie_jar ] - if isinstance(cookies, dict): - data['cookies'] = { - cookie.pop('name'): cookie - for cookie in normalized_cookies - } - else: - data['cookies'] = normalized_cookies + data['cookies'] = legacy_cookies.post_process( + normalized_cookies, + original_type=type(cookies) + ) return data @@ -251,7 +211,7 @@ def cookies(self) -> RequestsCookieJar: def cookies(self, jar: RequestsCookieJar): self.cookie_jar = jar - def remove_cookies(self, cookies: Dict[str, str]): + def remove_cookies(self, cookies: List[Dict[str, str]]): for cookie in cookies: remove_cookie_by_name( self.cookie_jar, @@ -293,3 +253,7 @@ def auth(self) -> Optional[AuthBase]: def auth(self, auth: dict): assert {'type', 'raw_auth'} == auth.keys() self['auth'] = auth + + @property + def is_anonymous(self): + return is_anonymous_session(self.session_id) From e5e8a6e9982fe9932ed10368681a7c936d5b51ce Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 7 Mar 2022 20:43:48 +0100 Subject: [PATCH 3/7] Polish sessions docs --- docs/README.md | 154 +++++++++++++++++++------------------------------ 1 file changed, 59 insertions(+), 95 deletions(-) diff --git a/docs/README.md b/docs/README.md index 30daa4b678..5091295536 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2157,30 +2157,28 @@ $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:orig- $ http --session-read-only=./ro-session.json pie.dev/headers Custom-Header:new-value ``` -### Host-based Cookie Policy +### Host-based cookie policy -Cookies in stored HTTPie sessions have a `domain` field which is binding them to the -specified hostname. For example, in the following session: +Cookies persisted in sessions files have a `domain` field. This _binds_ them to a specified hostname. For example: ```json { "cookies": [ { "domain": "pie.dev", - "name": "secret_cookie", - "value": "value_1" + "name": "pie", + "value": "apple" }, { "domain": "httpbin.org", - "name": "secret_cookie", - "value": "value_2" + "name": "bin", + "value": "http" } ] } ``` -we will send `Cookie:secret_cookie=value_1` only when you are making a request against `pie.dev` (it -also includes the domains, like `api.pie.dev`), and `Cookie:secret_cookie=value_2` when you use `httpbin.org`. +Using this session file, we include `Cookie: pie=apple` only in requests against `pie.dev` and subdomains (e.g., `foo.pie.dev` or `foo.bar.pie.dev`): ```bash $ http --session=./session.json pie.dev/cookies @@ -2189,36 +2187,20 @@ $ http --session=./session.json pie.dev/cookies ```json { "cookies": { - "secret_cookie": "value_1" + "pie": "apple" } } ``` -```bash -$ http --session=./session.json httpbin.org/cookies -``` - -```json -{ - "cookies": { - "secret_cookie": "value_2" - } -} -``` - -If you want to make a cookie domain unbound, you can simply set the `domain` -field to `null` by editing the session file directly: +To make a cookie domain _unbound_ (i.e., to make it available to all hosts, including throughout a cross-domain redirect chain), you can set the `domain` field to `null` in the session file: ```json { "cookies": [ { "domain": null, - "expires": null, - "name": "generic_cookie", - "path": "/", - "secure": false, - "value": "generic_value" + "name": "unbound-cookie", + "value": "send-me-to-any-host" } ] } @@ -2231,105 +2213,87 @@ $ http --session=./session.json pie.dev/cookies ```json { "cookies": { - "generic_cookie": "generic_value" + "unbound-cookie": "send-me-to-any-host" } } ``` -### Cookie Storage Behavior -**TL;DR:** Cookie storage priority: Server response > Command line request > Session file +### Cookie storage behavior -To set a cookie within a Session there are three options: +There are three possible sources of persisted cookies within a session. They have the following storage priority: 1—response; 2—command line; 3—session file. -1. Get a `Set-Cookie` header in a response from a server +1. Receive a response with a `Set-Cookie` header: - ```bash - $ http --session=./session.json pie.dev/cookie/set?foo=bar - ``` +```bash +$ http --session=./session.json pie.dev/cookie/set?foo=bar +``` -2. Set the cookie name and value through the command line as seen in [cookies](#cookies) +2. Send a cookie specified on the command line as seen in [cookies](#cookies): - ```bash - $ http --session=./session.json pie.dev/headers Cookie:foo=bar - ``` +```bash +$ http --session=./session.json pie.dev/headers Cookie:foo=bar +``` -3. Manually set cookie parameters in the JSON file of the session +3. Manually set cookie parameters in the session file: - ```json - { - "__meta__": { - "about": "HTTPie session file", - "help": "https://httpie.org/doc#sessions", - "httpie": "2.2.0-dev" - }, - "auth": { - "password": null, - "type": null, - "username": null - }, - "cookies": { - "foo": { - "expires": null, - "path": "/", - "secure": false, - "value": "bar" - } - } +```json +{ + "cookies": { + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } } - ``` - -Cookies will be set in the session file with the priority specified above. -For example, a cookie set through the command line will overwrite a cookie of the same name stored in the session file. -If the server returns a `Set-Cookie` header with a cookie of the same name, the returned cookie will overwrite the preexisting cookie. +} +``` -Expired cookies are never stored. -If a cookie in a session file expires, it will be removed before sending a new request. -If the server expires an existing cookie, it will also be removed from the session file. +In summary: -### Upgrading Sessions +- Cookies set via the CLI overwrite cookies of the same name inside session files. +- Server-sent `Set-Cookie` header cookies overwrite any pre-existing ones with the same name. -In rare circumstances, HTTPie makes changes in it's session layout. For allowing a smoother transition of existing files -from the old layout to the new layout we offer 2 interfaces: +Cookie expiration handling: -- `httpie cli sessions upgrade` -- `httpie cli sessions upgrade-all` +- When the server expires an existing cookie, HTTPie removes it from the session file. +- When a cookie in a session file expires, HTTPie removes it before sending a new request. +### Upgrading sessions -With `httpie cli sessions upgrade`, you can upgrade a single session with it's name (or it's path, if it is an -[anonymous session](#anonymous-sessions)) and the hostname it belongs to. For example: +HTTPie may introduce changes in the session file format. When HTTPie detects an obsolete format, it shows a warning. You can upgrade your session files using the following commands: -([named session](#named-sessions)) +Upgrade all existing [named sessions](#named-sessions) inside the `sessions` subfolder of your [config directory](https://httpie.io/docs/cli/config-file-directory): ```bash -$ httpie cli sessions upgrade pie.dev api_auth -Refactored 'api_auth' (for 'pie.dev') to the version 3.1.0. +$ httpie cli sessions upgrade-all +Upgraded 'api_auth' @ 'pie.dev' to v3.1.0 +Upgraded 'login_cookies' @ 'httpie.io' to v3.1.0 ``` -([anonymous session](#anonymous-sessions)) +Upgrading individual sessions requires you to specify the session's hostname. That allows HTTPie to find the correct file in the case of name sessions. Additionally, it allows it to correctly bind cookies when upgrading with [`--bind-cookies`](#session-upgrade-options). + +Upgrade a single [named session](#named-sessions): ```bash -$ httpie cli sessions upgrade pie.dev ./session.json -Refactored 'session' (for 'pie.dev') to the version 3.1.0. +$ httpie cli sessions upgrade pie.dev api_auth +Upgraded 'api_auth' @ 'pie.dev' to v3.1.0 ``` -If you want to upgrade every existing [named session](#named-sessions), you can use `httpie cli sessions upgrade-all` (be aware -that this won't upgrade [anonymous sessions](#anonymous-sessions)): +Upgrade a single [anonymous session](#anonymous-sessions) using a file path: ```bash -$ httpie cli sessions upgrade-all -Refactored 'api_auth' (for 'pie.dev') to the version 3.1.0. -Refactored 'login_cookies' (for 'httpie.io') to the version 3.1.0. +$ httpie cli sessions upgrade pie.dev ./session.json +Upgraded 'session.json' @ 'pie.dev' to v3.1.0 ``` -#### Additional Customizations +#### Session upgrade options -| Flag | Description | -|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--bind-cookies` | Bind all the unbound cookies to the hostname that session belongs. By default, if the cookie is unbound (the `domain` attribute does not exist / set to an empty string) then it will still continue to be a generic cookie. | +These flags are available for both `sessions upgrade` and `sessions upgrade-all`: -These flags can be used to customize the defaults during an `upgrade` operation. They can -be used in both `sessions upgrade` and `sessions upgrade-all`. +------------------|------------------------------------------ +`--bind-cookies` | Bind all previously [unbound cookies](#host-based-cookie-policy) to the session’s host. ## Config @@ -2342,7 +2306,7 @@ To see the exact location for your installation, run `http --debug` and look for The default location of the configuration file on most platforms is `$XDG_CONFIG_HOME/httpie/config.json` (defaulting to `~/.config/httpie/config.json`). -For backwards compatibility, if the directory `~/.httpie` exists, the configuration file there will be used instead. +For backward compatibility, if the directory `~/.httpie` exists, the configuration file there will be used instead. On Windows, the config file is located at `%APPDATA%\httpie\config.json`. From b66baa7a4b015f78dea241c531009ab0d66384d2 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 7 Mar 2022 20:55:51 +0100 Subject: [PATCH 4/7] Tweak SECURITY and add a Security policy section to docs --- SECURITY.md | 18 +++++++++++------- docs/README.md | 8 ++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index b10980cbb6..6d1b95da54 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,10 +1,14 @@ -# Security Policy +# Security policy -## Reporting a Vulnerability +## Reporting a vulnerability -To report a vulnerability, please send an email to `security@httpie.io` describing the: +When you identify a vulnerability in HTTPie, please report it privately using one of the following channels: -- The description of the vulnerability itself -- A short reproducer to verify it (you can submit a small HTTP server, a shell script, a docker image etc.) -- The severity level classification (`LOW`/`MEDIUM`/`HIGH`/`CRITICAL`) -- If associated with any, the [CWE](https://cwe.mitre.org/) ID. +- Email to [`security@httpie.io`](mailto:security@httpie.io) +- Report on [huntr.dev](https://huntr.dev/) + +In addition to the description of the vulnerability, please include also: + +- A short reproducer to verify it (it can be a small HTTP server, shell script, docker image, etc.) +- Your deemed severity level of the vulnerability (`LOW`/`MEDIUM`/`HIGH`/`CRITICAL`) +- [CWE](https://cwe.mitre.org/) ID, if available. diff --git a/docs/README.md b/docs/README.md index 5091295536..836c478dee 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2252,7 +2252,7 @@ $ http --session=./session.json pie.dev/headers Cookie:foo=bar In summary: -- Cookies set via the CLI overwrite cookies of the same name inside session files. +- Cookies set via the CLI overwrite cookies of the same name inside session files. - Server-sent `Set-Cookie` header cookies overwrite any pre-existing ones with the same name. Cookie expiration handling: @@ -2293,7 +2293,7 @@ Upgraded 'session.json' @ 'pie.dev' to v3.1.0 These flags are available for both `sessions upgrade` and `sessions upgrade-all`: ------------------|------------------------------------------ -`--bind-cookies` | Bind all previously [unbound cookies](#host-based-cookie-policy) to the session’s host. +`--bind-cookies` | Bind all previously [unbound cookies](#host-based-cookie-policy) to the session’s host. ## Config @@ -2532,6 +2532,10 @@ Helpers to convert from other client tools: See [CONTRIBUTING](https://github.com/httpie/httpie/blob/master/CONTRIBUTING.md). +### Security policy + +See [github.com/httpie/httpie/security/policy](https://github.com/httpie/httpie/security/policy). + ### Change log See [CHANGELOG](https://github.com/httpie/httpie/blob/master/CHANGELOG.md). From 39b25e87e42e7a3587290fa9963ad4d4cc280fb8 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 7 Mar 2022 20:57:03 +0100 Subject: [PATCH 5/7] Tweak --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 6d1b95da54..542bcd7854 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,7 +7,7 @@ When you identify a vulnerability in HTTPie, please report it privately using on - Email to [`security@httpie.io`](mailto:security@httpie.io) - Report on [huntr.dev](https://huntr.dev/) -In addition to the description of the vulnerability, please include also: +In addition to the description of the vulnerability, include the following information: - A short reproducer to verify it (it can be a small HTTP server, shell script, docker image, etc.) - Your deemed severity level of the vulnerability (`LOW`/`MEDIUM`/`HIGH`/`CRITICAL`) From 176d378310047cb4245123d902195e95d74e4998 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 7 Mar 2022 23:10:50 +0300 Subject: [PATCH 6/7] Change error messages to use a better format. --- httpie/manager/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/httpie/manager/tasks.py b/httpie/manager/tasks.py index 297767b025..f039a142f1 100644 --- a/httpie/manager/tasks.py +++ b/httpie/manager/tasks.py @@ -69,7 +69,7 @@ def upgrade_session(env: Environment, args: argparse.Namespace, hostname: str, s session_name = session.path.stem if session.is_new(): - env.log_error(f'{session_name!r} (for {hostname!r}) does not exist.') + env.log_error(f'{session_name!r} @ {hostname!r} does not exist.') return ExitStatus.ERROR fixers = [ @@ -79,14 +79,14 @@ def upgrade_session(env: Environment, args: argparse.Namespace, hostname: str, s ] if len(fixers) == 0: - env.stdout.write(f'{session_name!r} (for {hostname!r}) is already up-to-date.\n') + env.stdout.write(f'{session_name!r} @ {hostname!r} is already up to date.\n') return ExitStatus.SUCCESS for fixer in fixers: fixer(session, hostname, args) session.save(bump_version=True) - env.stdout.write(f'Refactored {session_name!r} (for {hostname!r}) to the version {session.version}.\n') + env.stdout.write(f'Upgraded {session_name!r} @ {hostname!r} to v{session.version}\n') return ExitStatus.SUCCESS From 751faf0edee32be77220034e38145cb537529f42 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 7 Mar 2022 23:22:28 +0300 Subject: [PATCH 7/7] Fix documentation styling errors. --- docs/README.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/README.md b/docs/README.md index 836c478dee..81b0aa7643 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2225,30 +2225,30 @@ There are three possible sources of persisted cookies within a session. They hav 1. Receive a response with a `Set-Cookie` header: -```bash -$ http --session=./session.json pie.dev/cookie/set?foo=bar -``` + ```bash + $ http --session=./session.json pie.dev/cookie/set?foo=bar + ``` 2. Send a cookie specified on the command line as seen in [cookies](#cookies): -```bash -$ http --session=./session.json pie.dev/headers Cookie:foo=bar -``` + ```bash + $ http --session=./session.json pie.dev/headers Cookie:foo=bar + ``` 3. Manually set cookie parameters in the session file: -```json -{ - "cookies": { - "foo": { - "expires": null, - "path": "/", - "secure": false, - "value": "bar" - } - } -} -``` + ```json + { + "cookies": { + "foo": { + "expires": null, + "path": "/", + "secure": false, + "value": "bar" + } + } + } + ``` In summary: