From 9868e5db574410d3b188972917f8bee0e7045712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= Date: Tue, 19 Oct 2021 15:08:53 +0200 Subject: [PATCH] Add support for nested JSON syntax --- CHANGELOG.md | 2 + docs/README.md | 55 ++++++++++++- httpie/cli/constants.py | 6 +- httpie/cli/definition.py | 2 +- httpie/cli/json_form.py | 154 +++++++++++++++++++++++++++++++++++++ httpie/cli/requestitems.py | 32 +++++++- tests/test_cli.py | 6 +- tests/test_json.py | 92 ++++++++++++++++++++++ 8 files changed, 340 insertions(+), 9 deletions(-) create mode 100644 httpie/cli/json_form.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 35b54e9b4c..d24233acfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [2.7.0.dev0](https://github.com/httpie/httpie/compare/2.6.0...master) (unreleased) +- Added support for nested JSON syntax. ([#1169](https://github.com/httpie/httpie/issues/1169)) + ## [2.6.0](https://github.com/httpie/httpie/compare/2.5.0...2.6.0) (2021-10-14) - Added support for formatting & coloring of JSON bodies preceded by non-JSON data (e.g., an XXSI prefix). ([#1130](https://github.com/httpie/httpie/issues/1130)) diff --git a/docs/README.md b/docs/README.md index 688075fe4d..5fb5c2af67 100644 --- a/docs/README.md +++ b/docs/README.md @@ -698,6 +698,58 @@ Host: pie.dev } ``` +### Nested JSON fields + +Nested JSON fields can be set with both data field (`=`) and raw JSON field(`:=`) separators, which allows you to embed arbitrary JSON data into the resulting JSON object. +HTTPie supports the [JSON form](https://www.w3.org/TR/html-json-forms/) syntax. + +```bash +$ http PUT pie.dev/put \ + 'object=scalar' \ # Object — blank key + 'object[0]=array 1' \ # Object — "0" key + 'object[key]=key key' \ # Object — "key" key + 'array:=1' \ # Array — first item + 'array:=2' \ # Array — second item + 'array[]:=3' \ # Array — append (third item) + 'wow[such][deep][3][much][power][!]=Amaze' # Nested object +``` + +```http +PUT /person/1 HTTP/1.1 +Accept: application/json, */*;q=0.5 +Content-Type: application/json +Host: pie.dev + +{ + "array": [ + 1, + 2, + 3 + ], + "object": { + "": "scalar", + "0": "array 1", + "key": "key key" + }, + "wow": { + "such": { + "deep": [ + null, + null, + null, + { + "much": { + "power": { + "!": "Amaze" + } + } + } + ] + } + } +} +``` + ### Raw and complex JSON Please note that with the [request items](#request-items) data field syntax, commands can quickly become unwieldy when sending complex structures. @@ -715,9 +767,6 @@ $ http --raw '{"hello": "world"}' POST pie.dev/post $ http POST pie.dev/post < files/data.json ``` -Furthermore, the structure syntax only allows you to send an object as the JSON document, but not an array, etc. -Here, again, the solution is to use [redirected input](#redirected-input). - ## Forms Submitting forms is very similar to sending [JSON](#json) requests. diff --git a/httpie/cli/constants.py b/httpie/cli/constants.py index a1b78d33b2..ff9ce9f234 100644 --- a/httpie/cli/constants.py +++ b/httpie/cli/constants.py @@ -44,10 +44,10 @@ SEPARATOR_DATA_EMBED_RAW_JSON_FILE, }) -# Separators for raw JSON items -SEPARATOR_GROUP_RAW_JSON_ITEMS = frozenset([ +# Separators for nested JSON items +SEPARATOR_GROUP_NESTED_JSON_ITEMS = frozenset([ + SEPARATOR_DATA_STRING, SEPARATOR_DATA_RAW_JSON, - SEPARATOR_DATA_EMBED_RAW_JSON_FILE, ]) # Separators allowed in ITEM arguments diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 6ebb50c50e..f331819d49 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -118,7 +118,7 @@ '=@' A data field like '=', but takes a file path and embeds its content: - essay=@Documents/essay.txt + essay=@Documents/essay.txt ':=@' A raw JSON field like ':=', but takes a file path and embeds its content: diff --git a/httpie/cli/json_form.py b/httpie/cli/json_form.py new file mode 100644 index 0000000000..2efad7d7a0 --- /dev/null +++ b/httpie/cli/json_form.py @@ -0,0 +1,154 @@ +""" +Routines for JSON form syntax, used to support nested JSON request items. + +Highly inspired from the great jarg project . +""" +import re +from typing import Optional + + +def step(value: str, is_escaped: bool) -> str: + if is_escaped: + value = value.replace(r'\[', '[').replace(r'\]', ']') + return value + + +def find_opening_bracket( + value: str, + search=re.compile(r'(? Optional[int]: + match = search(value) + if not match: + return None + return match.start() + + +def find_closing_bracket( + value: str, + search=re.compile(r'(? Optional[int]: + match = search(value) + if not match: + return None + return match.start() + + +def parse_path(path): + """ + Parse a string as a JSON path. + + An implementation of "steps to parse a JSON encoding path". + + + """ + original = path + is_escaped = r'\[' in original + + opening_bracket = find_opening_bracket(original) + last_step = [(step(path, is_escaped), {'last': True, 'type': 'object'})] + if opening_bracket is None: + return last_step + + steps = [(step(original[:opening_bracket], is_escaped), {'type': 'object'})] + path = original[opening_bracket:] + while path: + if path.startswith('[]'): + steps[-1][1]['append'] = True + path = path[2:] + if path: + return last_step + elif path[0] == '[': + path = path[1:] + closing_bracket = find_closing_bracket(path) + if closing_bracket is None: + return last_step + key = path[:closing_bracket] + path = path[closing_bracket + 1:] + try: + steps.append((int(key), {'type': 'array'})) + except ValueError: + steps.append((key, {'type': 'object'})) + elif path[:2] == r'\[': + key = step(path[1:path.index(r'\]') + 2], is_escaped) + path = path[path.index(r'\]') + 2:] + steps.append((key, {'type': 'object'})) + else: + return last_step + + for i in range(len(steps) - 1): + steps[i][1]['type'] = steps[i + 1][1]['type'] + steps[-1][1]['last'] = True + return steps + + +def set_value(context, step, current_value, entry_value): + """Apply a JSON value to a context object. + + An implementation of "steps to set a JSON encoding value". + + + """ + key, flags = step + if flags.get('last', False): + if current_value is None: + if flags.get('append', False): + context[key] = [entry_value] + else: + if isinstance(context, list) and len(context) <= key: + context.extend([None] * (key - len(context) + 1)) + context[key] = entry_value + elif isinstance(current_value, list): + context[key].append(entry_value) + else: + context[key] = [current_value, entry_value] + return context + + if current_value is None: + if flags.get('type') == 'array': + context[key] = [] + else: + if isinstance(context, list) and len(context) <= key: + context.extend([None] * (key - len(context) + 1)) + context[key] = {} + return context[key] + + if isinstance(current_value, dict): + return context[key] + + if isinstance(current_value, list): + if flags.get('type') == 'array': + return current_value + + obj = {} + for i, item in enumerate(current_value): + if item is not None: + obj[i] = item + else: + context[key] = obj + return obj + + obj = {'': current_value} + context[key] = obj + return obj + + +def interpret_json_form(pairs): + """The application/json form encoding algorithm. + + + + """ + result = {} + for key, value in pairs: + steps = parse_path(key) + context = result + for step in steps: + try: + current_value = context.get(step[0], None) + except AttributeError: + try: + current_value = context[step[0]] + except IndexError: + current_value = None + context = set_value(context, step, current_value, value) + return result diff --git a/httpie/cli/requestitems.py b/httpie/cli/requestitems.py index 1dd6594ccd..11a6b2c0cf 100644 --- a/httpie/cli/requestitems.py +++ b/httpie/cli/requestitems.py @@ -4,7 +4,7 @@ from .argtypes import KeyValueArg from .constants import ( SEPARATORS_GROUP_MULTIPART, SEPARATOR_DATA_EMBED_FILE_CONTENTS, - SEPARATOR_DATA_EMBED_RAW_JSON_FILE, + SEPARATOR_DATA_EMBED_RAW_JSON_FILE, SEPARATOR_GROUP_NESTED_JSON_ITEMS, SEPARATOR_DATA_RAW_JSON, SEPARATOR_DATA_STRING, SEPARATOR_FILE_UPLOAD, SEPARATOR_FILE_UPLOAD_TYPE, SEPARATOR_HEADER, SEPARATOR_HEADER_EMPTY, SEPARATOR_QUERY_PARAM, @@ -15,6 +15,7 @@ RequestQueryParamsDict, ) from .exceptions import ParseError +from .json_form import interpret_json_form from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys @@ -60,6 +61,10 @@ def from_args( process_data_embed_file_contents_arg, instance.data, ), + SEPARATOR_GROUP_NESTED_JSON_ITEMS: ( + process_data_nested_json_embed_args, + instance.data, + ), SEPARATOR_DATA_RAW_JSON: ( process_data_raw_json_embed_arg, instance.data, @@ -70,6 +75,27 @@ def from_args( ), } + # Nested JSON items must be treated as a whole. + nested_json_items = [] + if not as_form: + nested_json_items.extend( + arg for arg in request_item_args + if arg.sep in SEPARATOR_GROUP_NESTED_JSON_ITEMS + ) + if nested_json_items: + request_item_args = [ + arg for arg in request_item_args + if arg not in nested_json_items + ] + pairs = [ + (arg.key, rules[arg.sep][0](arg)) + for arg in nested_json_items + ] + processor_func, target_dict = rules[SEPARATOR_GROUP_NESTED_JSON_ITEMS] + value = processor_func(pairs) + target_dict.update(value) + + # Then handle all other items. for arg in request_item_args: processor_func, target_dict = rules[arg.sep] value = processor_func(arg) @@ -134,6 +160,10 @@ def process_data_raw_json_embed_arg(arg: KeyValueArg) -> JSONType: return value +def process_data_nested_json_embed_args(pairs) -> Dict[str, JSONType]: + return interpret_json_form(pairs) + + def load_text_file(item: KeyValueArg) -> str: path = item.value try: diff --git a/tests/test_cli.py b/tests/test_cli.py index 4562deb6ca..a55e5a4499 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -79,6 +79,8 @@ def test_valid_items(self): self.key_value_arg('Empty-Header;'), self.key_value_arg('list:=["a", 1, {}, false]'), self.key_value_arg('obj:={"a": "b"}'), + self.key_value_arg(r'nested\[2\][a][]=1'), + self.key_value_arg('nested[2][a][]:=1'), self.key_value_arg('ed='), self.key_value_arg('bool:=true'), self.key_value_arg('file@' + FILE_PATH_ARG), @@ -104,7 +106,9 @@ def test_valid_items(self): 'ed': '', 'string': 'value', 'bool': True, - 'list': ['a', 1, {}, False], + 'list': ['a', 1, load_json_preserve_order_and_dupe_keys('{}'), False], + 'nested[2]': {'a': ['1']}, + 'nested': [None, None, {'a': [1]}], 'obj': load_json_preserve_order_and_dupe_keys('{"a": "b"}'), 'string-embed': FILE_CONTENT, } diff --git a/tests/test_json.py b/tests/test_json.py index 9b0f17ce68..1e32ea691e 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -116,3 +116,95 @@ def test_duplicate_keys_support_from_input_file(): # Check --unsorted r = http(*args, '--unsorted') assert JSON_WITH_DUPES_FORMATTED_UNSORTED in r + + +@pytest.mark.parametrize('input_json, expected_json', [ + # Examples taken from https://www.w3.org/TR/html-json-forms/ + ( + ['bottle-on-wall:=1', 'bottle-on-wall:=2', 'bottle-on-wall:=3'], + {'bottle-on-wall': [1, 2, 3]}, + ), + ( + ['pet[species]=Dahut', 'pet[name]:="Hypatia"', 'kids[1]=Thelma', 'kids[0]:="Ashley"'], + {'pet': {'species': 'Dahut', 'name': 'Hypatia'}, 'kids': ['Ashley', 'Thelma']}, + ), + ( + ['pet[0][species]=Dahut', 'pet[0][name]=Hypatia', 'pet[1][species]=Felis Stultus', 'pet[1][name]:="Billie"'], + {'pet': [{'species': 'Dahut', 'name': 'Hypatia'}, {'species': 'Felis Stultus', 'name': 'Billie'}]}, + ), + ( + ['wow[such][deep][3][much][power][!]=Amaze'], + {'wow': {'such': {'deep': [None, None, None, {'much': {'power': {'!': 'Amaze'}}}]}}}, + ), + ( + ['mix=scalar', 'mix[0]=array 1', 'mix[2]:="array 2"', 'mix[key]:="key key"', 'mix[car]=car key'], + {'mix': {'': 'scalar', '0': 'array 1', '2': 'array 2', 'key': 'key key', 'car': 'car key'}}, + ), + ( + ['highlander[]=one'], + {'highlander': ['one']}, + ), + ( + ['error[good]=BOOM!', 'error[bad:="BOOM BOOM!"'], + {'error': {'good': 'BOOM!'}, 'error[bad': 'BOOM BOOM!'}, + ), + ( + ['special[]:=true', 'special[]:=false', 'special[]:="true"', 'special[]:=null'], + {'special': [True, False, 'true', None]}, + ), + ( + [r'\[\]:=1', r'escape\[d\]:=1', r'escaped\[\]:=1', r'e\[s\][c][a][p]\[ed\][]:=1'], + {'[]': 1, 'escape[d]': 1, 'escaped[]': 1, 'e[s]': {'c': {'a': {'p': {'[ed]': [1]}}}}}, + ), + ( + ['[]:=1', '[]=foo'], + {'': [1, 'foo']}, + ), + ( + [']:=1', '[]1:=1', '[1]]:=1'], + {']': 1, '[]1': 1, '[1]]': 1}, + ), +]) +def test_nested_json_syntax(input_json, expected_json, httpbin_both): + r = http(httpbin_both + '/post', *input_json) + assert r.json['json'] == expected_json + + +def test_nested_json_sparse_array(httpbin_both): + r = http(httpbin_both + '/post', 'test[0]:=1', 'test[100]:=1') + assert len(r.json['json']['test']) == 101 + + +def test_mixed_json_syntax(httpbin_both): + input_json = ( + 'foo2:=true', + 'bar2:="123"', + 'order:={"map":{"1":"first","2":"second"}}', + 'a=b', + 'foo[]:="bar"', + 'foo[bar][baz][1][refoo]:="rebaz"', + ) + expected_json = { + "a": "b", + "bar2": "123", + "foo": { + "0": "bar", + "bar": { + "baz": [ + None, + { + "refoo": "rebaz" + } + ] + } + }, + "foo2": True, + "order": { + "map": { + "1": "first", + "2": "second" + } + } + } + r = http(httpbin_both + '/post', *input_json) + assert r.json['json'] == expected_json