From 2d01d6a20f39881c692977564c1ea591d9f39027 Mon Sep 17 00:00:00 2001 From: Marcono1234 Date: Thu, 23 Jun 2022 02:42:19 +0200 Subject: [PATCH] Make `Object` and `JsonElement` deserialization iterative (#1912) * Make Object and JsonElement deserialization iterative Often when Object and JsonElement are deserialized the format of the JSON data is unknown and it might come from an untrusted source. To avoid a StackOverflowError from maliciously crafted JSON, deserialize Object and JsonElement iteratively instead of recursively. Concept based on https://github.com/FasterXML/jackson-databind/commit/51fd2faab70c9c8eb7ae43c200f8480f24ba74d8 But implementation is not based on it. * Improve imports grouping * Address review feedback --- .../gson/internal/bind/ObjectTypeAdapter.java | 117 ++++++++++---- .../gson/internal/bind/TypeAdapters.java | 152 ++++++++++++------ .../gson/JsonParserParameterizedTest.java | 41 +++++ .../java/com/google/gson/JsonParserTest.java | 50 +++++- .../ObjectTypeAdapterParameterizedTest.java | 41 +++++ .../google/gson/ObjectTypeAdapterTest.java | 49 +++++- 6 files changed, 370 insertions(+), 80 deletions(-) create mode 100644 gson/src/test/java/com/google/gson/JsonParserParameterizedTest.java create mode 100644 gson/src/test/java/com/google/gson/ObjectTypeAdapterParameterizedTest.java diff --git a/gson/src/main/java/com/google/gson/internal/bind/ObjectTypeAdapter.java b/gson/src/main/java/com/google/gson/internal/bind/ObjectTypeAdapter.java index 2faebb08f7..c5f2ec731c 100644 --- a/gson/src/main/java/com/google/gson/internal/bind/ObjectTypeAdapter.java +++ b/gson/src/main/java/com/google/gson/internal/bind/ObjectTypeAdapter.java @@ -17,8 +17,8 @@ package com.google.gson.internal.bind; import com.google.gson.Gson; -import com.google.gson.ToNumberStrategy; import com.google.gson.ToNumberPolicy; +import com.google.gson.ToNumberStrategy; import com.google.gson.TypeAdapter; import com.google.gson.TypeAdapterFactory; import com.google.gson.internal.LinkedTreeMap; @@ -26,9 +26,10 @@ import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import com.google.gson.stream.JsonWriter; - import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.List; import java.util.Map; @@ -70,42 +71,98 @@ public static TypeAdapterFactory getFactory(ToNumberStrategy toNumberStrategy) { } } + /** + * Tries to begin reading a JSON array or JSON object, returning {@code null} if + * the next element is neither of those. + */ + private Object tryBeginNesting(JsonReader in, JsonToken peeked) throws IOException { + switch (peeked) { + case BEGIN_ARRAY: + in.beginArray(); + return new ArrayList<>(); + case BEGIN_OBJECT: + in.beginObject(); + return new LinkedTreeMap<>(); + default: + return null; + } + } + + /** Reads an {@code Object} which cannot have any nested elements */ + private Object readTerminal(JsonReader in, JsonToken peeked) throws IOException { + switch (peeked) { + case STRING: + return in.nextString(); + case NUMBER: + return toNumberStrategy.readNumber(in); + case BOOLEAN: + return in.nextBoolean(); + case NULL: + in.nextNull(); + return null; + default: + // When read(JsonReader) is called with JsonReader in invalid state + throw new IllegalStateException("Unexpected token: " + peeked); + } + } + @Override public Object read(JsonReader in) throws IOException { - JsonToken token = in.peek(); - switch (token) { - case BEGIN_ARRAY: - List list = new ArrayList<>(); - in.beginArray(); - while (in.hasNext()) { - list.add(read(in)); - } - in.endArray(); - return list; + // Either List or Map + Object current; + JsonToken peeked = in.peek(); + + current = tryBeginNesting(in, peeked); + if (current == null) { + return readTerminal(in, peeked); + } + + Deque stack = new ArrayDeque<>(); - case BEGIN_OBJECT: - Map map = new LinkedTreeMap<>(); - in.beginObject(); + while (true) { while (in.hasNext()) { - map.put(in.nextName(), read(in)); - } - in.endObject(); - return map; + String name = null; + // Name is only used for JSON object members + if (current instanceof Map) { + name = in.nextName(); + } - case STRING: - return in.nextString(); + peeked = in.peek(); + Object value = tryBeginNesting(in, peeked); + boolean isNesting = value != null; - case NUMBER: - return toNumberStrategy.readNumber(in); + if (value == null) { + value = readTerminal(in, peeked); + } - case BOOLEAN: - return in.nextBoolean(); + if (current instanceof List) { + @SuppressWarnings("unchecked") + List list = (List) current; + list.add(value); + } else { + @SuppressWarnings("unchecked") + Map map = (Map) current; + map.put(name, value); + } + + if (isNesting) { + stack.addLast(current); + current = value; + } + } - case NULL: - in.nextNull(); - return null; + // End current element + if (current instanceof List) { + in.endArray(); + } else { + in.endObject(); + } - default: - throw new IllegalStateException(); + if (stack.isEmpty()) { + return current; + } else { + // Continue with enclosing element + current = stack.removeLast(); + } } } diff --git a/gson/src/main/java/com/google/gson/internal/bind/TypeAdapters.java b/gson/src/main/java/com/google/gson/internal/bind/TypeAdapters.java index e57c282f31..9ba136379d 100644 --- a/gson/src/main/java/com/google/gson/internal/bind/TypeAdapters.java +++ b/gson/src/main/java/com/google/gson/internal/bind/TypeAdapters.java @@ -16,6 +16,22 @@ package com.google.gson.internal.bind; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonIOException; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSyntaxException; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; +import com.google.gson.annotations.SerializedName; +import com.google.gson.internal.LazilyParsedNumber; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import com.google.gson.stream.JsonWriter; import java.io.IOException; import java.lang.reflect.AccessibleObject; import java.lang.reflect.Field; @@ -27,10 +43,12 @@ import java.net.URL; import java.security.AccessController; import java.security.PrivilegedAction; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.BitSet; import java.util.Calendar; import java.util.Currency; +import java.util.Deque; import java.util.GregorianCalendar; import java.util.HashMap; import java.util.List; @@ -42,23 +60,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerArray; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonIOException; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; -import com.google.gson.JsonSyntaxException; -import com.google.gson.TypeAdapter; -import com.google.gson.TypeAdapterFactory; -import com.google.gson.annotations.SerializedName; -import com.google.gson.internal.LazilyParsedNumber; -import com.google.gson.reflect.TypeToken; -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonToken; -import com.google.gson.stream.JsonWriter; - /** * Type adapters for basic types. */ @@ -695,44 +696,99 @@ public void write(JsonWriter out, Locale value) throws IOException { public static final TypeAdapterFactory LOCALE_FACTORY = newFactory(Locale.class, LOCALE); public static final TypeAdapter JSON_ELEMENT = new TypeAdapter() { + /** + * Tries to begin reading a JSON array or JSON object, returning {@code null} if + * the next element is neither of those. + */ + private JsonElement tryBeginNesting(JsonReader in, JsonToken peeked) throws IOException { + switch (peeked) { + case BEGIN_ARRAY: + in.beginArray(); + return new JsonArray(); + case BEGIN_OBJECT: + in.beginObject(); + return new JsonObject(); + default: + return null; + } + } + + /** Reads a {@link JsonElement} which cannot have any nested elements */ + private JsonElement readTerminal(JsonReader in, JsonToken peeked) throws IOException { + switch (peeked) { + case STRING: + return new JsonPrimitive(in.nextString()); + case NUMBER: + String number = in.nextString(); + return new JsonPrimitive(new LazilyParsedNumber(number)); + case BOOLEAN: + return new JsonPrimitive(in.nextBoolean()); + case NULL: + in.nextNull(); + return JsonNull.INSTANCE; + default: + // When read(JsonReader) is called with JsonReader in invalid state + throw new IllegalStateException("Unexpected token: " + peeked); + } + } + @Override public JsonElement read(JsonReader in) throws IOException { if (in instanceof JsonTreeReader) { return ((JsonTreeReader) in).nextJsonElement(); } - switch (in.peek()) { - case STRING: - return new JsonPrimitive(in.nextString()); - case NUMBER: - String number = in.nextString(); - return new JsonPrimitive(new LazilyParsedNumber(number)); - case BOOLEAN: - return new JsonPrimitive(in.nextBoolean()); - case NULL: - in.nextNull(); - return JsonNull.INSTANCE; - case BEGIN_ARRAY: - JsonArray array = new JsonArray(); - in.beginArray(); + // Either JsonArray or JsonObject + JsonElement current; + JsonToken peeked = in.peek(); + + current = tryBeginNesting(in, peeked); + if (current == null) { + return readTerminal(in, peeked); + } + + Deque stack = new ArrayDeque<>(); + + while (true) { while (in.hasNext()) { - array.add(read(in)); + String name = null; + // Name is only used for JSON object members + if (current instanceof JsonObject) { + name = in.nextName(); + } + + peeked = in.peek(); + JsonElement value = tryBeginNesting(in, peeked); + boolean isNesting = value != null; + + if (value == null) { + value = readTerminal(in, peeked); + } + + if (current instanceof JsonArray) { + ((JsonArray) current).add(value); + } else { + ((JsonObject) current).add(name, value); + } + + if (isNesting) { + stack.addLast(current); + current = value; + } } - in.endArray(); - return array; - case BEGIN_OBJECT: - JsonObject object = new JsonObject(); - in.beginObject(); - while (in.hasNext()) { - object.add(in.nextName(), read(in)); + + // End current element + if (current instanceof JsonArray) { + in.endArray(); + } else { + in.endObject(); + } + + if (stack.isEmpty()) { + return current; + } else { + // Continue with enclosing element + current = stack.removeLast(); } - in.endObject(); - return object; - case END_DOCUMENT: - case NAME: - case END_OBJECT: - case END_ARRAY: - default: - throw new IllegalArgumentException(); } } @@ -803,7 +859,7 @@ public EnumTypeAdapter(final Class classOfT) { T constant = (T)(constantField.get(null)); String name = constant.name(); String toStringVal = constant.toString(); - + SerializedName annotation = constantField.getAnnotation(SerializedName.class); if (annotation != null) { name = annotation.value(); diff --git a/gson/src/test/java/com/google/gson/JsonParserParameterizedTest.java b/gson/src/test/java/com/google/gson/JsonParserParameterizedTest.java new file mode 100644 index 0000000000..8671fd8369 --- /dev/null +++ b/gson/src/test/java/com/google/gson/JsonParserParameterizedTest.java @@ -0,0 +1,41 @@ +package com.google.gson; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class JsonParserParameterizedTest { + @Parameters + public static Iterable data() { + return Arrays.asList( + "[]", + "{}", + "null", + "1.0", + "true", + "\"string\"", + "[true,1.0,null,{},2.0,{\"a\":[false]},[3.0,\"test\"],4.0]", + "{\"\":1.0,\"a\":true,\"b\":null,\"c\":[],\"d\":{\"a1\":2.0,\"b2\":[true,{\"a3\":3.0}]},\"e\":[{\"f\":4.0},\"test\"]}" + ); + } + + private final TypeAdapter adapter = new Gson().getAdapter(JsonElement.class); + @Parameter + public String json; + + @Test + public void testParse() throws IOException { + JsonElement deserialized = JsonParser.parseString(json); + String actualSerialized = adapter.toJson(deserialized); + + // Serialized JsonElement should be the same as original JSON + assertEquals(json, actualSerialized); + } +} diff --git a/gson/src/test/java/com/google/gson/JsonParserTest.java b/gson/src/test/java/com/google/gson/JsonParserTest.java index cc18238be4..a05aa32296 100644 --- a/gson/src/test/java/com/google/gson/JsonParserTest.java +++ b/gson/src/test/java/com/google/gson/JsonParserTest.java @@ -18,8 +18,8 @@ import java.io.CharArrayReader; import java.io.CharArrayWriter; +import java.io.IOException; import java.io.StringReader; - import junit.framework.TestCase; import com.google.gson.common.TestTypes.BagOfPrimitives; @@ -90,6 +90,54 @@ public void testParseMixedArray() { assertEquals("stringValue", array.get(2).getAsString()); } + private static String repeat(String s, int times) { + StringBuilder stringBuilder = new StringBuilder(s.length() * times); + for (int i = 0; i < times; i++) { + stringBuilder.append(s); + } + return stringBuilder.toString(); + } + + /** Deeply nested JSON arrays should not cause {@link StackOverflowError} */ + public void testParseDeeplyNestedArrays() throws IOException { + int times = 10000; + // [[[ ... ]]] + String json = repeat("[", times) + repeat("]", times); + + int actualTimes = 0; + JsonArray current = JsonParser.parseString(json).getAsJsonArray(); + while (true) { + actualTimes++; + if (current.isEmpty()) { + break; + } + assertEquals(1, current.size()); + current = current.get(0).getAsJsonArray(); + } + assertEquals(times, actualTimes); + } + + /** Deeply nested JSON objects should not cause {@link StackOverflowError} */ + public void testParseDeeplyNestedObjects() throws IOException { + int times = 10000; + // {"a":{"a": ... {"a":null} ... }} + String json = repeat("{\"a\":", times) + "null" + repeat("}", times); + + int actualTimes = 0; + JsonObject current = JsonParser.parseString(json).getAsJsonObject(); + while (true) { + assertEquals(1, current.size()); + actualTimes++; + JsonElement next = current.get("a"); + if (next.isJsonNull()) { + break; + } else { + current = next.getAsJsonObject(); + } + } + assertEquals(times, actualTimes); + } + public void testParseReader() { StringReader reader = new StringReader("{a:10,b:'c'}"); JsonElement e = JsonParser.parseReader(reader); diff --git a/gson/src/test/java/com/google/gson/ObjectTypeAdapterParameterizedTest.java b/gson/src/test/java/com/google/gson/ObjectTypeAdapterParameterizedTest.java new file mode 100644 index 0000000000..60740ce0f5 --- /dev/null +++ b/gson/src/test/java/com/google/gson/ObjectTypeAdapterParameterizedTest.java @@ -0,0 +1,41 @@ +package com.google.gson; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class ObjectTypeAdapterParameterizedTest { + @Parameters + public static Iterable data() { + return Arrays.asList( + "[]", + "{}", + "null", + "1.0", + "true", + "\"string\"", + "[true,1.0,null,{},2.0,{\"a\":[false]},[3.0,\"test\"],4.0]", + "{\"\":1.0,\"a\":true,\"b\":null,\"c\":[],\"d\":{\"a1\":2.0,\"b2\":[true,{\"a3\":3.0}]},\"e\":[{\"f\":4.0},\"test\"]}" + ); + } + + private final TypeAdapter adapter = new Gson().getAdapter(Object.class); + @Parameter + public String json; + + @Test + public void testReadWrite() throws IOException { + Object deserialized = adapter.fromJson(json); + String actualSerialized = adapter.toJson(deserialized); + + // Serialized Object should be the same as original JSON + assertEquals(json, actualSerialized); + } +} diff --git a/gson/src/test/java/com/google/gson/ObjectTypeAdapterTest.java b/gson/src/test/java/com/google/gson/ObjectTypeAdapterTest.java index d5afc1539a..534c398d50 100644 --- a/gson/src/test/java/com/google/gson/ObjectTypeAdapterTest.java +++ b/gson/src/test/java/com/google/gson/ObjectTypeAdapterTest.java @@ -16,9 +16,11 @@ package com.google.gson; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import junit.framework.TestCase; @@ -38,7 +40,7 @@ public void testSerialize() throws Exception { Object object = new RuntimeType(); assertEquals("{'a':5,'b':[1,2,null]}", adapter.toJson(object).replace("\"", "'")); } - + public void testSerializeNullValue() throws Exception { Map map = new LinkedHashMap<>(); map.put("a", null); @@ -55,6 +57,51 @@ public void testSerializeObject() throws Exception { assertEquals("{}", adapter.toJson(new Object())); } + private static String repeat(String s, int times) { + StringBuilder stringBuilder = new StringBuilder(s.length() * times); + for (int i = 0; i < times; i++) { + stringBuilder.append(s); + } + return stringBuilder.toString(); + } + + /** Deeply nested JSON arrays should not cause {@link StackOverflowError} */ + @SuppressWarnings("unchecked") + public void testDeserializeDeeplyNestedArrays() throws IOException { + int times = 10000; + // [[[ ... ]]] + String json = repeat("[", times) + repeat("]", times); + + int actualTimes = 0; + List> current = (List>) adapter.fromJson(json); + while (true) { + actualTimes++; + if (current.isEmpty()) { + break; + } + assertEquals(1, current.size()); + current = (List>) current.get(0); + } + assertEquals(times, actualTimes); + } + + /** Deeply nested JSON objects should not cause {@link StackOverflowError} */ + @SuppressWarnings("unchecked") + public void testDeserializeDeeplyNestedObjects() throws IOException { + int times = 10000; + // {"a":{"a": ... {"a":null} ... }} + String json = repeat("{\"a\":", times) + "null" + repeat("}", times); + + int actualTimes = 0; + Map> current = (Map>) adapter.fromJson(json); + while (current != null) { + assertEquals(1, current.size()); + actualTimes++; + current = (Map>) current.get("a"); + } + assertEquals(times, actualTimes); + } + @SuppressWarnings("unused") private class RuntimeType { Object a = 5;