Correctly handle buffer boundaries while decoding escape sequences from json stream

formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/AbstractJsonLexer.kt

@@ -318,6 +318,9 @@ internal abstract class AbstractJsonLexer {
             if (char == STRING_ESC) {
                 usedAppend = true
                 currentPosition = appendEscape(lastPosition, currentPosition)
+                currentPosition = definitelyNotEof(currentPosition)
+                if (currentPosition == -1)
+                    fail("EOF", currentPosition)
                 lastPosition = currentPosition
             } else if (++currentPosition >= source.length) {
                 usedAppend = true
@@ -435,7 +438,13 @@ internal abstract class AbstractJsonLexer {
     }
 
     private fun appendHex(source: CharSequence, startPos: Int): Int {
-        if (startPos + 4 >= source.length) fail("Unexpected EOF during unicode escape")
+        if (startPos + 4 >= source.length) {
+            currentPosition = startPos
+            ensureHaveChars()
+            if (currentPosition + 4 >= source.length)
+                fail("Unexpected EOF during unicode escape")
+            return appendHex(source, currentPosition)
+        }
         escapedString.append(
             ((fromHexChar(source, startPos) shl 12) +
                     (fromHexChar(source, startPos + 1) shl 8) +

formats/json/commonTest/src/kotlinx/serialization/json/JsonUnicodeTest.kt

@@ -1,8 +1,11 @@
+/*
+ * Copyright 2017-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
 package kotlinx.serialization.json
 
 import kotlinx.serialization.*
 import kotlinx.serialization.builtins.*
-import kotlinx.serialization.json.internal.*
 import kotlinx.serialization.test.*
 import kotlin.random.*
 import kotlin.test.*
@@ -59,7 +62,7 @@ class JsonUnicodeTest : JsonTestBase() {
     @Test
     fun testRandomEscapeSequences() = noJs { // Too slow on JS
         repeat(10_000) {
-            val s = generateRandomString()
+            val s = generateRandomUnicodeString(Random.nextInt(1, 2047))
             try {
                 assertSerializedAndRestored(s, String.serializer())
             } catch (e: Throwable) {
@@ -68,21 +71,4 @@ class JsonUnicodeTest : JsonTestBase() {
             }
         }
     }
-
-    private fun generateRandomString(): String {
-        val size = Random.nextInt(1, 2047)
-        return buildString(size) {
-            repeat(size) {
-                val pickEscape = Random.nextBoolean()
-                if (pickEscape) {
-                    // Definitely escape symbol
-                    // null can be appended as well, completely okay
-                    append(ESCAPE_STRINGS.random())
-                } else {
-                    // Any symbol, including escaping one
-                    append(Char(Random.nextInt(Char.MIN_VALUE.code..Char.MAX_VALUE.code)))
-                }
-            }
-        }
-    }
 }

formats/json/commonTest/src/kotlinx/serialization/test/TestHelpers.kt

@@ -1,10 +1,13 @@
 /*
- * Copyright 2017-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2017-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
  */
 package kotlinx.serialization.test
 
 import kotlinx.serialization.*
 import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.json.internal.ESCAPE_STRINGS
+import kotlin.random.Random
+import kotlin.random.nextInt
 import kotlin.test.*
 
 fun SerialDescriptor.assertDescriptorEqualsTo(other: SerialDescriptor) {
@@ -40,3 +43,18 @@ inline fun assertFailsWithMissingField(block: () -> Unit) {
     val e = assertFailsWith<SerializationException>(block = block)
     assertTrue(e.message?.contains("but it was missing") ?: false)
 }
+
+fun generateRandomUnicodeString(size: Int): String {
+    return buildString(size) {
+        repeat(size) {
+            val pickEscape = Random.nextBoolean()
+            if (pickEscape) {
+                // Definitely escape symbol
+                append(ESCAPE_STRINGS.random().takeIf { it != null } ?: 'N')
+            } else {
+                // Any symbol, including escaping one
+                append(Char(Random.nextInt(Char.MIN_VALUE.code..Char.MAX_VALUE.code)).takeIf { it.isDefined() && !it.isSurrogate()} ?: 'U')
+            }
+        }
+    }
+}

formats/json/jvmTest/src/kotlinx/serialization/features/JsonJvmStreamsTest.kt

@@ -4,13 +4,15 @@
 
 package kotlinx.serialization.features
 
-import kotlinx.serialization.*
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.StringData
 import kotlinx.serialization.builtins.serializer
-import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.*
 import kotlinx.serialization.json.internal.BATCH_SIZE
-import kotlinx.serialization.test.decodeViaStream
-import kotlinx.serialization.test.encodeViaStream
+import kotlinx.serialization.test.*
 import org.junit.Test
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
 
@@ -65,4 +67,22 @@ class JsonJvmStreamsTest {
             Json.decodeViaStream(String.serializer(), "\"")
         }
     }
+
+    @Test
+    fun testRandomEscapeSequences()  {
+        repeat(10_000) {
+            val s = generateRandomUnicodeString(strLen)
+            try {
+                val serializer = String.serializer()
+                val b = ByteArrayOutputStream()
+                Json.encodeToStream(serializer, s, b)
+                val restored = Json.decodeFromStream(serializer, ByteArrayInputStream(b.toByteArray()))
+                assertEquals(s, restored)
+            } catch (e: Throwable) {
+                // Not assertion error to preserve cause
+                throw IllegalStateException("Unexpectedly failed test, cause string: $s", e)
+            }
+        }
+    }
+
 }