New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
IO Streams #1901
IO Streams #1901
Changes from all commits
922c709
62c6c0a
4030241
ec82d13
a39b014
58d780e
f6fece3
4cd82f2
c336eec
17dd813
50bd0b6
15fc853
c17ae93
b1ed21a
d78e98a
9b14d52
cd6a69b
0670aff
bf38cdb
9330a33
7eb0479
aa268b9
5afaf42
f1f2f02
844d147
95f9bbb
51a6faa
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
public final class kotlinx/serialization/json/okio/OkioStreamsKt { | ||
public static final fun decodeFromSource (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/DeserializationStrategy;Lokio/Source;)Ljava/lang/Object; | ||
public static final fun decodeSourceToSequence (Lkotlinx/serialization/json/Json;Lokio/Source;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/DecodeSequenceMode;)Lkotlin/sequences/Sequence; | ||
public static synthetic fun decodeSourceToSequence$default (Lkotlinx/serialization/json/Json;Lokio/Source;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/DecodeSequenceMode;ILjava/lang/Object;)Lkotlin/sequences/Sequence; | ||
public static final fun encodeToSink (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;Lokio/Sink;)V | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
/* | ||
* Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. | ||
*/ | ||
import Java9Modularity.configureJava9ModuleInfo | ||
|
||
plugins { | ||
kotlin("multiplatform") | ||
kotlin("plugin.serialization") | ||
} | ||
|
||
apply(from = rootProject.file("gradle/native-targets.gradle")) | ||
apply(from = rootProject.file("gradle/configure-source-sets.gradle")) | ||
|
||
kotlin { | ||
sourceSets { | ||
val commonMain by getting { | ||
dependencies { | ||
api(project(":kotlinx-serialization-core")) | ||
api(project(":kotlinx-serialization-json")) | ||
implementation("com.squareup.okio:okio:${property("okio_version")}") | ||
} | ||
} | ||
val commonTest by getting { | ||
dependencies { | ||
implementation("com.squareup.okio:okio:${property("okio_version")}") | ||
} | ||
} | ||
} | ||
} | ||
|
||
project.configureJava9ModuleInfo() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,139 @@ | ||
/* | ||
* Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. | ||
*/ | ||
|
||
package kotlinx.serialization.json.okio | ||
|
||
import kotlinx.serialization.* | ||
import kotlinx.serialization.json.DecodeSequenceMode | ||
import kotlinx.serialization.json.Json | ||
import kotlinx.serialization.json.internal.* | ||
import kotlinx.serialization.json.okio.internal.JsonToOkioStreamWriter | ||
import kotlinx.serialization.json.internal.decodeToSequenceByReader | ||
import kotlinx.serialization.json.okio.internal.OkioSerialReader | ||
import okio.* | ||
|
||
/** | ||
* Serializes the [value] with [serializer] into a [target] using JSON format and UTF-8 encoding. | ||
* | ||
* If [target] is not a [BufferedSink] then called [Sink.buffer] for it to create buffered wrapper. | ||
* | ||
* @throws [SerializationException] if the given value cannot be serialized to JSON. | ||
* @throws [okio.IOException] If an I/O error occurs and sink can't be written to. | ||
*/ | ||
@ExperimentalSerializationApi | ||
public fun <T> Json.encodeToSink( | ||
serializer: SerializationStrategy<T>, | ||
value: T, | ||
target: Sink | ||
) { | ||
val buffered = if (target is BufferedSink) target else target.buffer() | ||
val writer = JsonToOkioStreamWriter(buffered) | ||
try { | ||
encodeByWriter(writer, serializer, value) | ||
} finally { | ||
writer.release() | ||
} | ||
} | ||
|
||
/** | ||
* Serializes given [value] to a [target] using UTF-8 encoding and serializer retrieved from the reified type parameter. | ||
* | ||
* If [target] is not a [BufferedSink] then called [Sink.buffer] for it to create buffered wrapper. | ||
* | ||
* @throws [SerializationException] if the given value cannot be serialized to JSON. | ||
shanshin marked this conversation as resolved.
Show resolved
Hide resolved
|
||
* @throws [okio.IOException] If an I/O error occurs and sink can't be written to. | ||
*/ | ||
@ExperimentalSerializationApi | ||
public inline fun <reified T> Json.encodeToSink( | ||
value: T, | ||
target: Sink | ||
): Unit = encodeToSink(serializersModule.serializer(), value, target) | ||
|
||
|
||
/** | ||
* Deserializes JSON from [source] using UTF-8 encoding to a value of type [T] using [deserializer]. | ||
* | ||
* If [source] is not a [BufferedSource] then called [Source.buffer] for it to create buffered wrapper. | ||
* | ||
* Note that this functions expects that exactly one object would be present in the source | ||
* and throws an exception if there are any dangling bytes after an object. | ||
* | ||
* @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T]. | ||
* @throws [okio.IOException] If an I/O error occurs and source can't be read from. | ||
*/ | ||
@ExperimentalSerializationApi | ||
public fun <T> Json.decodeFromSource( | ||
deserializer: DeserializationStrategy<T>, | ||
source: Source | ||
): T { | ||
val buffered = if (source is BufferedSource) source else source.buffer() | ||
return decodeByReader(deserializer, OkioSerialReader(buffered)) | ||
} | ||
|
||
/** | ||
* Deserializes the contents of given [source] to the value of type [T] using UTF-8 encoding and | ||
* deserializer retrieved from the reified type parameter. | ||
* | ||
* If [source] is not a [BufferedSource] then called [Source.buffer] for it to create buffered wrapper. | ||
* | ||
* Note that this functions expects that exactly one object would be present in the stream | ||
* and throws an exception if there are any dangling bytes after an object. | ||
* | ||
* @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T]. | ||
* @throws [okio.IOException] If an I/O error occurs and source can't be read from. | ||
*/ | ||
@ExperimentalSerializationApi | ||
public inline fun <reified T> Json.decodeFromSource(source: Source): T = | ||
decodeFromSource(serializersModule.serializer(), source) | ||
|
||
|
||
/** | ||
* Transforms the given [source] into lazily deserialized sequence of elements of type [T] using UTF-8 encoding and [deserializer]. | ||
* Unlike [decodeFromSource], [source] is allowed to have more than one element, separated as [format] declares. | ||
* | ||
* If [source] is not a [BufferedSource] then called [Source.buffer] for it to create buffered wrapper. | ||
* | ||
* Elements must all be of type [T]. | ||
* Elements are parsed lazily when resulting [Sequence] is evaluated. | ||
* Resulting sequence is tied to the stream and can be evaluated only once. | ||
* | ||
* **Resource caution:** this method neither closes the [source] when the parsing is finished nor provides a method to close it manually. | ||
* It is a caller responsibility to hold a reference to a stream and close it. Moreover, because stream is parsed lazily, | ||
* closing it before returned sequence is evaluated completely will result in [Exception] from decoder. | ||
* | ||
* @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T]. | ||
* @throws [okio.IOException] If an I/O error occurs and source can't be read from. | ||
*/ | ||
@ExperimentalSerializationApi | ||
public fun <T> Json.decodeSourceToSequence( | ||
source: Source, | ||
deserializer: DeserializationStrategy<T>, | ||
format: DecodeSequenceMode = DecodeSequenceMode.AUTO_DETECT | ||
): Sequence<T> { | ||
val buffered = if (source is BufferedSource) source else source.buffer() | ||
return decodeToSequenceByReader(OkioSerialReader(buffered), deserializer, format) | ||
} | ||
|
||
/** | ||
* Transforms the given [source] into lazily deserialized sequence of elements of type [T] using UTF-8 encoding and deserializer retrieved from the reified type parameter. | ||
* Unlike [decodeFromSource], [source] is allowed to have more than one element, separated as [format] declares. | ||
* | ||
* If [source] is not a [BufferedSource] then called [Source.buffer] for it to create buffered wrapper. | ||
* | ||
* Elements must all be of type [T]. | ||
* Elements are parsed lazily when resulting [Sequence] is evaluated. | ||
* Resulting sequence is tied to the stream and constrained to be evaluated only once. | ||
* | ||
* **Resource caution:** this method does not close [source] when the parsing is finished neither provides method to close it manually. | ||
* It is a caller responsibility to hold a reference to a stream and close it. Moreover, because stream is parsed lazily, | ||
* closing it before returned sequence is evaluated fully would result in [Exception] from decoder. | ||
* | ||
* @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T]. | ||
* @throws [okio.IOException] If an I/O error occurs and source can't be read from. | ||
*/ | ||
@ExperimentalSerializationApi | ||
public inline fun <reified T> Json.decodeSourceToSequence( | ||
source: Source, | ||
format: DecodeSequenceMode = DecodeSequenceMode.AUTO_DETECT | ||
): Sequence<T> = decodeSourceToSequence(source, serializersModule.serializer(), format) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
/* | ||
* Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. | ||
*/ | ||
|
||
@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") | ||
|
||
package kotlinx.serialization.json.okio.internal | ||
|
||
import kotlinx.serialization.json.internal.ESCAPE_STRINGS | ||
import kotlinx.serialization.json.internal.JsonWriter | ||
import kotlinx.serialization.json.internal.SerialReader | ||
import okio.* | ||
|
||
internal class JsonToOkioStreamWriter(private val target: BufferedSink) : JsonWriter { | ||
override fun writeLong(value: Long) { | ||
write(value.toString()) | ||
} | ||
|
||
override fun writeChar(char: Char) { | ||
target.writeUtf8CodePoint(char.code) | ||
} | ||
|
||
override fun write(text: String) { | ||
target.writeUtf8(text) | ||
} | ||
|
||
override fun writeQuoted(text: String) { | ||
target.writeUtf8CodePoint('"'.code) | ||
var lastPos = 0 | ||
for (i in text.indices) { | ||
val c = text[i].code | ||
if (c < ESCAPE_STRINGS.size && ESCAPE_STRINGS[c] != null) { | ||
target.writeUtf8(text, lastPos, i) // flush prev | ||
target.writeUtf8(ESCAPE_STRINGS[c]!!) | ||
lastPos = i + 1 | ||
} | ||
} | ||
|
||
if (lastPos != 0) target.writeUtf8(text, lastPos, text.length) | ||
else target.writeUtf8(text) | ||
target.writeUtf8CodePoint('"'.code) | ||
} | ||
|
||
override fun release() { | ||
target.flush() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Tempted to say that this should be a call to More here: https://jakewharton.com/forcing-bytes-downward-in-okio/ |
||
} | ||
} | ||
|
||
internal class OkioSerialReader(private val source: BufferedSource): SerialReader { | ||
override fun read(buffer: CharArray, bufferOffset: Int, count: Int): Int { | ||
var i = 0 | ||
while (i < count && !source.exhausted()) { | ||
buffer[i] = source.readUtf8CodePoint().toChar() | ||
i++ | ||
} | ||
return if (i > 0) i else -1 | ||
} | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
kotlin.mpp.enableCompatibilityMetadataVariant=false |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sorry I'm late to the party! Why not making the "buffer" requirement in the API itself?
The current API fails if someone expects to be able to resume reading the stream after decoding an element.
See #1789 (comment) for a related discussion.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For encoding this isn't a problem. You are always free to call
buffer()
and thenemit()
as implementation details.The problem you are referring to, however, is for decoding. Decoding absolutely should take a
BufferedSource
and not aSource
. This is what Moshi and Wire do as well.https://github.com/square/moshi/blob/moshi-parent-1.13.0/moshi/src/main/java/com/squareup/moshi/JsonAdapter.java#L57
https://github.com/square/wire/blob/4.4.0/wire-library/wire-runtime/src/commonMain/kotlin/com/squareup/wire/ProtoAdapter.kt#L145
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Right! Sorry for the confusion, I never realised how asymmetric this was 👍 . Maybe it's worth removing the "BufferSink" mention from the kdoc then? Since it's an implementation detail, the caller doesn't really need to know about it?
And in all cases, make the
BufferedSource
mandatory fordecodeFromSource
:There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Agree on both points!
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks for your suggestions! Looks like we probably need to change existing functions working with Java streams applying the same logic.
However, this may lead to some asymmetry in the API. What do you think would be better — change both
encodeToSink/decodeFromSource
so they accept buffered versions or leaveencodeToSink
intact?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No strong opinion from me. Both Moshi and Wire use
BufferedSink
so I'd lean towardsBufferedSink
here as well for consistency/aesthetics reasons.Since the sink is ending up being buffered in all cases, it's pretty easy to let the caller call
.buffer()
if required. Plus it might even save anemit()
when writing multiple Json to the sameBufferedSink
(So tiny performance gain maybe...). But if there's a use case for passing a rawSink
, that works with me too.