From 77aa167449f17638d24a75fa279161888796a7b3 Mon Sep 17 00:00:00 2001 From: tobiaslieber Date: Mon, 20 Dec 2021 13:34:59 +0100 Subject: [PATCH] HOCON: parse strings into integers and booleans if possible (#1795) HOCON suggests that parsers should apply certain automatic conversions, especially when reading integers/numbers or booleans from strings (https://github.com/lightbend/config/blob/main/HOCON.md#automatic-type-conversions). This is an attempt to resolve the most pressing issues. Fixes #1439 This PR changes parsing so that it now relies on the parsing capabilities for the Config class, and not on the obtained ConfigValues. This PR does not claim to cover all automatic conversions mentioned but is focused on parsing numbers and booleans from strings only. Co-authored-by: saibot --- .../kotlinx/serialization/hocon/Hocon.kt | 63 +++++++++++-------- .../serialization/hocon/HoconValuesTest.kt | 56 +++++++++++++++++ 2 files changed, 94 insertions(+), 25 deletions(-) diff --git a/formats/hocon/src/main/kotlin/kotlinx/serialization/hocon/Hocon.kt b/formats/hocon/src/main/kotlin/kotlinx/serialization/hocon/Hocon.kt index e65c3d70f..d8b0f4cd9 100644 --- a/formats/hocon/src/main/kotlin/kotlinx/serialization/hocon/Hocon.kt +++ b/formats/hocon/src/main/kotlin/kotlinx/serialization/hocon/Hocon.kt @@ -47,18 +47,28 @@ public sealed class Hocon( override val serializersModule: SerializersModule get() = this@Hocon.serializersModule - abstract fun getTaggedConfigValue(tag: T): ConfigValue - - private inline fun validateAndCast(tag: T, wrappedType: ConfigValueType): E { - val cfValue = getTaggedConfigValue(tag) - if (cfValue.valueType() != wrappedType) throw SerializationException("${cfValue.origin().description()} required to be a $wrappedType") - return cfValue.unwrapped() as E + abstract fun getValueFromTaggedConfig(tag: T, valueResolver: (Config, String) -> E): E + + private inline fun validateAndCast(tag: T): E { + return try { + when (E::class) { + Number::class -> getValueFromTaggedConfig(tag) { config, path -> config.getNumber(path) } as E + Boolean::class -> getValueFromTaggedConfig(tag) { config, path -> config.getBoolean(path) } as E + String::class -> getValueFromTaggedConfig(tag) { config, path -> config.getString(path) } as E + else -> getValueFromTaggedConfig(tag) { config, path -> config.getAnyRef(path) } as E + } + } catch (e: ConfigException) { + val configOrigin = e.origin() + val requiredType = E::class.simpleName + throw SerializationException("${configOrigin.description()} required to be of type $requiredType") + } } - private fun getTaggedNumber(tag: T) = validateAndCast(tag, ConfigValueType.NUMBER) + private fun getTaggedNumber(tag: T) = validateAndCast(tag) - override fun decodeTaggedString(tag: T) = validateAndCast(tag, ConfigValueType.STRING) + override fun decodeTaggedString(tag: T) = validateAndCast(tag) + override fun decodeTaggedBoolean(tag: T) = validateAndCast(tag) override fun decodeTaggedByte(tag: T): Byte = getTaggedNumber(tag).toByte() override fun decodeTaggedShort(tag: T): Short = getTaggedNumber(tag).toShort() override fun decodeTaggedInt(tag: T): Int = getTaggedNumber(tag).toInt() @@ -67,17 +77,17 @@ public sealed class Hocon( override fun decodeTaggedDouble(tag: T): Double = getTaggedNumber(tag).toDouble() override fun decodeTaggedChar(tag: T): Char { - val s = validateAndCast(tag, ConfigValueType.STRING) + val s = validateAndCast(tag) if (s.length != 1) throw SerializationException("String \"$s\" is not convertible to Char") return s[0] } - override fun decodeTaggedValue(tag: T): Any = getTaggedConfigValue(tag).unwrapped() + override fun decodeTaggedValue(tag: T): Any = getValueFromTaggedConfig(tag) { c, s -> c.getAnyRef(s) } - override fun decodeTaggedNotNullMark(tag: T) = getTaggedConfigValue(tag).valueType() != ConfigValueType.NULL + override fun decodeTaggedNotNullMark(tag: T) = getValueFromTaggedConfig(tag) { c, s -> !c.getIsNull(s) } override fun decodeTaggedEnum(tag: T, enumDescriptor: SerialDescriptor): Int { - val s = validateAndCast(tag, ConfigValueType.STRING) + val s = validateAndCast(tag) return enumDescriptor.getElementIndexOrThrow(s) } } @@ -107,14 +117,6 @@ public sealed class Hocon( else originalName.replace(NAMING_CONVENTION_REGEX) { "-${it.value.lowercase()}" } } - override fun getTaggedConfigValue(tag: String): ConfigValue { - return conf.getValue(tag) - } - - override fun decodeTaggedNotNullMark(tag: String): Boolean { - return !conf.getIsNull(tag) - } - override fun decodeNotNullMark(): Boolean { // Tag might be null for top-level deserialization val currentTag = currentTagOrNull ?: return !conf.isEmpty @@ -159,6 +161,10 @@ public sealed class Hocon( else -> this } } + + override fun getValueFromTaggedConfig(tag: String, valueResolver: (Config, String) -> E): E { + return valueResolver(conf, tag) + } } private inner class ListConfigReader(private val list: ConfigList) : ConfigConverter() { @@ -179,7 +185,11 @@ public sealed class Hocon( return if (ind > list.size - 1) DECODE_DONE else ind } - override fun getTaggedConfigValue(tag: Int): ConfigValue = list[tag] + override fun getValueFromTaggedConfig(tag: Int, valueResolver: (Config, String) -> E): E { + val tagString = tag.toString() + val configValue = valueResolver(list[tag].atKey(tagString), tagString) + return configValue + } } private inner class MapConfigReader(map: ConfigObject) : ConfigConverter() { @@ -210,13 +220,16 @@ public sealed class Hocon( return if (ind >= indexSize) DECODE_DONE else ind } - override fun getTaggedConfigValue(tag: Int): ConfigValue { + override fun getValueFromTaggedConfig(tag: Int, valueResolver: (Config, String) -> E): E { val idx = tag / 2 - return if (tag % 2 == 0) { // entry as string - ConfigValueFactory.fromAnyRef(keys[idx]) + val tagString = tag.toString() + val configValue = if (tag % 2 == 0) { // entry as string + ConfigValueFactory.fromAnyRef(keys[idx]).atKey(tagString) } else { - values[idx] + val configValue = values[idx] + configValue.atKey(tagString) } + return valueResolver(configValue, tagString) } } diff --git a/formats/hocon/src/test/kotlin/kotlinx/serialization/hocon/HoconValuesTest.kt b/formats/hocon/src/test/kotlin/kotlinx/serialization/hocon/HoconValuesTest.kt index f7fd1d6b6..1b050e44e 100644 --- a/formats/hocon/src/test/kotlin/kotlinx/serialization/hocon/HoconValuesTest.kt +++ b/formats/hocon/src/test/kotlin/kotlinx/serialization/hocon/HoconValuesTest.kt @@ -4,6 +4,7 @@ package kotlinx.serialization.hocon +import kotlin.test.* import kotlinx.serialization.* import kotlinx.serialization.builtins.* import org.junit.* @@ -31,6 +32,12 @@ class HoconValuesTest { @Serializable data class WithNullableList(val i1: List, val i2: List?, val i3: List?) + @Serializable + data class WithList(val i1: List) + + @Serializable + data class WithMap(val m: Map) + @Test fun `deserialize numbers`() { val conf = "b=42, s=1337, i=100500, l = 4294967294, f=0.0, d=-0.123" @@ -45,6 +52,20 @@ class HoconValuesTest { } } + @Test + fun `deserialize numbers from strings`() { + val conf = """b="42", s="1337", i="100500", l = "4294967294", f="0.0", d="-0.123" """ + val nums = deserializeConfig(conf, NumbersConfig.serializer()) + with(nums) { + assertEquals(42.toByte(), b) + assertEquals(1337.toShort(), s) + assertEquals(100500, i) + assertEquals(4294967294L, l) + assertEquals(0.0f, f) + assertEquals(-0.123, d, 1e-9) + } + } + @Test fun `deserialize string types`() { val obj = deserializeConfig("c=f, s=foo", StringConfig.serializer()) @@ -59,6 +80,20 @@ class HoconValuesTest { assertEquals(true, obj.b) } + @Test + fun `unparseable data fails with exception`() { + val e = assertFailsWith { + deserializeConfig("e = A, b=not-a-boolean", OtherConfig.serializer()) + } + } + + @Test + fun `deserialize other types from strings`() { + val obj = deserializeConfig("""e = "A", b="true" """, OtherConfig.serializer()) + assertEquals(Choice.A, obj.e) + assertEquals(true, obj.b) + } + @Test fun `deserialize default values`() { val obj = deserializeConfig("", WithDefault.serializer()) @@ -103,4 +138,25 @@ class HoconValuesTest { assertEquals(listOf(null, WithNullable(10, "bar")), i3) } } + + @Test + fun `deserialize list of integer string values`() { + val configString = """i1 = [ "1","3" ]""" + val obj = deserializeConfig(configString, WithList.serializer()) + assertEquals(listOf(1, 3), obj.i1) + } + + @Test + fun `deserialize map with integers`() { + val configString = """m = { 2: 1, 4: 3 }""" + val obj = deserializeConfig(configString, WithMap.serializer()) + assertEquals(mapOf(2 to 1, 4 to 3), obj.m) + } + + @Test + fun `deserialize map with integers as strings`() { + val configString = """m = { "2": "1", "4":"3" }""" + val obj = deserializeConfig(configString, WithMap.serializer()) + assertEquals(mapOf(2 to 1, 4 to 3), obj.m) + } }