diff --git a/hoplite-aws/build.gradle.kts b/hoplite-aws/build.gradle.kts index e19f3ee0..a8ff0861 100644 --- a/hoplite-aws/build.gradle.kts +++ b/hoplite-aws/build.gradle.kts @@ -1,11 +1,15 @@ +plugins { + kotlin("plugin.serialization") version "1.6.21" +} + dependencies { api(project(":hoplite-core")) api(Libs.Aws.core) api(Libs.Aws.ssm) api(Libs.Aws.secrets) + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3") testApi(Libs.Kotest.testContainers) testApi(Libs.TestContainers.localstack) - testApi(Libs.Logback.classic) testApi(Libs.Slf4j.api) } diff --git a/hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessor.kt b/hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessor.kt index c692575a..32c6ed75 100644 --- a/hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessor.kt +++ b/hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessor.kt @@ -19,6 +19,8 @@ import com.sksamuel.hoplite.fp.invalid import com.sksamuel.hoplite.fp.valid import com.sksamuel.hoplite.preprocessor.TraversingPrimitivePreprocessor import com.sksamuel.hoplite.withMeta +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json class AwsSecretsManagerPreprocessor( private val createClient: () -> AWSSecretsManager = { AWSSecretsManagerClientBuilder.standard().build() } @@ -28,6 +30,7 @@ class AwsSecretsManagerPreprocessor( private val regex1 = "\\$\\{awssecret:(.+?)}".toRegex() private val regex2 = "secretsmanager://(.+?)".toRegex() private val regex3 = "awssm://(.+?)".toRegex() + private val keyRegex = "(.+)\\[(.+)]".toRegex() override fun handle(node: PrimitiveNode): ConfigResult = when (node) { is StringNode -> { @@ -35,24 +38,43 @@ class AwsSecretsManagerPreprocessor( val match = regex1.matchEntire(node.value) ?: regex2.matchEntire(node.value) ?: regex3.matchEntire(node.value) ) { null -> node.valid() - else -> fetchSecret(match.groupValues[1].trim(), node) + else -> { + val value = match.groupValues[1].trim() + val keyMatch = keyRegex.matchEntire(value) + val (key, index) = if (keyMatch == null) Pair(value, null) else + Pair(keyMatch.groupValues[1], keyMatch.groupValues[2]) + fetchSecret(key, index, node) + } } } else -> node.valid() } - private fun fetchSecret(key: String, node: StringNode): ConfigResult { + private fun fetchSecret(key: String, index: String?, node: StringNode): ConfigResult { return try { val req = GetSecretValueRequest().withSecretId(key) - val value = client.getSecretValue(req).secretString - if (value.isNullOrBlank()) + val secret = client.getSecretValue(req).secretString + if (secret.isNullOrBlank()) ConfigFailure.PreprocessorWarning("Empty secret '$key' in AWS SecretsManager").invalid() else { - val copied = node.copy(value = value) - .withMeta(CommonMetadata.Secret, true) - .withMeta(CommonMetadata.UnprocessedValue, node.value) - .withMeta(CommonMetadata.RemoteLookup, "AWS '$key'") - copied.valid() + if (index == null) { + node.copy(value = secret) + .withMeta(CommonMetadata.Secret, true) + .withMeta(CommonMetadata.UnprocessedValue, node.value) + .withMeta(CommonMetadata.RemoteLookup, "AWS '$key'") + .valid() + } else { + val map = runCatching { Json.Default.decodeFromString>(secret) }.getOrElse { emptyMap() } + val indexedValue = map[index] + if (indexedValue == null) + ConfigFailure.PreprocessorWarning("Index '$index' not present in secret '$key'. Available keys are ${map.keys.joinToString(",")}").invalid() + else + node.copy(value = indexedValue) + .withMeta(CommonMetadata.Secret, true) + .withMeta(CommonMetadata.UnprocessedValue, node.value) + .withMeta(CommonMetadata.RemoteLookup, "AWS '$key[$index]'") + .valid() + } } } catch (e: ResourceNotFoundException) { ConfigFailure.PreprocessorWarning("Could not locate resource '$key' in AWS SecretsManager").invalid() diff --git a/hoplite-aws/src/test/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessorTest.kt b/hoplite-aws/src/test/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessorTest.kt index 175bd1fa..1e7ff46d 100644 --- a/hoplite-aws/src/test/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessorTest.kt +++ b/hoplite-aws/src/test/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessorTest.kt @@ -10,6 +10,7 @@ import com.sksamuel.hoplite.Pos import com.sksamuel.hoplite.StringNode import com.sksamuel.hoplite.decoder.DotPath import com.sksamuel.hoplite.fp.Validated +import com.sksamuel.hoplite.parsers.PropsPropertySource import com.sksamuel.hoplite.traverse import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.extensions.install @@ -22,6 +23,7 @@ import io.kotest.matchers.string.shouldNotContain import io.kotest.matchers.types.shouldBeInstanceOf import org.testcontainers.containers.localstack.LocalStackContainer import org.testcontainers.utility.DockerImageName +import java.util.Properties class AwsSecretsManagerPreprocessorTest : FunSpec() { @@ -38,6 +40,7 @@ class AwsSecretsManagerPreprocessorTest : FunSpec() { .build() client.createSecret(CreateSecretRequest().withName("foo").withSecretString("secret!")) + client.createSecret(CreateSecretRequest().withName("bubble").withSecretString("""{"f": "1", "g": "2"}""")) test("placeholder should be detected and used") { ConfigLoaderBuilder.default() @@ -109,6 +112,17 @@ class AwsSecretsManagerPreprocessorTest : FunSpec() { .loadConfigOrThrow("/multiple_secrets.props") }.message.shouldContain("foo.bar").shouldContain("bar.baz") } + + test("should support index keys") { + val props = Properties() + props["a"] = "awssm://bubble[f]" + ConfigLoaderBuilder.default() + .addPreprocessor(AwsSecretsManagerPreprocessor { client }) + .addPropertySource(PropsPropertySource(props)) + .build() + .loadConfigOrThrow() + .a shouldBe "1" + } } data class ConfigHolder(val a: String) diff --git a/hoplite-aws2/build.gradle.kts b/hoplite-aws2/build.gradle.kts index aba75ce8..40a5786f 100644 --- a/hoplite-aws2/build.gradle.kts +++ b/hoplite-aws2/build.gradle.kts @@ -1,11 +1,12 @@ plugins { - kotlin("jvm") + kotlin("plugin.serialization") version "1.6.21" } dependencies { api(project(":hoplite-core")) api(Libs.Aws2.regions) api(Libs.Aws2.secretsmanager) + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3") } apply("../publish.gradle.kts") diff --git a/hoplite-aws2/src/main/kotlin/com/sksamuel/hoplite/aws2/AwsSecretsManagerPreprocessor.kt b/hoplite-aws2/src/main/kotlin/com/sksamuel/hoplite/aws2/AwsSecretsManagerPreprocessor.kt index 2f0db0d4..d23890ac 100644 --- a/hoplite-aws2/src/main/kotlin/com/sksamuel/hoplite/aws2/AwsSecretsManagerPreprocessor.kt +++ b/hoplite-aws2/src/main/kotlin/com/sksamuel/hoplite/aws2/AwsSecretsManagerPreprocessor.kt @@ -10,6 +10,8 @@ import com.sksamuel.hoplite.fp.invalid import com.sksamuel.hoplite.fp.valid import com.sksamuel.hoplite.preprocessor.TraversingPrimitivePreprocessor import com.sksamuel.hoplite.withMeta +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient import software.amazon.awssdk.services.secretsmanager.model.DecryptionFailureException import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest @@ -26,6 +28,7 @@ class AwsSecretsManagerPreprocessor( private val regex1 = "\\$\\{awssecret:(.+?)}".toRegex() private val regex2 = "secretsmanager://(.+?)".toRegex() private val regex3 = "awssm://(.+?)".toRegex() + private val keyRegex = "(.+)\\[(.+)]".toRegex() override fun handle(node: PrimitiveNode): ConfigResult = when (node) { is StringNode -> { @@ -33,25 +36,45 @@ class AwsSecretsManagerPreprocessor( val match = regex1.matchEntire(node.value) ?: regex2.matchEntire(node.value) ?: regex3.matchEntire(node.value) ) { null -> node.valid() - else -> fetchSecret(match.groupValues[1].trim(), node) + else -> { + val value = match.groupValues[1].trim() + val keyMatch = keyRegex.matchEntire(value) + val (key, index) = if (keyMatch == null) Pair(value, null) else + Pair(keyMatch.groupValues[1], keyMatch.groupValues[2]) + fetchSecret(key, index, node) + } } } else -> node.valid() } - private fun fetchSecret(key: String, node: StringNode): ConfigResult { + private fun fetchSecret(key: String, index: String?, node: StringNode): ConfigResult { return try { val valueRequest = GetSecretValueRequest.builder().secretId(key).build() - val valueResponse = client.getSecretValue(valueRequest) - val value = valueResponse.secretString() - if (value.isNullOrBlank()) + val value = client.getSecretValue(valueRequest) + val secret = value.secretString() + if (secret.isNullOrBlank()) ConfigFailure.PreprocessorWarning("Empty secret '$key' in AWS SecretsManager").invalid() - else - node.copy(value = value) - .withMeta(CommonMetadata.Secret, true) - .withMeta(CommonMetadata.UnprocessedValue, node.value) - .withMeta(CommonMetadata.RemoteLookup, "AWS '$key'") - .valid() + else { + if (index == null) { + node.copy(value = secret) + .withMeta(CommonMetadata.Secret, true) + .withMeta(CommonMetadata.UnprocessedValue, node.value) + .withMeta(CommonMetadata.RemoteLookup, "AWS '$key'") + .valid() + } else { + val map = runCatching { Json.Default.decodeFromString>(secret) }.getOrElse { emptyMap() } + val indexedValue = map[index] + if (indexedValue == null) + ConfigFailure.PreprocessorWarning("Index '$index' not present in secret '$key'. Available keys are ${map.keys.joinToString(",")}").invalid() + else + node.copy(value = indexedValue) + .withMeta(CommonMetadata.Secret, true) + .withMeta(CommonMetadata.UnprocessedValue, node.value) + .withMeta(CommonMetadata.RemoteLookup, "AWS '$key[$index]'") + .valid() + } + } } catch (e: ResourceNotFoundException) { ConfigFailure.PreprocessorWarning("Could not locate resource '$key' in AWS SecretsManager").invalid() } catch (e: DecryptionFailureException) { diff --git a/settings.gradle.kts b/settings.gradle.kts index 6bcedcc1..7c9ff15c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -17,6 +17,9 @@ plugins { refreshVersions { enableBuildSrcLibs() + rejectVersionIf { + candidate.stabilityLevel != de.fayard.refreshVersions.core.StabilityLevel.Stable + } } include("hoplite-core") diff --git a/versions.properties b/versions.properties index 585890db..35c8615f 100644 --- a/versions.properties +++ b/versions.properties @@ -6,48 +6,26 @@ #### #### suppress inspection "SpellCheckingInspection" for whole file #### suppress inspection "UnusedProperty" for whole file +#### +#### NOTE: Some versions are filtered by the rejectVersionsIf predicate. See the settings.gradle.kts file. version.org.testcontainers..vault=1.17.3 version.org.springframework.vault..spring-vault-core=2.3.2 -version.software.amazon.awssdk..regions=2.17.252 +version.software.amazon.awssdk..regions=2.17.256 version.org.yaml..snakeyaml=1.30 -## # available=1.31-SNAPSHOT version.org.tomlj..tomlj=1.0.0 version.org.testcontainers..localstack=1.17.3 version.org.slf4j..slf4j-api=1.7.36 -## # available=1.8.0-alpha0 -## # available=1.8.0-alpha1 -## # available=1.8.0-alpha2 -## # available=1.8.0-beta0 -## # available=1.8.0-beta1 -## # available=1.8.0-beta2 -## # available=1.8.0-beta4 -## # available=2.0.0-alpha0 -## # available=2.0.0-alpha1 -## # available=2.0.0-alpha2 -## # available=2.0.0-alpha3 -## # available=2.0.0-alpha4 -## # available=2.0.0-alpha5 -## # available=2.0.0-alpha6 -## # available=2.0.0-alpha7 -## # available=2.0.0-beta0 -## # available=2.0.0-beta1 - -version.org.postgresql..postgresql=42.4.1 -## # available=42.4.2-SNAPSHOT + +version.org.postgresql..postgresql=42.4.2 version.org.apache.hadoop..hadoop-common=2.10.2 -## # available=3.0.0-alpha1 -## # available=3.0.0-alpha2 -## # available=3.0.0-alpha3 -## # available=3.0.0-alpha4 -## # available=3.0.0-beta1 ## # available=3.0.0 ## # available=3.0.1 ## # available=3.0.2 @@ -75,38 +53,14 @@ version.kotlinx.datetime=0.3.2 version.kotlinx.coroutines=1.6.4 version.kotlin=1.6.21 -## # available=1.7.0-Beta -## # available=1.7.0-RC -## # available=1.7.0-RC2 ## # available=1.7.0 ## # available=1.7.10 -## # available=1.7.20-Beta -## # available=1.7.255-SNAPSHOT -## # available=1.8.255-SNAPSHOT version.kotest.extensions.testcontainers=1.3.4 -## # available=1.4.0.52-SNAPSHOT -## # available=1.4.0.53-SNAPSHOT -## # available=1.4.0.54-SNAPSHOT -## # available=1.4.0.55-SNAPSHOT version.kotest=5.4.2 -## # available=5.5.0.1059-SNAPSHOT -## # available=5.5.0.1060-SNAPSHOT -## # available=5.5.0.1061-SNAPSHOT -## # available=5.5.0.1062-SNAPSHOT -## # available=5.5.0.1063-SNAPSHOT -## # available=5.5.0.1064-SNAPSHOT -## # available=5.5.0.1065-SNAPSHOT -## # available=5.5.0.1066-SNAPSHOT -## # available=5.5.0.1067-SNAPSHOT -## # available=5.5.0.1070-SNAPSHOT -## # available=5.5.0.1071-SNAPSHOT -## # available=5.5.0.1072-SNAPSHOT -## # available=5.5.0.1073-SNAPSHOT version.io.vavr..vavr-kotlin=0.10.2 -## # available=1.0.0-SNAPSHOT version.io.micrometer..micrometer-registry-statsd=1.9.3 @@ -115,55 +69,8 @@ version.io.micrometer..micrometer-registry-prometheus=1.9.3 version.io.micrometer..micrometer-registry-datadog=1.9.3 version.io.arrow-kt..arrow-core=1.1.2 -## # available=1.1.3-alpha.1 -## # available=1.1.3-alpha.2 -## # available=1.1.3-alpha.3.0+2022-05-16T16-21-58-758705Z -## # available=1.1.3-alpha.4.0+2022-05-17T09-11-10-723810Z -## # available=1.1.3-alpha.5.0+2022-05-17T11-44-11-714740Z -## # available=1.1.3-alpha.6 -## # available=1.1.3-alpha.7 -## # available=1.1.3-alpha.8 -## # available=1.1.3-alpha.9 -## # available=1.1.3-alpha.10 -## # available=1.1.3-alpha.11 -## # available=1.1.3-alpha.12 -## # available=1.1.3-alpha.13 -## # available=1.1.3-alpha.14 -## # available=1.1.3-alpha.15 -## # available=1.1.3-alpha.16 -## # available=1.1.3-alpha.17 -## # available=1.1.3-alpha.18 -## # available=1.1.3-alpha.19 -## # available=1.1.3-alpha.20 -## # available=1.1.3-alpha.21 -## # available=1.1.3-alpha.22 -## # available=1.1.3-alpha.23 -## # available=1.1.3-alpha.24 -## # available=1.1.3-alpha.25 -## # available=1.1.3-alpha.26 -## # available=1.1.3-alpha.27 -## # available=1.1.3-alpha.28 -## # available=1.1.3-alpha.29 -## # available=1.1.3-alpha.30 -## # available=1.1.3-alpha.31 -## # available=1.1.3-alpha.32 -## # available=1.1.3-alpha.33 -## # available=1.1.3-alpha.34 -## # available=1.1.3-alpha.35 -## # available=1.1.3-alpha.36 -## # available=1.1.3-alpha.37 -## # available=1.1.3-alpha.38 -## # available=1.1.3-alpha.39 -## # available=1.1.3-alpha.40 -## # available=1.1.3-alpha.41 -## # available=1.1.3-alpha.42 -## # available=1.1.3-alpha.43 -## # available=1.1.3-alpha.44 -## # available=1.1.3-alpha.45 -## # available=1.1.3-alpha.46 version.com.zaxxer..HikariCP=5.0.1 -## # available=5.0.2-SNAPSHOT version.com.typesafe..config=1.4.2 @@ -172,46 +79,21 @@ version.com.pszymczyk.consul..embedded-consul=2.2.1 version.com.orbitz.consul..consul-client=1.5.3 version.com.fasterxml.jackson.core..jackson-databind=2.13.3 -## # available=2.13.4-SNAPSHOT -## # available=2.14.0-SNAPSHOT -## # available=3.0.0-SNAPSHOT version.com.fasterxml.jackson.core..jackson-core=2.13.3 -## # available=2.13.4-SNAPSHOT -## # available=2.14.0-SNAPSHOT -## # available=3.0.0-SNAPSHOT version.com.cronutils..cron-utils=9.2.0 -version.com.charleskorn.kaml..kaml=0.46.0 -## # available=0.47.0-SNAPSHOT +version.com.charleskorn.kaml..kaml=0.47.0 -version.com.amazonaws..aws-java-sdk-ssm=1.12.281 +version.com.amazonaws..aws-java-sdk-ssm=1.12.285 -version.com.amazonaws..aws-java-sdk-secretsmanager=1.12.281 +version.com.amazonaws..aws-java-sdk-secretsmanager=1.12.285 -version.com.amazonaws..aws-java-sdk-core=1.12.281 +version.com.amazonaws..aws-java-sdk-core=1.12.285 version.ch.qos.logback..logback-classic=1.2.11 -## # available=1.3.0-alpha0 -## # available=1.3.0-alpha1 -## # available=1.3.0-alpha2 -## # available=1.3.0-alpha3 -## # available=1.3.0-alpha4 -## # available=1.3.0-alpha5 -## # available=1.3.0-alpha6 -## # available=1.3.0-alpha7 -## # available=1.3.0-alpha8 -## # available=1.3.0-alpha9 -## # available=1.3.0-alpha10 -## # available=1.3.0-alpha11 -## # available=1.3.0-alpha12 -## # available=1.3.0-alpha13 -## # available=1.3.0-alpha14 -## # available=1.3.0-alpha15 -## # available=1.3.0-alpha16 -## # available=1.3.0-beta0 plugin.io.kotest=0.3.9 -version.software.amazon.awssdk..secretsmanager=2.17.252 +version.software.amazon.awssdk..secretsmanager=2.17.256