Support map lookup in AWS preprocessors

hoplite-aws/build.gradle.kts

@@ -1,11 +1,15 @@
+plugins {
+   kotlin("plugin.serialization") version "1.6.21"
+}
+
 dependencies {
    api(project(":hoplite-core"))
    api(Libs.Aws.core)
    api(Libs.Aws.ssm)
    api(Libs.Aws.secrets)
+   implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
    testApi(Libs.Kotest.testContainers)
    testApi(Libs.TestContainers.localstack)
-
    testApi(Libs.Logback.classic)
    testApi(Libs.Slf4j.api)
 }

hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessor.kt

@@ -19,6 +19,8 @@ import com.sksamuel.hoplite.fp.invalid
 import com.sksamuel.hoplite.fp.valid
 import com.sksamuel.hoplite.preprocessor.TraversingPrimitivePreprocessor
 import com.sksamuel.hoplite.withMeta
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
 
 class AwsSecretsManagerPreprocessor(
   private val createClient: () -> AWSSecretsManager = { AWSSecretsManagerClientBuilder.standard().build() }
@@ -28,31 +30,51 @@ class AwsSecretsManagerPreprocessor(
   private val regex1 = "\\$\\{awssecret:(.+?)}".toRegex()
   private val regex2 = "secretsmanager://(.+?)".toRegex()
   private val regex3 = "awssm://(.+?)".toRegex()
+  private val keyRegex = "(.+)\\[(.+)]".toRegex()
 
   override fun handle(node: PrimitiveNode): ConfigResult<Node> = when (node) {
     is StringNode -> {
       when (
         val match = regex1.matchEntire(node.value) ?: regex2.matchEntire(node.value) ?: regex3.matchEntire(node.value)
       ) {
         null -> node.valid()
-        else -> fetchSecret(match.groupValues[1].trim(), node)
+        else -> {
+          val value = match.groupValues[1].trim()
+          val keyMatch = keyRegex.matchEntire(value)
+          val (key, index) = if (keyMatch == null) Pair(value, null) else
+            Pair(keyMatch.groupValues[1], keyMatch.groupValues[2])
+          fetchSecret(key, index, node)
+        }
       }
     }
     else -> node.valid()
   }
 
-  private fun fetchSecret(key: String, node: StringNode): ConfigResult<Node> {
+  private fun fetchSecret(key: String, index: String?, node: StringNode): ConfigResult<Node> {
     return try {
       val req = GetSecretValueRequest().withSecretId(key)
-      val value = client.getSecretValue(req).secretString
-      if (value.isNullOrBlank())
+      val secret = client.getSecretValue(req).secretString
+      if (secret.isNullOrBlank())
         ConfigFailure.PreprocessorWarning("Empty secret '$key' in AWS SecretsManager").invalid()
       else {
-        val copied = node.copy(value = value)
-          .withMeta(CommonMetadata.Secret, true)
-          .withMeta(CommonMetadata.UnprocessedValue, node.value)
-          .withMeta(CommonMetadata.RemoteLookup, "AWS '$key'")
-        copied.valid()
+        if (index == null) {
+          node.copy(value = secret)
+            .withMeta(CommonMetadata.Secret, true)
+            .withMeta(CommonMetadata.UnprocessedValue, node.value)
+            .withMeta(CommonMetadata.RemoteLookup, "AWS '$key'")
+            .valid()
+        } else {
+          val map = runCatching { Json.Default.decodeFromString<Map<String, String>>(secret) }.getOrElse { emptyMap() }
+          val indexedValue = map[index]
+          if (indexedValue == null)
+            ConfigFailure.PreprocessorWarning("Index '$index' not present in secret '$key'. Available keys are ${map.keys.joinToString(",")}").invalid()
+          else
+            node.copy(value = indexedValue)
+              .withMeta(CommonMetadata.Secret, true)
+              .withMeta(CommonMetadata.UnprocessedValue, node.value)
+              .withMeta(CommonMetadata.RemoteLookup, "AWS '$key[$index]'")
+              .valid()
+        }
       }
     } catch (e: ResourceNotFoundException) {
       ConfigFailure.PreprocessorWarning("Could not locate resource '$key' in AWS SecretsManager").invalid()

hoplite-aws/src/test/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPreprocessorTest.kt

@@ -10,6 +10,7 @@ import com.sksamuel.hoplite.Pos
 import com.sksamuel.hoplite.StringNode
 import com.sksamuel.hoplite.decoder.DotPath
 import com.sksamuel.hoplite.fp.Validated
+import com.sksamuel.hoplite.parsers.PropsPropertySource
 import com.sksamuel.hoplite.traverse
 import io.kotest.assertions.throwables.shouldThrow
 import io.kotest.core.extensions.install
@@ -22,6 +23,7 @@ import io.kotest.matchers.string.shouldNotContain
 import io.kotest.matchers.types.shouldBeInstanceOf
 import org.testcontainers.containers.localstack.LocalStackContainer
 import org.testcontainers.utility.DockerImageName
+import java.util.Properties
 
 class AwsSecretsManagerPreprocessorTest : FunSpec() {
 
@@ -38,6 +40,7 @@ class AwsSecretsManagerPreprocessorTest : FunSpec() {
       .build()
 
     client.createSecret(CreateSecretRequest().withName("foo").withSecretString("secret!"))
+    client.createSecret(CreateSecretRequest().withName("bubble").withSecretString("""{"f": "1", "g": "2"}"""))
 
     test("placeholder should be detected and used") {
       ConfigLoaderBuilder.default()
@@ -109,6 +112,17 @@ class AwsSecretsManagerPreprocessorTest : FunSpec() {
           .loadConfigOrThrow<ConfigHolder>("/multiple_secrets.props")
       }.message.shouldContain("foo.bar").shouldContain("bar.baz")
     }
+
+    test("should support index keys") {
+      val props = Properties()
+      props["a"] = "awssm://bubble[f]"
+      ConfigLoaderBuilder.default()
+        .addPreprocessor(AwsSecretsManagerPreprocessor { client })
+        .addPropertySource(PropsPropertySource(props))
+        .build()
+        .loadConfigOrThrow<ConfigHolder>()
+        .a shouldBe "1"
+    }
   }
 
   data class ConfigHolder(val a: String)

hoplite-aws2/build.gradle.kts

@@ -1,11 +1,12 @@
 plugins {
-   kotlin("jvm")
+   kotlin("plugin.serialization") version "1.6.21"
 }
 
 dependencies {
    api(project(":hoplite-core"))
    api(Libs.Aws2.regions)
    api(Libs.Aws2.secretsmanager)
+   implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
 }
 
 apply("../publish.gradle.kts")

hoplite-aws2/src/main/kotlin/com/sksamuel/hoplite/aws2/AwsSecretsManagerPreprocessor.kt

@@ -10,6 +10,8 @@ import com.sksamuel.hoplite.fp.invalid
 import com.sksamuel.hoplite.fp.valid
 import com.sksamuel.hoplite.preprocessor.TraversingPrimitivePreprocessor
 import com.sksamuel.hoplite.withMeta
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
 import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient
 import software.amazon.awssdk.services.secretsmanager.model.DecryptionFailureException
 import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest
@@ -26,32 +28,53 @@ class AwsSecretsManagerPreprocessor(
   private val regex1 = "\\$\\{awssecret:(.+?)}".toRegex()
   private val regex2 = "secretsmanager://(.+?)".toRegex()
   private val regex3 = "awssm://(.+?)".toRegex()
+  private val keyRegex = "(.+)\\[(.+)]".toRegex()
 
   override fun handle(node: PrimitiveNode): ConfigResult<Node> = when (node) {
     is StringNode -> {
       when (
         val match = regex1.matchEntire(node.value) ?: regex2.matchEntire(node.value) ?: regex3.matchEntire(node.value)
       ) {
         null -> node.valid()
-        else -> fetchSecret(match.groupValues[1].trim(), node)
+        else -> {
+          val value = match.groupValues[1].trim()
+          val keyMatch = keyRegex.matchEntire(value)
+          val (key, index) = if (keyMatch == null) Pair(value, null) else
+            Pair(keyMatch.groupValues[1], keyMatch.groupValues[2])
+          fetchSecret(key, index, node)
+        }
       }
     }
     else -> node.valid()
   }
 
-  private fun fetchSecret(key: String, node: StringNode): ConfigResult<Node> {
+  private fun fetchSecret(key: String, index: String?, node: StringNode): ConfigResult<Node> {
     return try {
       val valueRequest = GetSecretValueRequest.builder().secretId(key).build()
-      val valueResponse = client.getSecretValue(valueRequest)
-      val value = valueResponse.secretString()
-      if (value.isNullOrBlank())
+      val value = client.getSecretValue(valueRequest)
+      val secret = value.secretString()
+      if (secret.isNullOrBlank())
         ConfigFailure.PreprocessorWarning("Empty secret '$key' in AWS SecretsManager").invalid()
-      else
-        node.copy(value = value)
-          .withMeta(CommonMetadata.Secret, true)
-          .withMeta(CommonMetadata.UnprocessedValue, node.value)
-          .withMeta(CommonMetadata.RemoteLookup, "AWS '$key'")
-          .valid()
+      else {
+        if (index == null) {
+          node.copy(value = secret)
+            .withMeta(CommonMetadata.Secret, true)
+            .withMeta(CommonMetadata.UnprocessedValue, node.value)
+            .withMeta(CommonMetadata.RemoteLookup, "AWS '$key'")
+            .valid()
+        } else {
+          val map = runCatching { Json.Default.decodeFromString<Map<String, String>>(secret) }.getOrElse { emptyMap() }
+          val indexedValue = map[index]
+          if (indexedValue == null)
+            ConfigFailure.PreprocessorWarning("Index '$index' not present in secret '$key'. Available keys are ${map.keys.joinToString(",")}").invalid()
+          else
+            node.copy(value = indexedValue)
+              .withMeta(CommonMetadata.Secret, true)
+              .withMeta(CommonMetadata.UnprocessedValue, node.value)
+              .withMeta(CommonMetadata.RemoteLookup, "AWS '$key[$index]'")
+              .valid()
+        }
+      }
     } catch (e: ResourceNotFoundException) {
       ConfigFailure.PreprocessorWarning("Could not locate resource '$key' in AWS SecretsManager").invalid()
     } catch (e: DecryptionFailureException) {

settings.gradle.kts

@@ -17,6 +17,9 @@ plugins {
 
 refreshVersions {
    enableBuildSrcLibs()
+   rejectVersionIf {
+      candidate.stabilityLevel != de.fayard.refreshVersions.core.StabilityLevel.Stable
+   }
 }
 
 include("hoplite-core")