OIDC Add remaining environments (azure, gcp), evergreen testing, API …

…naming updates (#1371)

JAVA-5353
JAVA-5395
JAVA-4834
JAVA-4932

---------

Co-authored-by: Valentin Kovalenko <valentin.male.kovalenko@gmail.com>

.evergreen/.evg.yml

@@ -12,9 +12,8 @@ stepback: true
 # Actual testing tasks are marked with `type: test`
 command_type: system
 
-# Protect ourself against rogue test case, or curl gone wild, that runs forever
-# 12 minutes is the longest we'll ever run
-exec_timeout_secs: 3600 # 12 minutes is the longest we'll ever run
+# Protect ourselves against rogue test case, or curl gone wild, that runs forever
+exec_timeout_secs: 3600
 
 # What to do when evergreen hits the timeout (`post:` tasks are run automatically)
 timeout:
@@ -968,6 +967,60 @@ tasks:
         - func: "run load-balancer"
         - func: "run load-balancer tests"
 
+    - name: "oidc-auth-test"
+      commands:
+        - command: subprocess.exec
+          type: test
+          params:
+            working_dir: "src"
+            binary: bash
+            include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+            env:
+              OIDC_ENV: "test"
+            args:
+              - .evergreen/run-mongodb-oidc-test.sh
+
+    - name: "oidc-auth-test-azure"
+      commands:
+        - command: shell.exec
+          params:
+            shell: bash
+            env:
+              JAVA_HOME: ${JAVA_HOME}
+            script: |-
+              set -o errexit
+              ${PREPARE_SHELL}
+              cd src
+              git add .
+              git commit -m "add files"
+              # uncompressed tar used to allow appending .git folder
+              export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tar
+              git archive -o $AZUREOIDC_DRIVERS_TAR_FILE HEAD
+              tar -rf $AZUREOIDC_DRIVERS_TAR_FILE .git
+              export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh"
+              bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh
+
+    - name: "oidc-auth-test-gcp"
+      commands:
+        - command: shell.exec
+          params:
+            shell: bash
+            script: |-
+              set -o errexit
+              ${PREPARE_SHELL}
+              cd src
+              git add .
+              git commit -m "add files"
+              # uncompressed tar used to allow appending .git folder
+              export GCPOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tar
+              git archive -o $GCPOIDC_DRIVERS_TAR_FILE HEAD
+              tar -rf $GCPOIDC_DRIVERS_TAR_FILE .git
+              # Define the command to run on the VM.
+              # Ensure that we source the environment file created for us, set up any other variables we need,
+              # and then run our test suite on the vm.
+              export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh"
+              bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh
+
     - name: serverless-test
       commands:
         - func: "run serverless"
@@ -2065,6 +2118,78 @@ task_groups:
     tasks:
       - test-aws-lambda-deployed
 
+  - name: testoidc_task_group
+    setup_group:
+      - func: fetch source
+      - func: prepare resources
+      - func: fix absolute paths
+      - command: ec2.assume_role
+        params:
+          role_arn: ${aws_test_secrets_role}
+      - command: subprocess.exec
+        params:
+          binary: bash
+          include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/setup.sh
+    teardown_task:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/teardown.sh
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    tasks:
+      - oidc-auth-test
+
+  - name: testazureoidc_task_group
+    setup_group:
+      - func: fetch source
+      - func: prepare resources
+      - func: fix absolute paths
+      - command: subprocess.exec
+        params:
+          binary: bash
+          env:
+            AZUREOIDC_VMNAME_PREFIX: "JAVA_DRIVER"
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/create-and-setup-vm.sh
+    teardown_task:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/delete-vm.sh
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    tasks:
+      - oidc-auth-test-azure
+
+  - name: testgcpoidc_task_group
+    setup_group:
+      - func: fetch source
+      - func: prepare resources
+      - func: fix absolute paths
+      - command: subprocess.exec
+        params:
+          binary: bash
+          env:
+            GCPOIDC_VMNAME_PREFIX: "JAVA_DRIVER"
+            GCPKMS_MACHINETYPE: "e2-medium" # comparable elapsed time to Azure; default was starved, caused timeouts
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/setup.sh
+    teardown_task:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/teardown.sh
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    tasks:
+      - oidc-auth-test-gcp
+
 buildvariants:
 
 # Test packaging and other release related routines
@@ -2216,6 +2341,27 @@ buildvariants:
   tasks:
     - name: "test_atlas_task_group_search_indexes"
 
+- name: "oidc-auth-test"
+  display_name: "OIDC Auth"
+  run_on: ubuntu2204-small
+  tasks:
+    - name: testoidc_task_group
+      batchtime: 20160 # 14 days
+
+- name: testazureoidc-variant
+  display_name: "OIDC Auth Azure"
+  run_on: ubuntu2204-small
+  tasks:
+    - name: testazureoidc_task_group
+      batchtime: 20160 # 14 days
+
+- name: testgcpoidc-variant
+  display_name: "OIDC Auth GCP"
+  run_on: ubuntu2204-small
+  tasks:
+    - name: testgcpoidc_task_group
+      batchtime: 20160 # 14 days
+
 - matrix_name: "aws-auth-test"
   matrix_spec: { ssl: "nossl", jdk: ["jdk8", "jdk17", "jdk21"], version: ["4.4", "5.0", "6.0", "7.0", "latest"], os: "ubuntu",
                  aws-credential-provider: "*" }

.evergreen/run-mongodb-oidc-test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set +x          # Disable debug trace
+set -eu
+
+echo "Running MONGODB-OIDC authentication tests"
+echo "OIDC_ENV $OIDC_ENV"
+
+if [ $OIDC_ENV == "test" ]; then
+    if [ -z "$DRIVERS_TOOLS" ]; then
+        echo "Must specify DRIVERS_TOOLS"
+        exit 1
+    fi
+    source ${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh
+    # java will not need to be installed, but we need to config
+    RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")"
+    source "${RELATIVE_DIR_PATH}/javaConfig.bash"
+elif [ $OIDC_ENV == "azure" ]; then
+    source ./env.sh
+elif [ $OIDC_ENV == "gcp" ]; then
+    source ./secrets-export.sh
+else
+    echo "Unrecognized OIDC_ENV $OIDC_ENV"
+    exit 1
+fi
+
+
+if ! which java ; then
+    echo "Installing java..."
+    sudo apt install openjdk-17-jdk -y
+    echo "Installed java."
+fi
+
+which java
+export OIDC_TESTS_ENABLED=true
+
+./gradlew -Dorg.mongodb.test.uri="$MONGODB_URI" \
+  --stacktrace --debug --info --no-build-cache driver-core:cleanTest \
+  driver-sync:test --tests OidcAuthenticationProseTests --tests UnifiedAuthTest \
+  driver-reactive-streams:test --tests OidcAuthenticationAsyncProseTests \

driver-core/src/main/com/mongodb/ConnectionString.java

@@ -38,6 +38,7 @@
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -229,17 +230,19 @@
  * </ul>
  * <p>Authentication configuration:</p>
  * <ul>
- * <li>{@code authMechanism=MONGO-CR|GSSAPI|PLAIN|MONGODB-X509}: The authentication mechanism to use if a credential was supplied.
+ * <li>{@code authMechanism=MONGO-CR|GSSAPI|PLAIN|MONGODB-X509|MONGODB-OIDC}: The authentication mechanism to use if a credential was supplied.
  * The default is unspecified, in which case the client will pick the most secure mechanism available based on the sever version.  For the
- * GSSAPI and MONGODB-X509 mechanisms, no password is accepted, only the username.
+ * GSSAPI, MONGODB-X509, and MONGODB-OIDC mechanisms, no password is accepted, only the username.
  * </li>
  * <li>{@code authSource=string}: The source of the authentication credentials.  This is typically the database that
  * the credentials have been created.  The value defaults to the database specified in the path portion of the connection string.
  * If the database is specified in neither place, the default value is "admin".  This option is only respected when using the MONGO-CR
  * mechanism (the default).
  * </li>
  * <li>{@code authMechanismProperties=PROPERTY_NAME:PROPERTY_VALUE,PROPERTY_NAME2:PROPERTY_VALUE2}: This option allows authentication
- * mechanism properties to be set on the connection string.
+ * mechanism properties to be set on the connection string. Property values must be percent-encoded individually, when
+ * separator or escape characters are used (including {@code ,} (comma), {@code =}, {@code +}, {@code &}, and {@code %}). The
+ * entire substring following the {@code =} should not itself be encoded.
  * </li>
  * <li>{@code gssapiServiceName=string}: This option only applies to the GSSAPI mechanism and is used to alter the service name.
  *   Deprecated, please use {@code authMechanismProperties=SERVICE_NAME:string} instead.
@@ -916,13 +919,16 @@ private MongoCredential createCredentials(final Map<String, List<String>> option
 
         if (credential != null && authMechanismProperties != null) {
             for (String part : authMechanismProperties.split(",")) {
-                String[] mechanismPropertyKeyValue = part.split(":");
+                String[] mechanismPropertyKeyValue = part.split(":", 2);
                 if (mechanismPropertyKeyValue.length != 2) {
                     throw new IllegalArgumentException(format("The connection string contains invalid authentication properties. "
                             + "'%s' is not a key value pair", part));
                 }
                 String key = mechanismPropertyKeyValue[0].trim().toLowerCase();
                 String value = mechanismPropertyKeyValue[1].trim();
+                if (decodeValueOfKeyValuePair(credential.getMechanism())) {
+                    value = urldecode(value);
+                }
                 if (MECHANISM_KEYS_DISALLOWED_IN_CONNECTION_STRING.contains(key)) {
                     throw new IllegalArgumentException(format("The connection string contains disallowed mechanism properties. "
                             + "'%s' must be set on the credential programmatically.", key));
@@ -938,6 +944,27 @@ private MongoCredential createCredentials(final Map<String, List<String>> option
         return credential;
     }
 
+    private static boolean decodeWholeOptionValue(final boolean isOidc, final String key) {
+        // The "whole option value" is the entire string following = in an option,
+        // including separators when the value is a list or list of key-values.
+        // This is the original parsing behaviour, but implies that users can
+        // encode separators (much like they might with URL parameters). This
+        // behaviour implies that users cannot encode "key-value" values that
+        // contain a comma, because this will (after this "whole value decoding)
+        // be parsed as a key-value separator, rather than part of a value.
+        return !(isOidc && key.equals("authmechanismproperties"));
+    }
+
+    private static boolean decodeValueOfKeyValuePair(@Nullable final String mechanismName) {
+        // Only authMechanismProperties should be individually decoded, and only
+        // when the mechanism is OIDC. These will not have been decoded.
+        return AuthenticationMechanism.MONGODB_OIDC.getMechanismName().equals(mechanismName);
+    }
+
+    private static boolean isOidc(final List<String> options) {
+        return options.contains("authMechanism=" + AuthenticationMechanism.MONGODB_OIDC.getMechanismName());
+    }
+
     private MongoCredential createMongoCredentialWithMechanism(final AuthenticationMechanism mechanism, final String userName,
                                                                @Nullable final char[] password,
                                                                @Nullable final String authSource,
@@ -1018,12 +1045,14 @@ private String getLastValue(final Map<String, List<String>> optionsMap, final St
 
     private Map<String, List<String>> parseOptions(final String optionsPart) {
         Map<String, List<String>> optionsMap = new HashMap<>();
-        if (optionsPart.length() == 0) {
+        if (optionsPart.isEmpty()) {
             return optionsMap;
         }
 
-        for (final String part : optionsPart.split("&|;")) {
-            if (part.length() == 0) {
+        List<String> options = Arrays.asList(optionsPart.split("&|;"));
+        boolean isOidc = isOidc(options);
+        for (final String part : options) {
+            if (part.isEmpty()) {
                 continue;
             }
             int idx = part.indexOf("=");
@@ -1034,7 +1063,10 @@ private Map<String, List<String>> parseOptions(final String optionsPart) {
                 if (valueList == null) {
                     valueList = new ArrayList<>(1);
                 }
-                valueList.add(urldecode(value));
+                if (decodeWholeOptionValue(isOidc, key)) {
+                    value = urldecode(value);
+                }
+                valueList.add(value);
                 optionsMap.put(key, valueList);
             } else {
                 throw new IllegalArgumentException(format("The connection string contains an invalid option '%s'. "