update for libmongocrypt 1.5.2 (#1037)

Resolve GODRIVER-2511: * Add CSFLE prose test 16. * Update tests to use libmongocrypt 1.5.2. Resolve GODRIVER-2513: * Return error if libmongocrypt < 1.5.2 is detected in RewrapManyDataKey. * Retract Go driver release v1.10.0. Resolve GODRIVER-2509: * Resync RewrapManyDataKey specification tests to mongodb/specifications@10b4a41. * Resolve GODRIVER-2512: Resolve GODRIVER-2512: - Resync fle2-InsertFind-Unindexed.json
mongodb · Aug 2, 2022 · 82a6eeb · 82a6eeb
1 parent 5073f14
commit 82a6eeb
Show file tree

Hide file tree

Showing 13 changed files with 183 additions and 15 deletions.
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -94,9 +94,10 @@ functions:
           go version
           go env
 
-          LIBMONGOCRYPT_TAG="1.5.0"
+          LIBMONGOCRYPT_BRANCH="r1.5"
+          LIBMONGOCRYPT_TAG="1.5.2"
           # LIBMONGOCRYPT_COMMIT is the commit on libmongocrypt for the tag LIBMONGOCRYPT_TAG.
-          LIBMONGOCRYPT_COMMIT="c3be59f9b0d756caa4c22c254e0704084cf6bca4"
+          LIBMONGOCRYPT_COMMIT="8f8675fa11922f00a4516a7f8a60621aa1ca1550"
           # Install libmongocrypt based on OS.
           if [ "Windows_NT" = "$OS" ]; then
              mkdir -p c:/libmongocrypt/include
@@ -105,7 +106,7 @@ functions:
              mkdir libmongocrypt-all
              cd libmongocrypt-all
              # The following URL is published from the upload-all task in the libmongocrypt Evergreen project.
-             curl https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/$LIBMONGOCRYPT_COMMIT/libmongocrypt-all.tar.gz -o libmongocrypt-all.tar.gz
+             curl https://mciuploads.s3.amazonaws.com/libmongocrypt/all/$LIBMONGOCRYPT_BRANCH/$LIBMONGOCRYPT_COMMIT/libmongocrypt-all.tar.gz -o libmongocrypt-all.tar.gz
              tar -xf libmongocrypt-all.tar.gz
              cd ..
              cp libmongocrypt-all/windows-test/bin/mongocrypt.dll c:/libmongocrypt/bin
@@ -120,7 +121,7 @@ functions:
              mkdir libmongocrypt-all
              cd libmongocrypt-all
              # The following URL is published from the upload-all task in the libmongocrypt Evergreen project.
-             curl https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/$LIBMONGOCRYPT_COMMIT/libmongocrypt-all.tar.gz -o libmongocrypt-all.tar.gz
+             curl https://mciuploads.s3.amazonaws.com/libmongocrypt/all/$LIBMONGOCRYPT_BRANCH/$LIBMONGOCRYPT_COMMIT/libmongocrypt-all.tar.gz -o libmongocrypt-all.tar.gz
              tar -xf libmongocrypt-all.tar.gz
              cd ..
              mv libmongocrypt-all/macos/include ./install/libmongocrypt

diff --git a/data/client-side-encryption/legacy/fle2-InsertFind-Unindexed.json b/data/client-side-encryption/legacy/fle2-InsertFind-Unindexed.json
@@ -241,7 +241,7 @@
             }
           },
           "result": {
-            "errorContains": "Cannot query"
+            "errorContains": "encrypt"
           }
         }
       ]

diff --git a/data/client-side-encryption/legacy/fle2-InsertFind-Unindexed.yml b/data/client-side-encryption/legacy/fle2-InsertFind-Unindexed.yml
@@ -80,4 +80,8 @@ tests:
         arguments:
           filter: { encryptedUnindexed: "value123" }
         result:
-          errorContains: "Cannot query"
+          # Expected error message changed in https://github.com/10gen/mongo-enterprise-modules/commit/212b584d4f7a44bed41c826a180a4aff00923d7a#diff-5f12b55e8d5c52c2f62853ec595dc2c1e2e5cb4fdbf7a32739a8e3acb3c6f818
+          # Before the message was "cannot query non-indexed fields with the randomized encryption algorithm"
+          # After: "can only execute encrypted equality queries with an encrypted equality index"
+          # Use a small common substring.
+          errorContains: "encrypt"
diff --git a/data/client-side-encryption/unified/createDataKey-kms_providers-invalid.json b/data/client-side-encryption/unified/createDataKey-kms_providers-invalid.json
@@ -1,5 +1,5 @@
 {
-  "description": "createDataKey-provider-invalid",
+  "description": "createDataKey-kms_providers-invalid",
   "schemaVersion": "1.8",
   "runOnRequirements": [
     {

diff --git a/data/client-side-encryption/unified/createDataKey-kms_providers-invalid.yml b/data/client-side-encryption/unified/createDataKey-kms_providers-invalid.yml
@@ -1,4 +1,4 @@
-description: createDataKey-provider-invalid
+description: createDataKey-kms_providers-invalid
 
 schemaVersion: "1.8"
 

diff --git a/data/client-side-encryption/unified/rewrapManyDataKey.json b/data/client-side-encryption/unified/rewrapManyDataKey.json
@@ -1,5 +1,5 @@
 {
-  "description": "rewrapManyDataKey-kms_providers",
+  "description": "rewrapManyDataKey",
   "schemaVersion": "1.8",
   "runOnRequirements": [
     {
@@ -128,7 +128,7 @@
           ],
           "keyMaterial": {
             "$binary": {
-              "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEGkNTybTc7Eyif0f+qqE0lAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDB2j78AeuIQxcRh8cQIBEIB7vj9buHEaT7XHFIsKBJiyzZRmNnjvqMK5LSdzonKdx97jlqauvPvTDXSsdQDcspUs5oLrGmAXpbFResscxmbwZoKgUtWiuIOpeAcYuszCiMKt15s1WIMLDXUhYtfCmhRhekvgHnRAaK4HJMlGE+lKJXYI84E0b86Cd/g+",
+              "base64": "pr01l7qDygUkFE/0peFwpnNlv3iIy8zrQK38Q9i12UCN2jwZHDmfyx8wokiIKMb9kAleeY+vnt3Cf1MKu9kcDmI+KxbNDd+V3ytAAGzOVLDJr77CiWjF9f8ntkXRHrAY9WwnVDANYkDwXlyU0Y2GQFTiW65jiQhUtYLYH63Tk48SsJuQvnWw1Q+PzY8ga+QeVec8wbcThwtm+r2IHsCFnc72Gv73qq7weISw+O4mN08z3wOp5FOS2ZM3MK7tBGmPdBcktW7F8ODGsOQ1FU53OrWUnyX2aTi2ftFFFMWVHqQo7EYuBZHru8RRODNKMyQk0BFfKovAeTAVRv9WH9QU7g==",
               "subType": "00"
             }
           },
@@ -196,7 +196,7 @@
           ],
           "keyMaterial": {
             "$binary": {
-              "base64": "VoI9J8HusQ3u2gT9i8Awgg/6W4/igvLwRzn3SRDGx0Dl/1ayDMubphOw0ONPVKfuvS6HL3e4gAoCJ/uEz2KLFTVsEqYCpMhfAhgXxm8Ena8vDcOkCzFX+euvN/N2ES3wpzAD18b3qIH0MbBwKJP82d5GQ4pVfGnPW8Ujp9aO1qC/s0EqNqYyzJ1SyzhV9lAjHHGIENYJx+bBrekg2EeZBA==",
+              "base64": "CklVctHzke4mcytd0TxGqvepkdkQN8NUF4+jV7aZQITAKdz6WjdDpq3lMt9nSzWGG2vAEfvRb3mFEVjV57qqGqxjq2751gmiMRHXz0btStbIK3mQ5xbY9kdye4tsixlCryEwQONr96gwlwKKI9Nubl9/8+uRF6tgYjje7Q7OjauEf1SrJwKcoQ3WwnjZmEqAug0kImCpJ/irhdqPzivRiA==",
               "subType": "00"
             }
           },

diff --git a/data/client-side-encryption/unified/rewrapManyDataKey.yml b/data/client-side-encryption/unified/rewrapManyDataKey.yml
@@ -2,7 +2,7 @@
 # commands sort the resulting documents in ascending order by the single-element
 # keyAltNames array to ensure alphabetic order by original KMS provider as
 # defined in initialData.
-description: rewrapManyDataKey-kms_providers
+description: rewrapManyDataKey
 
 schemaVersion: "1.8"
 
@@ -50,7 +50,7 @@ initialData:
           region: us-east-1
       - _id: &azure_key_id { $binary: { base64: YXp1cmVhenVyZWF6dXJlYQ==, subType: "04" } }
         keyAltNames: ["azure_key"]
-        keyMaterial: { $binary: { base64: AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEGkNTybTc7Eyif0f+qqE0lAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDB2j78AeuIQxcRh8cQIBEIB7vj9buHEaT7XHFIsKBJiyzZRmNnjvqMK5LSdzonKdx97jlqauvPvTDXSsdQDcspUs5oLrGmAXpbFResscxmbwZoKgUtWiuIOpeAcYuszCiMKt15s1WIMLDXUhYtfCmhRhekvgHnRAaK4HJMlGE+lKJXYI84E0b86Cd/g+, subType: "00" } }
+        keyMaterial: { $binary: { base64: pr01l7qDygUkFE/0peFwpnNlv3iIy8zrQK38Q9i12UCN2jwZHDmfyx8wokiIKMb9kAleeY+vnt3Cf1MKu9kcDmI+KxbNDd+V3ytAAGzOVLDJr77CiWjF9f8ntkXRHrAY9WwnVDANYkDwXlyU0Y2GQFTiW65jiQhUtYLYH63Tk48SsJuQvnWw1Q+PzY8ga+QeVec8wbcThwtm+r2IHsCFnc72Gv73qq7weISw+O4mN08z3wOp5FOS2ZM3MK7tBGmPdBcktW7F8ODGsOQ1FU53OrWUnyX2aTi2ftFFFMWVHqQo7EYuBZHru8RRODNKMyQk0BFfKovAeTAVRv9WH9QU7g==, subType: "00" } }
         creationDate: { $date: { $numberLong: "1641024000000" } }
         updateDate: { $date: { $numberLong: "1641024000000" } }
         status: 1
@@ -72,7 +72,7 @@ initialData:
           keyName: key-name-csfle
       - _id: &kmip_key_id { $binary: { base64: a21pcGttaXBrbWlwa21pcA==, subType: "04" } }
         keyAltNames: ["kmip_key"]
-        keyMaterial: { $binary: { base64: VoI9J8HusQ3u2gT9i8Awgg/6W4/igvLwRzn3SRDGx0Dl/1ayDMubphOw0ONPVKfuvS6HL3e4gAoCJ/uEz2KLFTVsEqYCpMhfAhgXxm8Ena8vDcOkCzFX+euvN/N2ES3wpzAD18b3qIH0MbBwKJP82d5GQ4pVfGnPW8Ujp9aO1qC/s0EqNqYyzJ1SyzhV9lAjHHGIENYJx+bBrekg2EeZBA==, subType: "00" } }
+        keyMaterial: { $binary: { base64: CklVctHzke4mcytd0TxGqvepkdkQN8NUF4+jV7aZQITAKdz6WjdDpq3lMt9nSzWGG2vAEfvRb3mFEVjV57qqGqxjq2751gmiMRHXz0btStbIK3mQ5xbY9kdye4tsixlCryEwQONr96gwlwKKI9Nubl9/8+uRF6tgYjje7Q7OjauEf1SrJwKcoQ3WwnjZmEqAug0kImCpJ/irhdqPzivRiA==, subType: "00" } }
         creationDate: { $date: { $numberLong: "1641024000000" } }
         updateDate: { $date: { $numberLong: "1641024000000" } }
         status: 1

diff --git a/go.mod b/go.mod
@@ -3,6 +3,7 @@ module go.mongodb.org/mongo-driver
 go 1.10
 
 retract (
+	v1.10.0 // Contains a possible data corruption bug in RewrapManyDataKey when using libmongocrypt versions less than 1.5.2.
 	[v1.7.0, v1.7.1] // Contains data race bug in background connection establishment.
 	[v1.6.0, v1.6.1] // Contains data race bug in background connection establishment.
 )

diff --git a/mongo/client_encryption.go b/mongo/client_encryption.go
@@ -249,9 +249,17 @@ func setRewrapManyDataKeyWriteModels(rewrappedDocuments []bsoncore.Document, wri
 // RewrapManyDataKey decrypts and encrypts all matching data keys with a possibly new masterKey value. For all
 // matching documents, this method will overwrite the "masterKey", "updateDate", and "keyMaterial". On error, some
 // matching data keys may have been rewrapped.
+// libmongocrypt 1.5.2 is required. An error is returned if the detected version of libmongocrypt is less than 1.5.2.
 func (ce *ClientEncryption) RewrapManyDataKey(ctx context.Context, filter interface{},
 	opts ...*options.RewrapManyDataKeyOptions) (*RewrapManyDataKeyResult, error) {
 
+	// libmongocrypt versions 1.5.0 and 1.5.1 have a severe bug in RewrapManyDataKey.
+	// Check if the version string starts with 1.5.0 or 1.5.1. This accounts for pre-release versions, like 1.5.0-rc0.
+	libmongocryptVersion := mongocrypt.Version()
+	if strings.HasPrefix(libmongocryptVersion, "1.5.0") || strings.HasPrefix(libmongocryptVersion, "1.5.1") {
+		return nil, fmt.Errorf("RewrapManyDataKey requires libmongocrypt 1.5.2 or newer. Detected version: %v", libmongocryptVersion)
+	}
+
 	rmdko := options.MergeRewrapManyDataKeyOptions(opts...)
 	if ctx == nil {
 		ctx = context.Background()

diff --git a/mongo/doc.go b/mongo/doc.go
@@ -107,10 +107,17 @@
 //
 // The libmongocrypt C library is required when using client-side encryption. Specific versions of libmongocrypt
 // are required for different versions of the Go Driver:
+//
 // - Go Driver v1.2.0 requires libmongocrypt v1.0.0 or higher
+//
 // - Go Driver v1.5.0 requires libmongocrypt v1.1.0 or higher
+//
 // - Go Driver v1.8.0 requires libmongocrypt v1.3.0 or higher
-// - Go Driver v1.10.0 requires libmongocrypt v1.5.0 or higher
+//
+// - Go Driver v1.10.0 requires libmongocrypt v1.5.0 or higher.
+// There is a severe bug when calling RewrapManyDataKey with libmongocrypt versions less than 1.5.2.
+// This bug may result in data corruption.
+// Please use libmongocrypt 1.5.2 or higher when calling RewrapManyDataKey.
 //
 // To install libmongocrypt, follow the instructions for your
 // operating system:

diff --git a/mongo/integration/client_side_encryption_prose_test.go b/mongo/integration/client_side_encryption_prose_test.go
@@ -1855,6 +1855,140 @@ func TestClientSideEncryptionProse(t *testing.T) {
 		})
 
 	})
+
+	mt.RunOpts("16. Rewrap", runOpts, func(mt *mtest.T) {
+		mt.Run("Case 1: Rewrap with separate ClientEncryption", func(mt *mtest.T) {
+			dataKeyMap := map[string]bson.M{
+				"aws": {
+					"region": "us-east-1",
+					"key":    "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
+				},
+				"azure": {
+					"keyVaultEndpoint": "key-vault-csfle.vault.azure.net",
+					"keyName":          "key-name-csfle",
+				},
+				"gcp": {
+					"projectId": "devprod-drivers",
+					"location":  "global",
+					"keyRing":   "key-ring-csfle",
+					"keyName":   "key-name-csfle",
+				},
+				"kmip": {},
+			}
+
+			tlsConfig := make(map[string]*tls.Config)
+			if tlsCAFileKMIP != "" && tlsClientCertificateKeyFileKMIP != "" {
+				tlsOpts := map[string]interface{}{
+					"tlsCertificateKeyFile": tlsClientCertificateKeyFileKMIP,
+					"tlsCAFile":             tlsCAFileKMIP,
+				}
+				kmipConfig, err := options.BuildTLSConfig(tlsOpts)
+				assert.Nil(mt, err, "BuildTLSConfig error: %v", err)
+				tlsConfig["kmip"] = kmipConfig
+			}
+
+			kmsProviders := []string{"local", "aws", "gcp", "azure", "kmip"}
+			for _, srcProvider := range kmsProviders {
+				for _, dstProvider := range kmsProviders {
+					mt.Run(fmt.Sprintf("%s to %s", srcProvider, dstProvider), func(mt *mtest.T) {
+						var err error
+						// Drop the collection ``keyvault.datakeys``.
+						{
+							err = mt.Client.Database("keyvault").Collection("datakeys").Drop(context.Background())
+							assert.Nil(mt, err, "error on Drop: %v", err)
+						}
+
+						// Create a ``ClientEncryption`` object named ``clientEncryption1``.
+						var clientEncryption1 *mongo.ClientEncryption
+						{
+							var keyVaultClient *mongo.Client
+							{
+								co := options.Client().ApplyURI(mtest.ClusterURI())
+								keyVaultClient, err = mongo.Connect(context.Background(), co)
+								defer keyVaultClient.Disconnect(context.Background())
+								testutil.AddTestServerAPIVersion(co)
+								assert.Nil(mt, err, "error on Connect: %v", err)
+							}
+							ceOpts := options.ClientEncryption().
+								SetKeyVaultNamespace("keyvault.datakeys").
+								SetKmsProviders(fullKmsProvidersMap).
+								SetTLSConfig(tlsConfig)
+							clientEncryption1, err = mongo.NewClientEncryption(keyVaultClient, ceOpts)
+							assert.Nil(mt, err, "error in NewClientEncryption: %v", err)
+							defer clientEncryption1.Close(context.Background())
+						}
+
+						// Call ``clientEncryption1.createDataKey``.
+						var keyID primitive.Binary
+						{
+							dkOpts := options.DataKey()
+							if val, ok := dataKeyMap[srcProvider]; ok {
+								dkOpts.SetMasterKey(val)
+							}
+							keyID, err = clientEncryption1.CreateDataKey(context.Background(), srcProvider, dkOpts)
+							assert.Nil(mt, err, "error in CreateDataKey: %v", err)
+						}
+
+						// Call ``clientEncryption1.encrypt`` with the value "test".
+						var ciphertext primitive.Binary
+						{
+							t, value, err := bson.MarshalValue("test")
+							assert.Nil(mt, err, "error in MarshalValue: %v", err)
+							plaintext := bson.RawValue{Type: t, Value: value}
+							eOpts := options.Encrypt().SetKeyID(keyID).SetAlgorithm("AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic")
+							ciphertext, err = clientEncryption1.Encrypt(context.Background(), plaintext, eOpts)
+							assert.Nil(mt, err, "error in Encrypt: %v", err)
+						}
+
+						// Create a ``ClientEncryption`` object named ``clientEncryption2``.
+						var clientEncryption2 *mongo.ClientEncryption
+						{
+							var keyVaultClient *mongo.Client
+							{
+								co := options.Client().ApplyURI(mtest.ClusterURI())
+								keyVaultClient, err = mongo.Connect(context.Background(), co)
+								defer keyVaultClient.Disconnect(context.Background())
+								testutil.AddTestServerAPIVersion(co)
+								assert.Nil(mt, err, "error on Connect: %v", err)
+							}
+							ceOpts := options.ClientEncryption().
+								SetKeyVaultNamespace("keyvault.datakeys").
+								SetKmsProviders(fullKmsProvidersMap).
+								SetTLSConfig(tlsConfig)
+							clientEncryption2, err = mongo.NewClientEncryption(keyVaultClient, ceOpts)
+							assert.Nil(mt, err, "error in NewClientEncryption: %v", err)
+							defer clientEncryption2.Close(context.Background())
+						}
+
+						// Call ``clientEncryption2.rewrapManyDataKey`` with an empty ``filter``.
+						{
+							rwOpts := options.RewrapManyDataKey().SetProvider(dstProvider)
+							if val, ok := dataKeyMap[dstProvider]; ok {
+								rwOpts.SetMasterKey(val)
+							}
+							res, err := clientEncryption2.RewrapManyDataKey(context.Background(), bson.D{}, rwOpts)
+							assert.Nil(mt, err, "error in RewrapManyDataKey: %v", err)
+							assert.Equal(mt, res.BulkWriteResult.ModifiedCount, int64(1), "expected ModifiedCount of 1, got %v", res.BulkWriteResult.ModifiedCount)
+						}
+
+						// Call ``clientEncryption1.decrypt`` with the ``ciphertext``.
+						{
+							plaintext, err := clientEncryption1.Decrypt(context.Background(), ciphertext)
+							assert.Nil(mt, err, "error in Decrypt: %v", err)
+							assert.Equal(mt, plaintext.StringValue(), "test", "expected plaintext 'test', got %q", plaintext.StringValue())
+						}
+
+						// Call ``clientEncryption2.decrypt`` with the ``ciphertext``.
+						{
+							plaintext, err := clientEncryption2.Decrypt(context.Background(), ciphertext)
+							assert.Nil(mt, err, "error in Decrypt: %v", err)
+							assert.Equal(mt, plaintext.StringValue(), "test", "expected plaintext 'test', got %q", plaintext.StringValue())
+						}
+					})
+				}
+			}
+		})
+	})
 }
 
 func getWatcher(mt *mtest.T, streamType mongo.StreamType, cpt *cseProseTest) watcher {

diff --git a/x/mongo/driver/mongocrypt/mongocrypt.go b/x/mongo/driver/mongocrypt/mongocrypt.go
@@ -29,6 +29,13 @@ type MongoCrypt struct {
 	wrapped *C.mongocrypt_t
 }
 
+// Version returns the version string for the loaded libmongocrypt, or an empty string
+// if libmongocrypt was not loaded.
+func Version() string {
+	str := C.GoString(C.mongocrypt_version(nil))
+	return str
+}
+
 // NewMongoCrypt constructs a new MongoCrypt instance configured using the provided MongoCryptOptions.
 func NewMongoCrypt(opts *options.MongoCryptOptions) (*MongoCrypt, error) {
 	// create mongocrypt_t handle

diff --git a/x/mongo/driver/mongocrypt/mongocrypt_not_enabled.go b/x/mongo/driver/mongocrypt/mongocrypt_not_enabled.go
@@ -19,6 +19,12 @@ const cseNotSupportedMsg = "client-side encryption not enabled. add the cse buil
 // MongoCrypt represents a mongocrypt_t handle.
 type MongoCrypt struct{}
 
+// Version returns the version string for the loaded libmongocrypt, or an empty string
+// if libmongocrypt was not loaded.
+func Version() string {
+	return ""
+}
+
 // NewMongoCrypt constructs a new MongoCrypt instance configured using the provided MongoCryptOptions.
 func NewMongoCrypt(opts *options.MongoCryptOptions) (*MongoCrypt, error) {
 	panic(cseNotSupportedMsg)