RBD: OOMKills occurs when secret metadata encryption type is used wit…

…h multiple PVC create request. ceph#3472

Signed-off-by: Stefan Haas <shaas@suse.com>

go.mod

@@ -45,6 +45,7 @@ require (
 	k8s.io/pod-security-admission v0.0.0
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
 	sigs.k8s.io/controller-runtime v0.14.4
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 )
 
 require (

go.sum

@@ -1253,6 +1253,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

internal/kms/secretskms.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ceph/ceph-csi/internal/util/k8s"
 
 	"golang.org/x/crypto/scrypt"
+	"golang.org/x/sync/semaphore"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -48,6 +49,8 @@ const (
 	metadataSecretNamespaceKey = "secretNamespace"
 )
 
+var scryptSem = semaphore.NewWeighted(int64(1))
+
 // secretsKMS is default KMS implementation that means no KMS is in use.
 type secretsKMS struct {
 	integratedDEK
@@ -271,6 +274,13 @@ func (kms secretsMetadataKMS) GetSecret(volumeID string) (string, error) {
 // generateCipher returns a AEAD cipher based on a passphrase and salt
 // (volumeID). The cipher can then be used to encrypt/decrypt the DEK.
 func generateCipher(passphrase, salt string) (cipher.AEAD, error) {
+	// Note: This is memory heavy!
+	// Acquire blocks concurrent access so that only 1 worker can call scrypt.Key at a time.
+	if err := scryptSem.Acquire(context.TODO(), 1); err != nil {
+		return nil, err
+	}
+	defer scryptSem.Release(1)
+
 	key, err := scrypt.Key([]byte(passphrase), []byte(salt), 32768, 8, 1, 32)
 	if err != nil {
 		return nil, err

internal/kms/secretskms_test.go

@@ -62,6 +62,27 @@ func TestGenerateCipher(t *testing.T) {
 	assert.NotNil(t, aead)
 }
 
+func TestGenerateCipherConcurrent(t *testing.T) {
+	t.Parallel()
+	// nolint:gosec // this passphrase is intentionally hardcoded
+	passphrase := "my-cool-luks-passphrase"
+	salt := "unique-id-for-the-volume"
+
+	runGenerateCipher := func(passphrase string, salt string) {
+		aead, err := generateCipher(passphrase, salt)
+		assert.NoError(t, err)
+		assert.NotNil(t, aead)
+	}
+
+	for i := 0; i < 5; i++ {
+		go runGenerateCipher(passphrase, salt)
+	}
+
+	for i := 0; i < 5; i++ {
+		runGenerateCipher(passphrase, salt)
+	}
+}
+
 func TestInitSecretsMetadataKMS(t *testing.T) {
 	t.Parallel()
 	args := ProviderInitArgs{

vendor/modules.txt

@@ -578,6 +578,9 @@ golang.org/x/net/trace
 ## explicit; go 1.17
 golang.org/x/oauth2
 golang.org/x/oauth2/internal
+# golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+## explicit
+golang.org/x/sync/semaphore
 # golang.org/x/sys v0.5.0
 ## explicit; go 1.17
 golang.org/x/sys/cpu