From 18fcfb2cb73573b87706785ceff5a72723337246 Mon Sep 17 00:00:00 2001 From: Steven Clark Date: Thu, 31 Mar 2022 13:42:49 -0400 Subject: [PATCH 01/76] Starter PKI CA Storage API (#14796) * Simple starting PKI storage api for CA rotation * Add key and issuer storage apis * Add listKeys and listIssuers storage implementations * Add simple keys and issuers configuration storage api methods --- builtin/logical/pki/storage.go | 188 ++++++++++++++++++++++++++++ builtin/logical/pki/storage_test.go | 163 ++++++++++++++++++++++++ 2 files changed, 351 insertions(+) create mode 100644 builtin/logical/pki/storage.go create mode 100644 builtin/logical/pki/storage_test.go diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go new file mode 100644 index 0000000000000..ed440fcaf2a4f --- /dev/null +++ b/builtin/logical/pki/storage.go @@ -0,0 +1,188 @@ +package pki + +import ( + "context" + "fmt" + + "github.com/hashicorp/vault/sdk/helper/certutil" + "github.com/hashicorp/vault/sdk/helper/errutil" + "github.com/hashicorp/vault/sdk/logical" +) + +const ( + storageKeyConfig = "/config/keys" + storeageIssuerConfig = "/config/issuers" + keyPrefix = "/config/key/" + issuerPrefix = "/config/issuer/" +) + +type keyId string + +func (p keyId) String() string { + return string(p) +} + +type issuerId string + +func (p issuerId) String() string { + return string(p) +} + +type key struct { + ID keyId `json:"id" structs:"id" mapstructure:"id"` + PrivateKeyType certutil.PrivateKeyType `json:"private_key_type" structs:"private_key_type" mapstructure:"private_key_type"` + PrivateKey string `json:"private_key" structs:"private_key" mapstructure:"private_key"` +} + +type issuer struct { + ID issuerId `json:"id" structs:"id" mapstructure:"id"` + Name string `json:"name" structs:"name" mapstructure:"name"` + KeyID keyId `json:"key_id" structs:"key_id" mapstructure:"key_id"` + Certificate string `json:"certificate" structs:"certificate" mapstructure:"certificate"` + CAChain []string `json:"ca_chain" structs:"ca_chain" mapstructure:"ca_chain"` + SerialNumber string `json:"serial_number" structs:"serial_number" mapstructure:"serial_number"` +} + +type keyConfig struct { + DefaultKeyId keyId `json:"default" structs:"default" mapstructure:"default"` +} + +type issuerConfig struct { + DefaultIssuerId issuerId `json:"default" structs:"default" mapstructure:"default"` +} + +func listKeys(ctx context.Context, s logical.Storage) ([]keyId, error) { + strList, err := s.List(ctx, keyPrefix) + if err != nil { + return nil, err + } + + keyIds := make([]keyId, 0, len(strList)) + for _, entry := range strList { + keyIds = append(keyIds, keyId(entry)) + } + + return keyIds, nil +} + +func fetchKeyById(ctx context.Context, s logical.Storage, keyId keyId) (*key, error) { + keyEntry, err := s.Get(ctx, keyPrefix+keyId.String()) + if err != nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki key: %v", err)} + } + if keyEntry == nil { + // FIXME: Dedicated/specific error for this? + return nil, errutil.UserError{Err: fmt.Sprintf("pki key id %s does not exist", keyId.String())} + } + + var key key + if err := keyEntry.DecodeJSON(&key); err != nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode pki key with id %s: %v", keyId.String(), err)} + } + + return &key, nil +} + +func writeKey(ctx context.Context, s logical.Storage, key key) error { + keyId := key.ID + + json, err := logical.StorageEntryJSON(keyPrefix+keyId.String(), key) + if err != nil { + return err + } + + return s.Put(ctx, json) +} + +func listIssuers(ctx context.Context, s logical.Storage) ([]issuerId, error) { + strList, err := s.List(ctx, issuerPrefix) + if err != nil { + return nil, err + } + + issuerIds := make([]issuerId, 0, len(strList)) + for _, entry := range strList { + issuerIds = append(issuerIds, issuerId(entry)) + } + + return issuerIds, nil +} + +func fetchIssuerById(ctx context.Context, s logical.Storage, issuerId issuerId) (*issuer, error) { + issuerEntry, err := s.Get(ctx, issuerPrefix+issuerId.String()) + if err != nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki issuer: %v", err)} + } + if issuerEntry == nil { + // FIXME: Dedicated/specific error for this? + return nil, errutil.UserError{Err: fmt.Sprintf("pki issuer id %s does not exist", issuerId.String())} + } + + var issuer issuer + if err := issuerEntry.DecodeJSON(&issuer); err != nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode pki issuer with id %s: %v", issuerId.String(), err)} + } + + return &issuer, nil +} + +func writeIssuer(ctx context.Context, s logical.Storage, issuer issuer) error { + issuerId := issuer.ID + + json, err := logical.StorageEntryJSON(issuerPrefix+issuerId.String(), issuer) + if err != nil { + return err + } + + return s.Put(ctx, json) +} + +func setKeysConfig(ctx context.Context, s logical.Storage, config *keyConfig) error { + json, err := logical.StorageEntryJSON(storageKeyConfig, config) + if err != nil { + return err + } + + return s.Put(ctx, json) +} + +func getKeysConfig(ctx context.Context, s logical.Storage) (*keyConfig, error) { + keyConfigEntry, err := s.Get(ctx, storageKeyConfig) + if err != nil { + return nil, err + } + + keyConfig := &keyConfig{} + if keyConfigEntry != nil { + if err := keyConfigEntry.DecodeJSON(keyConfig); err != nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode key configuration: %v", err)} + } + } + + return keyConfig, nil +} + +func setIssuersConfig(ctx context.Context, s logical.Storage, config *issuerConfig) error { + json, err := logical.StorageEntryJSON(storeageIssuerConfig, config) + if err != nil { + return err + } + + return s.Put(ctx, json) +} + +func getIssuersConfig(ctx context.Context, s logical.Storage) (*issuerConfig, error) { + issuerConfigEntry, err := s.Get(ctx, storeageIssuerConfig) + if err != nil { + return nil, err + } + + issuerConfig := &issuerConfig{} + if issuerConfigEntry != nil { + if err := issuerConfigEntry.DecodeJSON(issuerConfig); err != nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode issuer configuration: %v", err)} + } + } + + return issuerConfig, nil +} diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go new file mode 100644 index 0000000000000..a3616fbf542a1 --- /dev/null +++ b/builtin/logical/pki/storage_test.go @@ -0,0 +1,163 @@ +package pki + +import ( + "context" + "crypto/rand" + "testing" + + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/helper/certutil" + "github.com/hashicorp/vault/sdk/logical" + "github.com/stretchr/testify/require" +) + +var ctx = context.Background() + +func Test_ConfigsRoundTrip(t *testing.T) { + _, s := createBackendWithStorage(t) + + // Verify we handle nothing stored properly + keyConfigEmpty, err := getKeysConfig(ctx, s) + require.NoError(t, err) + require.Equal(t, &keyConfig{}, keyConfigEmpty) + + issuerConfigEmpty, err := getIssuersConfig(ctx, s) + require.NoError(t, err) + require.Equal(t, &issuerConfig{}, issuerConfigEmpty) + + // Now attempt to store and reload properly + origKeyConfig := &keyConfig{ + DefaultKeyId: genKeyId(t), + } + origIssuerConfig := &issuerConfig{ + DefaultIssuerId: genIssuerId(t), + } + + err = setKeysConfig(ctx, s, origKeyConfig) + require.NoError(t, err) + err = setIssuersConfig(ctx, s, origIssuerConfig) + require.NoError(t, err) + + keyConfig, err := getKeysConfig(ctx, s) + require.NoError(t, err) + require.Equal(t, origKeyConfig, keyConfig) + + issuerConfig, err := getIssuersConfig(ctx, s) + require.NoError(t, err) + require.Equal(t, origIssuerConfig, issuerConfig) +} + +func Test_IssuerRoundTrip(t *testing.T) { + b, s := createBackendWithStorage(t) + issuer1, key1 := genIssuerAndKey(t, b) + issuer2, key2 := genIssuerAndKey(t, b) + + // We get an error when issuer id not found + _, err := fetchIssuerById(ctx, s, issuer1.ID) + require.Error(t, err) + + // We get an error when key id not found + _, err = fetchKeyById(ctx, s, key1.ID) + require.Error(t, err) + + // Now write out our issuers and keys + err = writeKey(ctx, s, key1) + require.NoError(t, err) + err = writeIssuer(ctx, s, issuer1) + require.NoError(t, err) + + err = writeKey(ctx, s, key2) + require.NoError(t, err) + err = writeIssuer(ctx, s, issuer2) + require.NoError(t, err) + + fetchedKey1, err := fetchKeyById(ctx, s, key1.ID) + require.NoError(t, err) + + fetchedIssuer1, err := fetchIssuerById(ctx, s, issuer1.ID) + require.NoError(t, err) + + require.Equal(t, &key1, fetchedKey1) + require.Equal(t, &issuer1, fetchedIssuer1) + + keys, err := listKeys(ctx, s) + require.NoError(t, err) + + require.ElementsMatch(t, []keyId{key1.ID, key2.ID}, keys) + + issuers, err := listIssuers(ctx, s) + require.NoError(t, err) + + require.ElementsMatch(t, []issuerId{issuer1.ID, issuer2.ID}, issuers) +} + +func genIssuerAndKey(t *testing.T, b *backend) (issuer, key) { + certBundle, err := genCertBundle(t, b) + require.NoError(t, err) + + keyId := genKeyId(t) + + pkiKey := key{ + ID: keyId, + PrivateKeyType: certBundle.PrivateKeyType, + PrivateKey: certBundle.PrivateKey, + } + + issuerId := genIssuerId(t) + + pkiIssuer := issuer{ + ID: issuerId, + KeyID: keyId, + Certificate: certBundle.Certificate, + CAChain: certBundle.CAChain, + SerialNumber: certBundle.SerialNumber, + } + + return pkiIssuer, pkiKey +} + +func genIssuerId(t *testing.T) issuerId { + issuerIdStr, err := uuid.GenerateUUID() + require.NoError(t, err) + return issuerId(issuerIdStr) +} + +func genKeyId(t *testing.T) keyId { + keyIdStr, err := uuid.GenerateUUID() + require.NoError(t, err) + return keyId(keyIdStr) +} + +func genCertBundle(t *testing.T, b *backend) (*certutil.CertBundle, error) { + // Pretty gross just to generate a cert bundle, but + fields := addCACommonFields(map[string]*framework.FieldSchema{}) + fields = addCAKeyGenerationFields(fields) + fields = addCAIssueFields(fields) + apiData := &framework.FieldData{ + Schema: fields, + Raw: map[string]interface{}{ + "exported": "internal", + "cn": "example.com", + "ttl": 3600, + }, + } + _, _, role, respErr := b.getGenerationParams(ctx, apiData, "/pki") + require.Nil(t, respErr) + + input := &inputBundle{ + req: &logical.Request{ + Operation: logical.UpdateOperation, + Path: "issue/testrole", + Storage: b.storage, + }, + apiData: apiData, + role: role, + } + parsedCertBundle, err := generateCert(ctx, b, input, nil, true, rand.Reader) + + require.NoError(t, err) + certBundle, err := parsedCertBundle.ToCertBundle() + require.NoError(t, err) + return certBundle, err +} From d57be55f8328420a68b23140fa1f5c8c88ea092e Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 31 Mar 2022 13:51:16 -0400 Subject: [PATCH 02/76] Handle resolving key, issuer references The API context will usually have a user-specified reference to the key. This is either the literal string "default" to select the default key, an identifier of the key, or a slug name for the key. Here, we wish to resolve this reference to an actual identifier that can be understood by storage. Also adds the missing Name field to keys. Signed-off-by: Alexander Scheel --- builtin/logical/pki/storage.go | 79 ++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index ed440fcaf2a4f..1aead6f1501b6 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -30,6 +30,7 @@ func (p issuerId) String() string { type key struct { ID keyId `json:"id" structs:"id" mapstructure:"id"` + Name string `json:"name" structs:"name" mapstructure:"name"` PrivateKeyType certutil.PrivateKeyType `json:"private_key_type" structs:"private_key_type" mapstructure:"private_key_type"` PrivateKey string `json:"private_key" structs:"private_key" mapstructure:"private_key"` } @@ -108,6 +109,45 @@ func listIssuers(ctx context.Context, s logical.Storage) ([]issuerId, error) { return issuerIds, nil } +func resolveKeyReference(ctx context.Context, s logical.Storage, reference string) (keyId, error) { + if reference == "default" { + // Handle fetching the default key. + config, err := getKeysConfig(ctx, s) + if err != nil { + return keyId("config-error"), err + } + + return config.DefaultKeyId, nil + } + + keys, err := listKeys(ctx, s) + if err != nil { + return keyId("list-error"), err + } + + // Cheaper to list keys and check if an id is a match... + for _, key_id := range keys { + if key_id == keyId(reference) { + return key_id, nil + } + } + + // ... than to pull all keys from storage. + for _, key_id := range keys { + key, err := fetchKeyById(ctx, s, key_id) + if err != nil { + return keyId("key-read"), err + } + + if key.Name == reference { + return key.ID, nil + } + } + + // Otherwise, we must not have found the key. + return keyId("not-found"), errutil.UserError{Err: fmt.Sprintf("unable to find PKI key for reference: %v", reference)} +} + func fetchIssuerById(ctx context.Context, s logical.Storage, issuerId issuerId) (*issuer, error) { issuerEntry, err := s.Get(ctx, issuerPrefix+issuerId.String()) if err != nil { @@ -186,3 +226,42 @@ func getIssuersConfig(ctx context.Context, s logical.Storage) (*issuerConfig, er return issuerConfig, nil } + +func resolveIssuerReference(ctx context.Context, s logical.Storage, reference string) (issuerId, error) { + if reference == "default" { + // Handle fetching the default issuer. + config, err := getIssuersConfig(ctx, s) + if err != nil { + return issuerId("config-error"), err + } + + return config.DefaultIssuerId, nil + } + + issuers, err := listIssuers(ctx, s) + if err != nil { + return issuerId("list-error"), err + } + + // Cheaper to list issuers and check if an id is a match... + for _, issuer_id := range issuers { + if issuer_id == issuerId(reference) { + return issuer_id, nil + } + } + + // ... than to pull all issuers from storage. + for _, issuer_id := range issuers { + issuer, err := fetchIssuerById(ctx, s, issuer_id) + if err != nil { + return issuerId("issuer-read"), err + } + + if issuer.Name == reference { + return issuer.ID, nil + } + } + + // Otherwise, we must not have found the issuer. + return issuerId("not-found"), errutil.UserError{Err: fmt.Sprintf("unable to find PKI issuer for reference: %v", reference)} +} From b8fe80da9938dde907bc9d5b8ce07930c543413b Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 31 Mar 2022 12:15:19 -0400 Subject: [PATCH 03/76] Add method to fetch an issuer's cert bundle This adds a method to construct a certutil.CertBundle from the specified issuer identifier, optionally loading its corresponding key for signing. Signed-off-by: Alexander Scheel --- builtin/logical/pki/storage.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 1aead6f1501b6..092046dbaed2a 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -265,3 +265,31 @@ func resolveIssuerReference(ctx context.Context, s logical.Storage, reference st // Otherwise, we must not have found the issuer. return issuerId("not-found"), errutil.UserError{Err: fmt.Sprintf("unable to find PKI issuer for reference: %v", reference)} } + +// Builds a certutil.CertBundle from the specified issuer identifier, +// optionally loading the key or not. +func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuerId, loadKey bool) (*certutil.CertBundle, error) { + issuer, err := fetchIssuerById(ctx, s, id) + if err != nil { + return nil, err + } + + var bundle certutil.CertBundle + bundle.Certificate = issuer.Certificate + bundle.IssuingCA = issuer.CAChain[0] + bundle.CAChain = issuer.CAChain + bundle.SerialNumber = issuer.SerialNumber + + // Fetch the key if it exists. Sometimes we don't need the key immediately. + if loadKey && issuer.KeyID != keyId("") { + key, err := fetchKeyById(ctx, s, issuer.KeyID) + if err != nil { + return nil, err + } + + bundle.PrivateKeyType = key.PrivateKeyType + bundle.PrivateKey = key.PrivateKey + } + + return &bundle, nil +} From 55f08412509250f7143b1fd42a212e7eb1e13dc0 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Fri, 1 Apr 2022 13:18:34 -0400 Subject: [PATCH 04/76] Refactor certutil PrivateKey PEM handling This refactors the parsing of PrivateKeys from PEM blobs into shared methods (ParsePEMKey, ParseDERKey) that can be reused by the existing Bundle parsing logic (ParsePEMBundle) or independently in the new issuers/key-based PKI storage code. Signed-off-by: Alexander Scheel --- sdk/helper/certutil/helpers.go | 76 ++++++++++++++++++++-------------- sdk/helper/certutil/types.go | 20 +++++++-- 2 files changed, 63 insertions(+), 33 deletions(-) diff --git a/sdk/helper/certutil/helpers.go b/sdk/helper/certutil/helpers.go index c26020f435758..6d415bbadfd34 100644 --- a/sdk/helper/certutil/helpers.go +++ b/sdk/helper/certutil/helpers.go @@ -150,6 +150,46 @@ func ParsePKIJSON(input []byte) (*ParsedCertBundle, error) { return nil, errutil.UserError{Err: "unable to parse out of either secret data or a secret object"} } +func ParseDERKey(privateKeyBytes []byte) (signer crypto.Signer, format BlockType, err error) { + if signer, err = x509.ParseECPrivateKey(privateKeyBytes); err == nil { + format = ECBlock + return + } + + if signer, err = x509.ParsePKCS1PrivateKey(privateKeyBytes); err == nil { + format = PKCS1Block + return + } + + var rawKey interface{} + if rawKey, err = x509.ParsePKCS8PrivateKey(privateKeyBytes); err == nil { + switch rawSigner := rawKey.(type) { + case *rsa.PrivateKey: + signer = rawSigner + case *ecdsa.PrivateKey: + signer = rawSigner + case ed25519.PrivateKey: + signer = rawSigner + default: + return nil, UnknownBlock, errutil.InternalError{Err: "unknown type for parsed PKCS8 Private Key"} + } + + format = PKCS8Block + return + } + + return nil, UnknownBlock, err +} + +func ParsePEMKey(keyPem string) (crypto.Signer, BlockType, error) { + pemBlock, _ := pem.Decode([]byte(keyPem)) + if pemBlock == nil { + return nil, UnknownBlock, errutil.UserError{Err: "no data found in PEM block"} + } + + return ParseDERKey(pemBlock.Bytes) +} + // ParsePEMBundle takes a string of concatenated PEM-format certificate // and private key values and decodes/parses them, checking validity along // the way. The first certificate must be the subject certificate and issuing @@ -170,43 +210,19 @@ func ParsePEMBundle(pemBundle string) (*ParsedCertBundle, error) { return nil, errutil.UserError{Err: "no data found in PEM block"} } - if signer, err := x509.ParseECPrivateKey(pemBlock.Bytes); err == nil { + if signer, format, err := ParseDERKey(pemBlock.Bytes); err == nil { if parsedBundle.PrivateKeyType != UnknownPrivateKey { return nil, errutil.UserError{Err: "more than one private key given; provide only one private key in the bundle"} } - parsedBundle.PrivateKeyFormat = ECBlock - parsedBundle.PrivateKeyType = ECPrivateKey - parsedBundle.PrivateKeyBytes = pemBlock.Bytes - parsedBundle.PrivateKey = signer - } else if signer, err := x509.ParsePKCS1PrivateKey(pemBlock.Bytes); err == nil { - if parsedBundle.PrivateKeyType != UnknownPrivateKey { - return nil, errutil.UserError{Err: "more than one private key given; provide only one private key in the bundle"} + parsedBundle.PrivateKeyFormat = format + parsedBundle.PrivateKeyType = GetPrivateKeyTypeFromSigner(signer) + if parsedBundle.PrivateKeyType == UnknownPrivateKey { + return nil, errutil.UserError{Err: "Unknown type of private key included in the bundle: %v"} } - parsedBundle.PrivateKeyType = RSAPrivateKey - parsedBundle.PrivateKeyFormat = PKCS1Block + parsedBundle.PrivateKeyBytes = pemBlock.Bytes parsedBundle.PrivateKey = signer - } else if signer, err := x509.ParsePKCS8PrivateKey(pemBlock.Bytes); err == nil { - parsedBundle.PrivateKeyFormat = PKCS8Block - - if parsedBundle.PrivateKeyType != UnknownPrivateKey { - return nil, errutil.UserError{Err: "More than one private key given; provide only one private key in the bundle"} - } - switch signer := signer.(type) { - case *rsa.PrivateKey: - parsedBundle.PrivateKey = signer - parsedBundle.PrivateKeyType = RSAPrivateKey - parsedBundle.PrivateKeyBytes = pemBlock.Bytes - case *ecdsa.PrivateKey: - parsedBundle.PrivateKey = signer - parsedBundle.PrivateKeyType = ECPrivateKey - parsedBundle.PrivateKeyBytes = pemBlock.Bytes - case ed25519.PrivateKey: - parsedBundle.PrivateKey = signer - parsedBundle.PrivateKeyType = Ed25519PrivateKey - parsedBundle.PrivateKeyBytes = pemBlock.Bytes - } } else if certificates, err := x509.ParseCertificates(pemBlock.Bytes); err == nil { certPath = append(certPath, &CertBlock{ Certificate: certificates[0], diff --git a/sdk/helper/certutil/types.go b/sdk/helper/certutil/types.go index 076a4e352854b..aab082dc08dc7 100644 --- a/sdk/helper/certutil/types.go +++ b/sdk/helper/certutil/types.go @@ -78,9 +78,10 @@ type BlockType string // Well-known formats const ( - PKCS1Block BlockType = "RSA PRIVATE KEY" - PKCS8Block BlockType = "PRIVATE KEY" - ECBlock BlockType = "EC PRIVATE KEY" + UnknownBlock BlockType = "" + PKCS1Block BlockType = "RSA PRIVATE KEY" + PKCS8Block BlockType = "PRIVATE KEY" + ECBlock BlockType = "EC PRIVATE KEY" ) // ParsedPrivateKeyContainer allows common key setting for certs and CSRs @@ -137,6 +138,19 @@ type ParsedCSRBundle struct { CSR *x509.CertificateRequest } +func GetPrivateKeyTypeFromSigner(signer crypto.Signer) PrivateKeyType { + switch signer.(type) { + case *rsa.PrivateKey: + return RSAPrivateKey + case *ecdsa.PrivateKey: + return ECPrivateKey + case ed25519.PrivateKey: + return Ed25519PrivateKey + default: + return UnknownPrivateKey + } +} + // ToPEMBundle converts a string-based certificate bundle // to a PEM-based string certificate bundle in trust path // order, leaf certificate first From ca3e48df247dc383cac85219b8a6933e354a41c4 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 5 Apr 2022 11:59:10 -0400 Subject: [PATCH 05/76] Add importKey, importCert to PKI storage importKey is generally preferable to the low-level writeKey for adding new entries. This takes only the contents of the private key (as a string -- so a PEM bundle or a managed key handle) and checks if it already exists in the storage. If it does, it returns the existing key instance. Otherwise, we create a new one. In the process, we detect any issuers using this key and link them back to the new key entry. The same holds for importCert over importKey, with the note that keys are not modified when importing certificates. Signed-off-by: Alexander Scheel --- builtin/logical/pki/storage.go | 221 +++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 092046dbaed2a..1f4c52541e7ae 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -2,8 +2,13 @@ package pki import ( "context" + "crypto" + "crypto/x509" + "encoding/pem" "fmt" + "strings" + "github.com/hashicorp/go-uuid" "github.com/hashicorp/vault/sdk/helper/certutil" "github.com/hashicorp/vault/sdk/helper/errutil" "github.com/hashicorp/vault/sdk/logical" @@ -52,6 +57,11 @@ type issuerConfig struct { DefaultIssuerId issuerId `json:"default" structs:"default" mapstructure:"default"` } +func (k key) GetSigner() (crypto.Signer, error) { + signer, _, err := certutil.ParsePEMKey(k.PrivateKey) + return signer, err +} + func listKeys(ctx context.Context, s logical.Storage) ([]keyId, error) { strList, err := s.List(ctx, keyPrefix) if err != nil { @@ -95,6 +105,118 @@ func writeKey(ctx context.Context, s logical.Storage, key key) error { return s.Put(ctx, json) } +func importKey(ctx context.Context, s logical.Storage, keyValue string) (*key, bool, error) { + // importKey imports the specified PEM-format key (from keyValue) into + // the new PKI storage format. The first return field is a reference to + // the new key; the second is whether or not the key already existed + // during import (in which case, *key points to the existing key reference + // and identifier); the last return field is whether or not an error + // occurred. + // + // Before we can import a known key, we first need to know if the key + // exists in storage already. This means iterating through all known + // keys and comparing their private value against this value. + knownKeys, err := listKeys(ctx, s) + if err != nil { + return nil, false, err + } + + // Before we return below, we need to iterate over _all_ issuers and see if + // one of them has a missing KeyId link, and if so, point it back to + // ourselves. We fetch the list of issuers up front, even when don't need + // it, to give ourselves a better chance of succeeding below. + knownIssuers, err := listIssuers(ctx, s) + if err != nil { + return nil, false, err + } + + for _, identifier := range knownKeys { + existingKey, err := fetchKeyById(ctx, s, identifier) + if err != nil { + return nil, false, err + } + + if existingKey.PrivateKey == keyValue { + // Here, we don't need to stitch together the issuer entries, + // because the last run should've done that for us (or, when + // importing an issuer). + return existingKey, true, nil + } + } + + // Haven't found a key, so we've gotta create it and write it into storage. + var result key + uuid, err := uuid.GenerateUUID() + if err != nil { + return nil, false, err + } + result.ID = keyId(uuid) + result.PrivateKey = keyValue + + // Extracting the signer is necessary for two reasons: first, to get the + // public key for comparison with existing issuers; second, to get the + // corresponding private key type. + keySigner, err := result.GetSigner() + if err != nil { + return nil, false, err + } + keyPublic := keySigner.Public() + result.PrivateKeyType = certutil.GetPrivateKeyTypeFromSigner(keySigner) + + // Finally we can write the key to storage. + if err := writeKey(ctx, s, result); err != nil { + return nil, false, err + } + + // Now, for each issuer, try and compute the issuer<->key link if missing. + for _, identifier := range knownIssuers { + existingIssuer, err := fetchIssuerById(ctx, s, identifier) + if err != nil { + return nil, false, err + } + + // If the KeyID value is already present, we can skip it. + if len(existingIssuer.KeyID) > 0 { + continue + } + + // Otherwise, compare public values. Note that there might be multiple + // certificates (e.g., cross-signed) with the same key. + + cert, err := existingIssuer.GetCertificate() + if err != nil { + // Malformed issuer. + return nil, false, err + } + + equal, err := certutil.ComparePublicKeys(cert.PublicKey, keyPublic) + if err != nil { + return nil, false, err + } + + if equal { + // These public keys are equal, so this key entry must be the + // corresponding private key to this issuer; update it accordingly. + existingIssuer.KeyID = result.ID + if err := writeIssuer(ctx, s, *existingIssuer); err != nil { + return nil, false, err + } + } + } + + // All done; return our new key reference. + return &result, false, nil +} + +func (i issuer) GetCertificate() (*x509.Certificate, error) { + block, _ := pem.Decode([]byte(i.Certificate)) + if block == nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to parse certificate from issuer: invalid PEM: %v", i.ID)} + } + + return x509.ParseCertificate(block.Bytes) +} + func listIssuers(ctx context.Context, s logical.Storage) ([]issuerId, error) { strList, err := s.List(ctx, issuerPrefix) if err != nil { @@ -177,6 +299,105 @@ func writeIssuer(ctx context.Context, s logical.Storage, issuer issuer) error { return s.Put(ctx, json) } +func importIssuer(ctx context.Context, s logical.Storage, certValue string) (*issuer, bool, error) { + // importIssuers imports the specified PEM-format certificate (from + // certValue) into the new PKI storage format. The first return field is a + // reference to the new issuer; the second is whether or not the issuer + // already existed during import (in which case, *issuer points to the + // existing issuer reference and identifier); the last return field is + // whether or not an error occurred. + // + // Before we can import a known issuer, we first need to know if the issuer + // exists in storage already. This means iterating through all known + // issuers and comparing their private value against this value. + knownIssuers, err := listIssuers(ctx, s) + if err != nil { + return nil, false, err + } + + // Before we return below, we need to iterate over _all_ keys and see if + // one of them a public key matching this certificate, and if so, update our + // link accordingly. We fetch the list of keys up front, even may not need + // it, to give ourselves a better chance of succeeding below. + knownKeys, err := listKeys(ctx, s) + if err != nil { + return nil, false, err + } + + for _, identifier := range knownIssuers { + existingIssuer, err := fetchIssuerById(ctx, s, identifier) + if err != nil { + return nil, false, err + } + + if existingIssuer.Certificate == certValue { + // Here, we don't need to stitch together the key entries, + // because the last run should've done that for us (or, when + // importing a key). + return existingIssuer, true, nil + } + } + + // Haven't found an issuer, so we've gotta create it and write it into + // storage. + var result issuer + uuid, err := uuid.GenerateUUID() + if err != nil { + return nil, false, err + } + result.ID = issuerId(uuid) + result.Certificate = certValue + result.CAChain = []string{certValue} + + // Extracting the certificate is necessary for two reasons: first, it lets + // us fetch the serial number; second, for the public key comparison with + // known keys. + issuerCert, err := result.GetCertificate() + if err != nil { + return nil, false, err + } + + result.SerialNumber = strings.TrimSpace(certutil.GetHexFormatted(issuerCert.SerialNumber.Bytes(), ":")) + + // Now, for each key, try and compute the issuer<->key link. We delay + // writing issuer to storage as we won't need to update the key, only + // the issuer. + for _, identifier := range knownKeys { + existingKey, err := fetchKeyById(ctx, s, identifier) + if err != nil { + return nil, false, err + } + + // Fetch the signer for its Public() value. + signer, err := existingKey.GetSigner() + if err != nil { + return nil, false, err + } + + equal, err := certutil.ComparePublicKeys(issuerCert.PublicKey, signer.Public()) + if err != nil { + return nil, false, err + } + + if equal { + result.KeyID = existingKey.ID + // Here, there's exactly one stored key with the same public key + // as us, per guarantees in importKey; as we're importing an + // issuer, there's no other keys or issuers we'd need to read or + // update, so exit. + break + } + } + + // Finally we can write the issuer to storage. + if err := writeIssuer(ctx, s, result); err != nil { + return nil, false, err + } + + // All done; return our new key reference. + return &result, false, nil +} + func setKeysConfig(ctx context.Context, s logical.Storage, config *keyConfig) error { json, err := logical.StorageEntryJSON(storageKeyConfig, config) if err != nil { From a93690f972c963b8b5b7aa03be7c0e26a094c20d Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 5 Apr 2022 12:21:02 -0400 Subject: [PATCH 06/76] Add tests for importing issuers, keys This adds tests for importing keys and issuers into the new storage layout, ensuring that identifiers are correctly inferred and linked. Note that directly writing entries to storage (writeKey/writeissuer) will take KeyID links from the parent entry and should not be used for import; only existing entries should be updated with this info. Signed-off-by: Alexander Scheel --- builtin/logical/pki/storage_test.go | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index a3616fbf542a1..7b4fc2b4e5853 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -92,6 +92,61 @@ func Test_IssuerRoundTrip(t *testing.T) { require.ElementsMatch(t, []issuerId{issuer1.ID, issuer2.ID}, issuers) } +func Test_KeysIssuerImport(t *testing.T) { + b, s := createBackendWithStorage(t) + issuer1, key1 := genIssuerAndKey(t, b) + issuer2, key2 := genIssuerAndKey(t, b) + + // Key 1 before Issuer 1; Issuer 2 before Key 2. + // Remove KeyIDs from non-written entities before beginning. + key1.ID = "" + issuer1.ID = "" + issuer1.KeyID = "" + + key1_ref1, existing, err := importKey(ctx, s, key1.PrivateKey) + require.NoError(t, err) + require.False(t, existing) + require.Equal(t, key1.PrivateKey, key1_ref1.PrivateKey) + + key1_ref2, existing, err := importKey(ctx, s, key1.PrivateKey) + require.NoError(t, err) + require.True(t, existing) + require.Equal(t, key1.PrivateKey, key1_ref1.PrivateKey) + require.Equal(t, key1_ref1.ID, key1_ref2.ID) + + issuer1_ref1, existing, err := importIssuer(ctx, s, issuer1.Certificate) + require.NoError(t, err) + require.False(t, existing) + require.Equal(t, issuer1.Certificate, issuer1_ref1.Certificate) + require.Equal(t, key1_ref1.ID, issuer1_ref1.KeyID) + + issuer1_ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate) + require.NoError(t, err) + require.True(t, existing) + require.Equal(t, issuer1.Certificate, issuer1_ref1.Certificate) + require.Equal(t, issuer1_ref1.ID, issuer1_ref2.ID) + require.Equal(t, key1_ref1.ID, issuer1_ref2.KeyID) + + err = writeIssuer(ctx, s, issuer2) + require.NoError(t, err) + + err = writeKey(ctx, s, key2) + require.NoError(t, err) + + issuer2_ref, existing, err := importIssuer(ctx, s, issuer2.Certificate) + require.NoError(t, err) + require.True(t, existing) + require.Equal(t, issuer2.Certificate, issuer2_ref.Certificate) + require.Equal(t, issuer2_ref.ID, issuer2.ID) + require.Equal(t, issuer2_ref.KeyID, issuer2.KeyID) + + key2_ref, existing, err := importKey(ctx, s, key2.PrivateKey) + require.NoError(t, err) + require.True(t, existing) + require.Equal(t, key2.PrivateKey, key2_ref.PrivateKey) + require.Equal(t, key2_ref.ID, key2.ID) +} + func genIssuerAndKey(t *testing.T, b *backend) (issuer, key) { certBundle, err := genCertBundle(t, b) require.NoError(t, err) From 1411be7ccd1bc3ddffffa7b2cb1649a373139300 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Thu, 31 Mar 2022 17:31:23 -0400 Subject: [PATCH 07/76] Implement PKI storage migration. - Hook into the backend::initialize function, calling the migration on a primary only. - Migrate an existing certificate bundle to the new issuers and key layout --- builtin/logical/pki/backend.go | 24 ++- builtin/logical/pki/config_util.go | 37 ++++ builtin/logical/pki/storage.go | 57 ++++-- builtin/logical/pki/storage_migrations.go | 166 ++++++++++++++++++ .../logical/pki/storage_migrations_test.go | 90 ++++++++++ builtin/logical/pki/storage_test.go | 34 ++-- 6 files changed, 364 insertions(+), 44 deletions(-) create mode 100644 builtin/logical/pki/config_util.go create mode 100644 builtin/logical/pki/storage_migrations.go create mode 100644 builtin/logical/pki/storage_migrations_test.go diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index c0a1a0916c8bc..162fcb5140d4f 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/hashicorp/vault/sdk/helper/consts" + "github.com/armon/go-metrics" "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/helper/namespace" @@ -119,7 +121,8 @@ func Backend(conf *logical.BackendConfig) *backend { secretCerts(&b), }, - BackendType: logical.TypeLogical, + BackendType: logical.TypeLogical, + InitializeFunc: b.initialize, } b.crlLifetime = time.Hour * 72 @@ -233,3 +236,22 @@ func (b *backend) metricsWrap(callType string, roleMode int, ofunc roleOperation return resp, err } } + +// initialize is used to perform a possible PKI storage migration if needed +func (b *backend) initialize(ctx context.Context, req *logical.InitializationRequest) error { + logger := b.Logger().Named("initialize") + + // Early exit if not a primary cluster or performance secondary with a local mount. + if b.System().ReplicationState().HasState(consts.ReplicationDRSecondary|consts.ReplicationPerformanceStandby) || + (!b.System().LocalMount() && b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary)) { + logger.Debug("skipping pki migration as we are not on primary or secondary with a local mount") + return nil + } + + if err := migrateStorage(ctx, req, logger); err != nil { + logger.Error("Error during migration of PKI mount: " + err.Error()) + return err + } + + return nil +} diff --git a/builtin/logical/pki/config_util.go b/builtin/logical/pki/config_util.go new file mode 100644 index 0000000000000..2ba36fe9d0fd4 --- /dev/null +++ b/builtin/logical/pki/config_util.go @@ -0,0 +1,37 @@ +package pki + +import ( + "context" + + "github.com/hashicorp/vault/sdk/logical" +) + +func updateDefaultKeyId(ctx context.Context, s logical.Storage, id keyId) error { + config, err := getKeysConfig(ctx, s) + if err != nil { + return err + } + + if config.DefaultKeyId != id { + return setKeysConfig(ctx, s, &keyConfig{ + DefaultKeyId: id, + }) + } + + return nil +} + +func updateDefaultIssuerId(ctx context.Context, s logical.Storage, id issuerId) error { + config, err := getIssuersConfig(ctx, s) + if err != nil { + return err + } + + if config.DefaultIssuerId != id { + return setIssuersConfig(ctx, s, &issuerConfig{ + DefaultIssuerId: id, + }) + } + + return nil +} diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 1f4c52541e7ae..4b6d5b23930e2 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -15,10 +15,13 @@ import ( ) const ( - storageKeyConfig = "/config/keys" - storeageIssuerConfig = "/config/issuers" - keyPrefix = "/config/key/" - issuerPrefix = "/config/issuer/" + storageKeyConfig = "config/keys" + storageIssuerConfig = "config/issuers" + keyPrefix = "config/key/" + issuerPrefix = "config/issuer/" + + legacyMigrationBundleLogKey = "config/legacyMigrationBundleLog" + legacyCertBundlePath = "config/ca_bundle" ) type keyId string @@ -105,6 +108,10 @@ func writeKey(ctx context.Context, s logical.Storage, key key) error { return s.Put(ctx, json) } +func deleteKey(ctx context.Context, s logical.Storage, id keyId) error { + return s.Delete(ctx, keyPrefix+id.String()) +} + func importKey(ctx context.Context, s logical.Storage, keyValue string) (*key, bool, error) { // importKey imports the specified PEM-format key (from keyValue) into // the new PKI storage format. The first return field is a reference to @@ -146,11 +153,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string) (*key, b // Haven't found a key, so we've gotta create it and write it into storage. var result key - uuid, err := uuid.GenerateUUID() - if err != nil { - return nil, false, err - } - result.ID = keyId(uuid) + result.ID = genKeyId() result.PrivateKey = keyValue // Extracting the signer is necessary for two reasons: first, to get the @@ -198,7 +201,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string) (*key, b // These public keys are equal, so this key entry must be the // corresponding private key to this issuer; update it accordingly. existingIssuer.KeyID = result.ID - if err := writeIssuer(ctx, s, *existingIssuer); err != nil { + if err := writeIssuer(ctx, s, existingIssuer); err != nil { return nil, false, err } } @@ -288,7 +291,7 @@ func fetchIssuerById(ctx context.Context, s logical.Storage, issuerId issuerId) return &issuer, nil } -func writeIssuer(ctx context.Context, s logical.Storage, issuer issuer) error { +func writeIssuer(ctx context.Context, s logical.Storage, issuer *issuer) error { issuerId := issuer.ID json, err := logical.StorageEntryJSON(issuerPrefix+issuerId.String(), issuer) @@ -299,6 +302,10 @@ func writeIssuer(ctx context.Context, s logical.Storage, issuer issuer) error { return s.Put(ctx, json) } +func deleteIssuer(ctx context.Context, s logical.Storage, id issuerId) error { + return s.Delete(ctx, issuerPrefix+id.String()) +} + func importIssuer(ctx context.Context, s logical.Storage, certValue string) (*issuer, bool, error) { // importIssuers imports the specified PEM-format certificate (from // certValue) into the new PKI storage format. The first return field is a @@ -341,11 +348,7 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string) (*is // Haven't found an issuer, so we've gotta create it and write it into // storage. var result issuer - uuid, err := uuid.GenerateUUID() - if err != nil { - return nil, false, err - } - result.ID = issuerId(uuid) + result.ID = genIssuerId() result.Certificate = certValue result.CAChain = []string{certValue} @@ -390,7 +393,7 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string) (*is } // Finally we can write the issuer to storage. - if err := writeIssuer(ctx, s, result); err != nil { + if err := writeIssuer(ctx, s, &result); err != nil { return nil, false, err } @@ -424,7 +427,7 @@ func getKeysConfig(ctx context.Context, s logical.Storage) (*keyConfig, error) { } func setIssuersConfig(ctx context.Context, s logical.Storage, config *issuerConfig) error { - json, err := logical.StorageEntryJSON(storeageIssuerConfig, config) + json, err := logical.StorageEntryJSON(storageIssuerConfig, config) if err != nil { return err } @@ -433,7 +436,7 @@ func setIssuersConfig(ctx context.Context, s logical.Storage, config *issuerConf } func getIssuersConfig(ctx context.Context, s logical.Storage) (*issuerConfig, error) { - issuerConfigEntry, err := s.Get(ctx, storeageIssuerConfig) + issuerConfigEntry, err := s.Get(ctx, storageIssuerConfig) if err != nil { return nil, err } @@ -514,3 +517,19 @@ func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuer return &bundle, nil } + +func genIssuerId() issuerId { + return issuerId(genUuid()) +} + +func genKeyId() keyId { + return keyId(genUuid()) +} + +func genUuid() string { + aUuid, err := uuid.GenerateUUID() + if err != nil { + panic(err) + } + return aUuid +} diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go new file mode 100644 index 0000000000000..73882f57461f6 --- /dev/null +++ b/builtin/logical/pki/storage_migrations.go @@ -0,0 +1,166 @@ +package pki + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "time" + + log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/vault/sdk/helper/certutil" + "github.com/hashicorp/vault/sdk/logical" +) + +// This allows us to record the version of the migration code within the log entry +// in case we find out in the future that something was horribly wrong with the migration, +// and we need to perform it again... +const latestMigrationVersion = 1 + +func migrateStorage(ctx context.Context, req *logical.InitializationRequest, logger log.Logger) error { + s := req.Storage + legacyBundle, err := getLegacyCertBundle(ctx, s) + if err != nil { + return err + } + + if legacyBundle == nil { + // No legacy certs to migrate, we are done... + logger.Debug("No legacy certs found, no migration required.") + return nil + } + + migrationEntry, err := getLegacyBundleMigrationLog(ctx, s) + if err != nil { + return err + } + hash, err := computeHashOfLegacyBundle(legacyBundle) + if err != nil { + return err + } + + if migrationEntry != nil { + // At this point we have already migrated something previously. + if migrationEntry.hash == hash && + migrationEntry.migrationVersion == latestMigrationVersion { + // The hashes are the same, no need to try and re-import... + logger.Debug("existing migration hash found and matched legacy bundle, skipping migration.") + return nil + } + } + + logger.Warn("performing PKI migration to new keys/issuers layout") + + err = migrateToIssuers(ctx, s, legacyBundle) + if err != nil { + return err + } + + err = setLegacyBundleMigrationLog(ctx, s, &legacyBundleMigration{ + hash: hash, + created: time.Now(), + migrationVersion: latestMigrationVersion, + }) + if err != nil { + return err + } + logger.Info("successfully completed migration to new keys/issuers layout") + return nil +} + +func computeHashOfLegacyBundle(bundle *certutil.CertBundle) (string, error) { + // We only hash the main certificate and the certs within the CAChain, + // assuming that any sort of change that occurred would have influenced one of those two fields. + hasher := sha256.New() + if _, err := hasher.Write([]byte(bundle.Certificate)); err != nil { + return "", err + } + for _, cert := range bundle.CAChain { + if _, err := hasher.Write([]byte(cert)); err != nil { + return "", err + } + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func migrateToIssuers(ctx context.Context, s logical.Storage, bundle *certutil.CertBundle) error { + defaultKey, _, err := importKey(ctx, s, bundle.PrivateKey) + if err != nil { + return err + } + + defaultIssuer, _, err := importIssuer(ctx, s, bundle.Certificate) + if err != nil { + return err + } + + for _, cert := range bundle.CAChain { + if _, _, err = importIssuer(ctx, s, cert); err != nil { + return err + } + } + + if err = updateDefaultKeyId(ctx, s, defaultKey.ID); err != nil { + return err + } + + if err = updateDefaultIssuerId(ctx, s, defaultIssuer.ID); err != nil { + return err + } + + // FIXME: Call function that will recompute the CAChain on issuers here. + return nil +} + +type legacyBundleMigration struct { + hash string + created time.Time + migrationVersion int +} + +func getLegacyBundleMigrationLog(ctx context.Context, s logical.Storage) (*legacyBundleMigration, error) { + entry, err := s.Get(ctx, legacyMigrationBundleLogKey) + if err != nil { + return nil, err + } + + if entry == nil { + return nil, nil + } + + lbm := &legacyBundleMigration{} + err = entry.DecodeJSON(lbm) + if err != nil { + // If we can't decode our bundle, lets scrap it and assume a blank value, + // re-running the migration will at most bring back an older certificate/private key + return nil, nil + } + return lbm, nil +} + +func setLegacyBundleMigrationLog(ctx context.Context, s logical.Storage, lbm *legacyBundleMigration) error { + json, err := logical.StorageEntryJSON(legacyMigrationBundleLogKey, lbm) + if err != nil { + return err + } + + return s.Put(ctx, json) +} + +func getLegacyCertBundle(ctx context.Context, s logical.Storage) (*certutil.CertBundle, error) { + entry, err := s.Get(ctx, legacyCertBundlePath) + if err != nil { + return nil, err + } + + if entry == nil { + return nil, nil + } + + cb := &certutil.CertBundle{} + err = entry.DecodeJSON(cb) + if err != nil { + return nil, err + } + + return cb, nil +} diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go new file mode 100644 index 0000000000000..1129319b4fd9b --- /dev/null +++ b/builtin/logical/pki/storage_migrations_test.go @@ -0,0 +1,90 @@ +package pki + +import ( + "context" + "testing" + + "github.com/hashicorp/vault/sdk/logical" + "github.com/stretchr/testify/require" +) + +func Test_migrateStorageEmptyStorage(t *testing.T) { + ctx := context.Background() + b, s := createBackendWithStorage(t) + request := &logical.InitializationRequest{Storage: s} + + err := migrateStorage(ctx, request, b.Logger()) + require.NoError(t, err) + + issuerIds, err := listIssuers(ctx, s) + require.NoError(t, err) + require.Empty(t, issuerIds) + + keyIds, err := listKeys(ctx, s) + require.NoError(t, err) + require.Empty(t, keyIds) + + logEntry, err := getLegacyBundleMigrationLog(ctx, s) + require.NoError(t, err) + require.Nil(t, logEntry) +} + +func Test_migrateStorageSimpleBundle(t *testing.T) { + ctx := context.Background() + b, s := createBackendWithStorage(t) + + bundle := genCertBundle(t, b) + json, err := logical.StorageEntryJSON(legacyCertBundlePath, bundle) + require.NoError(t, err) + err = s.Put(ctx, json) + require.NoError(t, err) + + request := &logical.InitializationRequest{Storage: s} + + err = migrateStorage(ctx, request, b.Logger()) + require.NoError(t, err) + + issuerIds, err := listIssuers(ctx, s) + require.NoError(t, err) + require.Equal(t, 1, len(issuerIds)) + + keyIds, err := listKeys(ctx, s) + require.NoError(t, err) + require.Equal(t, 1, len(keyIds)) + + logEntry, err := getLegacyBundleMigrationLog(ctx, s) + require.NoError(t, err) + require.NotNil(t, logEntry) + + issuerId := issuerIds[0] + keyId := keyIds[0] + issuer, err := fetchIssuerById(ctx, s, issuerId) + require.NoError(t, err) + + key, err := fetchKeyById(ctx, s, keyId) + require.NoError(t, err) + + require.Equal(t, issuerId, issuer.ID) + require.Equal(t, bundle.SerialNumber, issuer.SerialNumber) + require.Equal(t, bundle.Certificate, issuer.Certificate) + require.Equal(t, keyId, issuer.KeyID) + // FIXME: Add tests for CAChain... + + require.Equal(t, keyId, key.ID) + require.Equal(t, bundle.PrivateKey, key.PrivateKey) + require.Equal(t, bundle.PrivateKeyType, key.PrivateKeyType) + + // Make sure we kept the old bundle + certBundle, err := getLegacyCertBundle(ctx, s) + require.NoError(t, err) + require.Equal(t, bundle, certBundle) + + // Make sure we setup the default values + keysConfig, err := getKeysConfig(ctx, s) + require.NoError(t, err) + require.Equal(t, &keyConfig{DefaultKeyId: keyId}, keysConfig) + + issuersConfig, err := getIssuersConfig(ctx, s) + require.NoError(t, err) + require.Equal(t, &issuerConfig{DefaultIssuerId: issuerId}, issuersConfig) +} diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index 7b4fc2b4e5853..f8ec0e8978a4b 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "testing" - "github.com/hashicorp/go-uuid" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/helper/certutil" "github.com/hashicorp/vault/sdk/logical" @@ -28,10 +27,10 @@ func Test_ConfigsRoundTrip(t *testing.T) { // Now attempt to store and reload properly origKeyConfig := &keyConfig{ - DefaultKeyId: genKeyId(t), + DefaultKeyId: genKeyId(), } origIssuerConfig := &issuerConfig{ - DefaultIssuerId: genIssuerId(t), + DefaultIssuerId: genIssuerId(), } err = setKeysConfig(ctx, s, origKeyConfig) @@ -64,12 +63,12 @@ func Test_IssuerRoundTrip(t *testing.T) { // Now write out our issuers and keys err = writeKey(ctx, s, key1) require.NoError(t, err) - err = writeIssuer(ctx, s, issuer1) + err = writeIssuer(ctx, s, &issuer1) require.NoError(t, err) err = writeKey(ctx, s, key2) require.NoError(t, err) - err = writeIssuer(ctx, s, issuer2) + err = writeIssuer(ctx, s, &issuer2) require.NoError(t, err) fetchedKey1, err := fetchKeyById(ctx, s, key1.ID) @@ -127,7 +126,7 @@ func Test_KeysIssuerImport(t *testing.T) { require.Equal(t, issuer1_ref1.ID, issuer1_ref2.ID) require.Equal(t, key1_ref1.ID, issuer1_ref2.KeyID) - err = writeIssuer(ctx, s, issuer2) + err = writeIssuer(ctx, s, &issuer2) require.NoError(t, err) err = writeKey(ctx, s, key2) @@ -148,10 +147,9 @@ func Test_KeysIssuerImport(t *testing.T) { } func genIssuerAndKey(t *testing.T, b *backend) (issuer, key) { - certBundle, err := genCertBundle(t, b) - require.NoError(t, err) + certBundle := genCertBundle(t, b) - keyId := genKeyId(t) + keyId := genKeyId() pkiKey := key{ ID: keyId, @@ -159,7 +157,7 @@ func genIssuerAndKey(t *testing.T, b *backend) (issuer, key) { PrivateKey: certBundle.PrivateKey, } - issuerId := genIssuerId(t) + issuerId := genIssuerId() pkiIssuer := issuer{ ID: issuerId, @@ -172,19 +170,7 @@ func genIssuerAndKey(t *testing.T, b *backend) (issuer, key) { return pkiIssuer, pkiKey } -func genIssuerId(t *testing.T) issuerId { - issuerIdStr, err := uuid.GenerateUUID() - require.NoError(t, err) - return issuerId(issuerIdStr) -} - -func genKeyId(t *testing.T) keyId { - keyIdStr, err := uuid.GenerateUUID() - require.NoError(t, err) - return keyId(keyIdStr) -} - -func genCertBundle(t *testing.T, b *backend) (*certutil.CertBundle, error) { +func genCertBundle(t *testing.T, b *backend) *certutil.CertBundle { // Pretty gross just to generate a cert bundle, but fields := addCACommonFields(map[string]*framework.FieldSchema{}) fields = addCAKeyGenerationFields(fields) @@ -214,5 +200,5 @@ func genCertBundle(t *testing.T, b *backend) (*certutil.CertBundle, error) { require.NoError(t, err) certBundle, err := parsedCertBundle.ToCertBundle() require.NoError(t, err) - return certBundle, err + return certBundle } From 62467e06b4ae6cc82c9aeeb9e94f4f9b754f69a0 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 31 Mar 2022 12:17:56 -0400 Subject: [PATCH 08/76] Make fetchCAInfo aware of new storage layout This allows fetchCAInfo to fetch a specified issuer, via a reference parameter provided by the user. We pass that into the storage layer and have it return a cert bundle for us. Finally, we need to validate that it truly has the key desired. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend_test.go | 2 +- builtin/logical/pki/cert_util.go | 22 ++++++++++++---------- builtin/logical/pki/crl_util.go | 4 ++-- builtin/logical/pki/path_fetch.go | 2 +- builtin/logical/pki/path_issue_sign.go | 2 +- builtin/logical/pki/path_root.go | 4 ++-- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index bbb445eba8e0d..5c0b9b2a839c3 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -2585,7 +2585,7 @@ func TestBackend_SignSelfIssued(t *testing.T) { t.Fatal(err) } - signingBundle, err := fetchCAInfo(context.Background(), b, &logical.Request{Storage: storage}) + signingBundle, err := fetchCAInfo(context.Background(), b, &logical.Request{Storage: storage}, "default") if err != nil { t.Fatal(err) } diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index 11ac905e069ee..f11d5f2c859ee 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -91,21 +91,20 @@ func getFormat(data *framework.FieldData) string { // Fetches the CA info. Unlike other certificates, the CA info is stored // in the backend as a CertBundle, because we are storing its private key -func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request) (*certutil.CAInfoBundle, error) { - bundleEntry, err := req.Storage.Get(ctx, "config/ca_bundle") +func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRef string) (*certutil.CAInfoBundle, error) { + id, err := resolveIssuerReference(ctx, req.Storage, issuerRef) if err != nil { - return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch local CA certificate/key: %v", err)} - } - if bundleEntry == nil { - return nil, errutil.UserError{Err: "backend must be configured with a CA certificate/key"} + // Usually a bad label from the user or misconfigured default. + return nil, errutil.UserError{Err: err.Error()} } - var bundle certutil.CertBundle - if err := bundleEntry.DecodeJSON(&bundle); err != nil { - return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode local CA certificate/key: %v", err)} + bundle, err := fetchCertBundleByIssuerId(ctx, req.Storage, id, true) + if err != nil { + // Once we have an issuer id, usually a bug on our side if it isn't there. + return nil, errutil.InternalError{Err: err.Error()} } - parsedBundle, err := parseCABundle(ctx, b, req, &bundle) + parsedBundle, err := parseCABundle(ctx, b, req, bundle) if err != nil { return nil, errutil.InternalError{Err: err.Error()} } @@ -113,6 +112,9 @@ func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request) (*certut if parsedBundle.Certificate == nil { return nil, errutil.InternalError{Err: "stored CA information not able to be parsed"} } + if parsedBundle.PrivateKey == nil { + return nil, errutil.UserError{Err: fmt.Sprintf("unable to fetch corresponding key for issuer %v; unable to use this issuer for signing", issuerRef)} + } caInfo := &certutil.CAInfoBundle{ParsedCertBundle: *parsedBundle, URLs: nil} diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index 2fd0e33fd2952..27b72270eb416 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -32,7 +32,7 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st return nil, nil } - signingBundle, caErr := fetchCAInfo(ctx, b, req) + signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") if caErr != nil { switch caErr.(type) { case errutil.UserError: @@ -223,7 +223,7 @@ func buildCRL(ctx context.Context, b *backend, req *logical.Request, forceNew bo } WRITE: - signingBundle, caErr := fetchCAInfo(ctx, b, req) + signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") if caErr != nil { switch caErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index 634964e9dcb69..04098403f1b84 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -191,7 +191,7 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data } if serial == "ca_chain" { - caInfo, err := fetchCAInfo(ctx, b, req) + caInfo, err := fetchCAInfo(ctx, b, req, "default") if err != nil { switch err.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 218964f3ac779..7c9e85c350820 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -174,7 +174,7 @@ func (b *backend) pathIssueSignCert(ctx context.Context, req *logical.Request, d } var caErr error - signingBundle, caErr := fetchCAInfo(ctx, b, req) + signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") if caErr != nil { switch caErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 2edd4b181f0be..8d174f235d015 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -311,7 +311,7 @@ func (b *backend) pathCASignIntermediate(ctx context.Context, req *logical.Reque } var caErr error - signingBundle, caErr := fetchCAInfo(ctx, b, req) + signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") if caErr != nil { switch caErr.(type) { case errutil.UserError: @@ -442,7 +442,7 @@ func (b *backend) pathCASignSelfIssued(ctx context.Context, req *logical.Request } var caErr error - signingBundle, caErr := fetchCAInfo(ctx, b, req) + signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") if caErr != nil { switch caErr.(type) { case errutil.UserError: From c0c11c959ab16d98c67ec8d99ac23621aa3c2bb5 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Fri, 1 Apr 2022 10:49:15 -0400 Subject: [PATCH 09/76] Begin /issuers API endpoints This implements the fetch operations around issuers in the PKI Secrets Engine. We implement the following operations: - LIST /issuers - returns a list of known issuers' IDs and names. - GET /issuer/:ref - returns a JSON blob with information about this issuer. - POST /issuer/:ref - allows configuring information about issuers, presently just its name. - DELETE /issuer/:ref - allows deleting the specified issuer. - GET /issuer/:ref/{der,pem} - returns a raw API response with just the DER (or PEM) of the issuer's certificate. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 12 +- builtin/logical/pki/path_fetch_issuers.go | 252 ++++++++++++++++++++++ 2 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 builtin/logical/pki/path_fetch_issuers.go diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 162fcb5140d4f..35cc6879e9edb 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -105,6 +105,15 @@ func Backend(conf *logical.BackendConfig) *backend { pathSign(&b), pathIssue(&b), pathRotateCRL(&b), + pathRevoke(&b), + pathTidy(&b), + pathTidyStatus(&b), + + // Issuer APIs + pathListIssuers(&b), + pathGetIssuer(&b), + + // Fetch APIs have been lowered to favor the newer issuer API endpoints pathFetchCA(&b), pathFetchCAChain(&b), pathFetchCRL(&b), @@ -112,9 +121,6 @@ func Backend(conf *logical.BackendConfig) *backend { pathFetchValidRaw(&b), pathFetchValid(&b), pathFetchListCerts(&b), - pathRevoke(&b), - pathTidy(&b), - pathTidyStatus(&b), }, Secrets: []*framework.Secret{ diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go new file mode 100644 index 0000000000000..94f34edefa859 --- /dev/null +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -0,0 +1,252 @@ +package pki + +import ( + "context" + "encoding/pem" + "regexp" + "strings" + + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/logical" +) + +var nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex("ref") + "$") + +func pathListIssuers(b *backend) *framework.Path { + return &framework.Path{ + Pattern: "issuers/?$", + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.ListOperation: b.pathListIssuersHandler, + }, + + HelpSynopsis: pathListIssuersHelpSyn, + HelpDescription: pathListIssuersHelpDesc, + } +} + +func (b *backend) pathListIssuersHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + var responseKeys []string + responseInfo := make(map[string]interface{}) + + entries, err := listIssuers(ctx, req.Storage) + if err != nil { + return nil, err + } + + // For each issuer, we need not only the identifier (as returned by + // listIssuers), but also the name of the issuer. This means we have to + // fetch the actual issuer object as well. + for _, identifier := range entries { + issuer, err := fetchIssuerById(ctx, req.Storage, identifier) + if err != nil { + return nil, err + } + + responseKeys = append(responseKeys, string(identifier)) + responseInfo[string(identifier)] = map[string]interface{}{ + "name": issuer.Name, + } + } + + return logical.ListResponseWithInfo(responseKeys, responseInfo), nil +} + +const ( + pathListIssuersHelpSyn = `Fetch a list of CA certificates.` + pathListIssuersHelpDesc = ` +This endpoint allows listing of known issuing certificates, returning +their identifier and their name (if set). +` +) + +func pathGetIssuer(b *backend) *framework.Path { + pattern := "issuer/" + framework.GenericNameRegex("ref") + "(/der|/pem)?" + return buildPathGetIssuer(b, pattern) +} + +func buildPathGetIssuer(b *backend, pattern string) *framework.Path { + return &framework.Path{ + // Returns a JSON entry. + Pattern: pattern, + Fields: map[string]*framework.FieldSchema{ + "ref": { + Type: framework.TypeString, + Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, + Default: "default", + }, + "name": { + Type: framework.TypeString, + Description: `Human-readable name for this issuer.`, + }, + }, + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.ReadOperation: b.pathGetIssuer, + logical.UpdateOperation: b.pathUpdateIssuer, + logical.DeleteOperation: b.pathDeleteIssuer, + }, + + HelpSynopsis: pathGetIssuerHelpSyn, + HelpDescription: pathGetIssuerHelpDesc, + } +} + +func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Handle raw issuers first. + if strings.HasSuffix(req.Path, "/der") || strings.HasSuffix(req.Path, "/pem") { + return b.pathGetRawIssuer(ctx, req, data) + } + + issuerName := data.Get("ref").(string) + if len(issuerName) == 0 { + return logical.ErrorResponse("missing issuer reference"), nil + } + + ref, err := resolveIssuerReference(ctx, req.Storage, issuerName) + if err != nil { + return nil, err + } + if ref == "" { + return logical.ErrorResponse("unable to resolve issuer id for reference: " + issuerName), nil + } + + issuer, err := fetchIssuerById(ctx, req.Storage, ref) + if err != nil { + return nil, err + } + + return &logical.Response{ + Data: map[string]interface{}{ + "id": issuer.ID, + "name": issuer.Name, + "key_id": issuer.KeyID, + "certificate": issuer.Certificate, + }, + }, nil +} + +func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + issuerName := data.Get("ref").(string) + if len(issuerName) == 0 { + return logical.ErrorResponse("missing issuer reference"), nil + } + + newName := data.Get("name").(string) + if len(newName) > 0 && !nameMatcher.MatchString(newName) { + return logical.ErrorResponse("new issuer name outside of valid character limits"), nil + } + + ref, err := resolveIssuerReference(ctx, req.Storage, issuerName) + if err != nil { + return nil, err + } + if ref == "" { + return logical.ErrorResponse("unable to resolve issuer id for reference: " + issuerName), nil + } + + issuer, err := fetchIssuerById(ctx, req.Storage, ref) + if err != nil { + return nil, err + } + + if newName != issuer.Name { + issuer.Name = newName + + err := writeIssuer(ctx, req.Storage, issuer) + if err != nil { + return nil, err + } + } + + return &logical.Response{ + Data: map[string]interface{}{ + "id": issuer.ID, + "name": issuer.Name, + "key_id": issuer.KeyID, + "certificate": issuer.Certificate, + }, + }, nil +} + +func (b *backend) pathGetRawIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + issuerName := data.Get("ref").(string) + if len(issuerName) == 0 { + return logical.ErrorResponse("missing issuer reference"), nil + } + + ref, err := resolveIssuerReference(ctx, req.Storage, issuerName) + if err != nil { + return nil, err + } + if ref == "" { + return logical.ErrorResponse("unable to resolve issuer id for reference: " + issuerName), nil + } + + issuer, err := fetchIssuerById(ctx, req.Storage, ref) + if err != nil { + return nil, err + } + + certificate := []byte(issuer.Certificate) + contentType := "application/pem-certificate-chain" + + if strings.HasSuffix(req.Path, "/der") { + contentType = "application/pkix-cert" + + pemBlock, _ := pem.Decode(certificate) + if pemBlock == nil { + return nil, err + } + + certificate = pemBlock.Bytes + } + + statusCode := 200 + if len(certificate) == 0 { + statusCode = 204 + } + + return &logical.Response{ + Data: map[string]interface{}{ + logical.HTTPContentType: contentType, + logical.HTTPRawBody: certificate, + logical.HTTPStatusCode: statusCode, + }, + }, nil +} + +func (b *backend) pathDeleteIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + issuerName := data.Get("ref").(string) + if len(issuerName) == 0 { + return logical.ErrorResponse("missing issuer reference"), nil + } + + ref, err := resolveIssuerReference(ctx, req.Storage, issuerName) + if err != nil { + return nil, err + } + if ref == "" { + return logical.ErrorResponse("unable to resolve issuer id for reference: " + issuerName), nil + } + + return nil, deleteIssuer(ctx, req.Storage, ref) +} + +const ( + pathGetIssuerHelpSyn = `Fetch a single issuer certificate.` + pathGetIssuerHelpDesc = ` +This allows fetching information associated with the underlying issuer +certificate. + +:ref can be either the literal value "default", in which case /config/issuers +will be consulted for the present default issuer, an identifier of an issuer, +or its assigned name value. + +Use /issuer/:ref/der or /issuer/:ref/pem to return just the certificate in +raw DER or PEM form, without the JSON structure of /issuer/:ref. + +Writing to /issuer/:ref allows updating of the name field associated with +the certificate. +` +) From 5a4c6ea308a0c7db8b3bce367bf3cd562396154c Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Fri, 1 Apr 2022 15:49:48 -0400 Subject: [PATCH 10/76] Add import to PKI Issuers API This adds the two core import code paths to the API: /issuers/import/cert and /issuers/import/bundle. The former differs from the latter in that the latter allows the import of keys. This allows operators to restrict importing of keys to privileged roles, while allowing more operators permission to import additional certificates (not used for signing, but instead for path/chain building). Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 1 + builtin/logical/pki/path_manage_issuers.go | 124 +++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 builtin/logical/pki/path_manage_issuers.go diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 35cc6879e9edb..29589126425a7 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -112,6 +112,7 @@ func Backend(conf *logical.BackendConfig) *backend { // Issuer APIs pathListIssuers(&b), pathGetIssuer(&b), + pathImportIssuer(&b), // Fetch APIs have been lowered to favor the newer issuer API endpoints pathFetchCA(&b), diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go new file mode 100644 index 0000000000000..907f8aa8da4bb --- /dev/null +++ b/builtin/logical/pki/path_manage_issuers.go @@ -0,0 +1,124 @@ +package pki + +import ( + "bytes" + "context" + "encoding/pem" + "strings" + + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/helper/errutil" + "github.com/hashicorp/vault/sdk/logical" +) + +func pathImportIssuer(b *backend) *framework.Path { + return &framework.Path{ + Pattern: "issuers/import/(cert|bundle)", + Fields: map[string]*framework.FieldSchema{ + "pem_bundle": { + Type: framework.TypeString, + Description: `PEM-format, concatenated unencrypted +secret-key (optional) and certificates.`, + }, + }, + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.UpdateOperation: b.pathImportIssuers, + }, + + HelpSynopsis: pathImportIssuersHelpSyn, + HelpDescription: pathImportIssuersHelpDesc, + } +} + +func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + keysAllowed := strings.HasSuffix(req.Path, "bundle") + + pemBundle := data.Get("pem_bundle").(string) + if len(pemBundle) == 0 { + return logical.ErrorResponse("'pem_bundle' parameter was empty"), nil + } + + var createdKeys []keyId + var createdIssuers []issuerId + issuerKeyMap := make(map[issuerId]keyId) + + // Rather than using certutil.ParsePEMBundle (which restricts the + // construction of the PEM bundle), we manually parse the bundle instead. + pemBytes := []byte(pemBundle) + var pemBlock *pem.Block + + var issuers []string + var keys []string + + for len(bytes.TrimSpace(pemBytes)) > 0 { + pemBlock, pemBytes = pem.Decode(pemBytes) + if pemBlock == nil { + return nil, errutil.UserError{Err: "no data found in PEM block"} + } + + pemBlockString := string(pem.EncodeToMemory(pemBlock)) + + switch pemBlock.Type { + case "CERTIFICATE", "X509 CERTIFICATE": + // Must be a certificate + issuers = append(issuers, pemBlockString) + case "CRL", "X509 CRL": + // Ignore any CRL entries. + default: + // Otherwise, treat them as keys. + keys = append(keys, pemBlockString) + } + } + + if len(keys) > 0 && !keysAllowed { + return logical.ErrorResponse("private keys found in the PEM bundle but not allowed by the path; use /issuers/import/bundle"), nil + } + + for _, keyPem := range keys { + // Handle import of private key. + key, existing, err := importKey(ctx, req.Storage, keyPem) + if err != nil { + return logical.ErrorResponse(err.Error()), nil + } + + if !existing { + createdKeys = append(createdKeys, key.ID) + } + } + + for _, certPem := range issuers { + cert, existing, err := importIssuer(ctx, req.Storage, certPem) + if err != nil { + return logical.ErrorResponse(err.Error()), nil + } + + issuerKeyMap[cert.ID] = cert.KeyID + if !existing { + createdIssuers = append(createdIssuers, cert.ID) + } + } + + return &logical.Response{ + Data: map[string]interface{}{ + "mapping": issuerKeyMap, + "imported_keys": createdKeys, + "imported_issuers": createdIssuers, + }, + }, nil +} + +const ( + pathImportIssuersHelpSyn = `Import the specified issuing certificates.` + pathImportIssuersHelpDesc = ` +This endpoint allows importing the specified issuer certificates. + +:type is either the literal value "cert", to only allow importing +certificates, else "bundle" to allow importing keys as well as +certificates. + +Depending on the value of :type, the pem_bundle request parameter can +either take PEM-formatted certificates, and, if :type="bundle", unencrypted +secret-keys. +` +) From cfe2bc8a27027b842173ba35d98f954a5dfcbdd1 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 5 Apr 2022 15:29:09 -0400 Subject: [PATCH 11/76] Add /issuer/:ref/sign-intermediate endpoint This endpoint allows existing issuers to be used to sign intermediate CA certificates. In the process, we've updated the existing /root/sign-intermediate endpoint to be equivalent to a call to /issuer/default/sign-intermediate. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 1 + builtin/logical/pki/path_root.go | 58 +++-------------- builtin/logical/pki/path_sign_issuers.go | 79 ++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 51 deletions(-) create mode 100644 builtin/logical/pki/path_sign_issuers.go diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 29589126425a7..44f6b3c926980 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -113,6 +113,7 @@ func Backend(conf *logical.BackendConfig) *backend { pathListIssuers(&b), pathGetIssuer(&b), pathImportIssuer(&b), + pathIssuerSignIntermediate(&b), // Fetch APIs have been lowered to favor the newer issuer API endpoints pathFetchCA(&b), diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 8d174f235d015..5e9e32471dfef 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -68,47 +68,6 @@ func pathDeleteRoot(b *backend) *framework.Path { return ret } -func pathSignIntermediate(b *backend) *framework.Path { - ret := &framework.Path{ - Pattern: "root/sign-intermediate", - Operations: map[logical.Operation]framework.OperationHandler{ - logical.UpdateOperation: &framework.PathOperation{ - Callback: b.pathCASignIntermediate, - }, - }, - - HelpSynopsis: pathSignIntermediateHelpSyn, - HelpDescription: pathSignIntermediateHelpDesc, - } - - ret.Fields = addCACommonFields(map[string]*framework.FieldSchema{}) - ret.Fields = addCAIssueFields(ret.Fields) - - ret.Fields["csr"] = &framework.FieldSchema{ - Type: framework.TypeString, - Default: "", - Description: `PEM-format CSR to be signed.`, - } - - ret.Fields["use_csr_values"] = &framework.FieldSchema{ - Type: framework.TypeBool, - Default: false, - Description: `If true, then: -1) Subject information, including names and alternate -names, will be preserved from the CSR rather than -using values provided in the other parameters to -this path; -2) Any key usages requested in the CSR will be -added to the basic set of key usages used for CA -certs signed by this path; for instance, -the non-repudiation flag; -3) Extensions requested in the CSR will be copied -into the issued certificate.`, - } - - return ret -} - func pathSignSelfIssued(b *backend) *framework.Path { ret := &framework.Path{ Pattern: "root/sign-self-issued", @@ -273,9 +232,14 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, return resp, nil } -func (b *backend) pathCASignIntermediate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathIssuerSignIntermediate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { var err error + issuerName := data.Get("ref").(string) + if len(issuerName) == 0 { + return logical.ErrorResponse("missing issuer reference"), nil + } + format := getFormat(data) if format == "" { return logical.ErrorResponse( @@ -311,7 +275,7 @@ func (b *backend) pathCASignIntermediate(ctx context.Context, req *logical.Reque } var caErr error - signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") + signingBundle, caErr := fetchCAInfo(ctx, b, req, issuerName) if caErr != nil { switch caErr.(type) { case errutil.UserError: @@ -552,14 +516,6 @@ const pathDeleteRootHelpDesc = ` See the API documentation for more information. ` -const pathSignIntermediateHelpSyn = ` -Issue an intermediate CA certificate based on the provided CSR. -` - -const pathSignIntermediateHelpDesc = ` -see the API documentation for more information. -` - const pathSignSelfIssuedHelpSyn = ` Signs another CA's self-issued certificate. ` diff --git a/builtin/logical/pki/path_sign_issuers.go b/builtin/logical/pki/path_sign_issuers.go new file mode 100644 index 0000000000000..40f0ff83df508 --- /dev/null +++ b/builtin/logical/pki/path_sign_issuers.go @@ -0,0 +1,79 @@ +package pki + +import ( + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/logical" +) + +func pathIssuerSignIntermediate(b *backend) *framework.Path { + pattern := "issuers/" + framework.GenericNameRegex("ref") + "/sign-intermediate" + return pathIssuerSignIntermediateRaw(b, pattern) +} + +func pathSignIntermediate(b *backend) *framework.Path { + pattern := "root/sign-intermediate" + return pathIssuerSignIntermediateRaw(b, pattern) +} + +func pathIssuerSignIntermediateRaw(b *backend, pattern string) *framework.Path { + path := &framework.Path{ + Pattern: pattern, + Fields: map[string]*framework.FieldSchema{ + "ref": { + Type: framework.TypeString, + Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, + Default: "default", + }, + }, + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.UpdateOperation: b.pathIssuerSignIntermediate, + }, + + HelpSynopsis: pathIssuerSignIntermediateHelpSyn, + HelpDescription: pathIssuerSignIntermediateHelpDesc, + } + + path.Fields = addCACommonFields(path.Fields) + path.Fields = addCAIssueFields(path.Fields) + + path.Fields["csr"] = &framework.FieldSchema{ + Type: framework.TypeString, + Default: "", + Description: `PEM-format CSR to be signed.`, + } + + path.Fields["use_csr_values"] = &framework.FieldSchema{ + Type: framework.TypeBool, + Default: false, + Description: `If true, then: +1) Subject information, including names and alternate +names, will be preserved from the CSR rather than +using values provided in the other parameters to +this path; +2) Any key usages requested in the CSR will be +added to the basic set of key usages used for CA +certs signed by this path; for instance, +the non-repudiation flag; +3) Extensions requested in the CSR will be copied +into the issued certificate.`, + } + + return path +} + +const ( + pathIssuerSignIntermediateHelpSyn = `Issue an intermediate CA certificate based on the provided CSR.` + pathIssuerSignIntermediateHelpDesc = ` +This API endpoint allows for signing the specified CSR, adding to it a basic +constraint for IsCA=True. This allows the issued certificate to issue its own +leaf certificates. + +Note that the resulting certificate is not imported as an issuer in this PKI +mount. This means that you can use the resulting certificate in another Vault +PKI mount point or to issue an external intermediate (e.g., for use with +another X.509 CA). + +See the API documentation for more information about required parameters. +` +) From 1c589ebb5121aa4edd1e4f0525a7540f06c88edb Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 5 Apr 2022 15:45:09 -0400 Subject: [PATCH 12/76] Add /issuer/:ref/sign-self-issued endpoint This endpoint allows existing issuers to be used to sign self-signed certificates. In the process, we've updated the existing /root/sign-self-issued endpoint to be equivalent to a call to /issuer/default/sign-self-issued. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 1 + builtin/logical/pki/path_root.go | 49 +++--------------- builtin/logical/pki/path_sign_issuers.go | 65 ++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 42 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 44f6b3c926980..707d8b53800a2 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -114,6 +114,7 @@ func Backend(conf *logical.BackendConfig) *backend { pathGetIssuer(&b), pathImportIssuer(&b), pathIssuerSignIntermediate(&b), + pathIssuerSignSelfIssued(&b), // Fetch APIs have been lowered to favor the newer issuer API endpoints pathFetchCA(&b), diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 5e9e32471dfef..75d1dfda002f6 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -68,34 +68,6 @@ func pathDeleteRoot(b *backend) *framework.Path { return ret } -func pathSignSelfIssued(b *backend) *framework.Path { - ret := &framework.Path{ - Pattern: "root/sign-self-issued", - Operations: map[logical.Operation]framework.OperationHandler{ - logical.UpdateOperation: &framework.PathOperation{ - Callback: b.pathCASignSelfIssued, - }, - }, - - Fields: map[string]*framework.FieldSchema{ - "certificate": { - Type: framework.TypeString, - Description: `PEM-format self-issued certificate to be signed.`, - }, - "require_matching_certificate_algorithms": { - Type: framework.TypeBool, - Default: false, - Description: `If true, require the public key algorithm of the signer to match that of the self issued certificate.`, - }, - }, - - HelpSynopsis: pathSignSelfIssuedHelpSyn, - HelpDescription: pathSignSelfIssuedHelpDesc, - } - - return ret -} - func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { return nil, req.Storage.Delete(ctx, "config/ca_bundle") } @@ -381,9 +353,14 @@ func (b *backend) pathIssuerSignIntermediate(ctx context.Context, req *logical.R return resp, nil } -func (b *backend) pathCASignSelfIssued(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathIssuerSignSelfIssued(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { var err error + issuerName := data.Get("ref").(string) + if len(issuerName) == 0 { + return logical.ErrorResponse("missing issuer reference"), nil + } + certPem := data.Get("certificate").(string) block, _ := pem.Decode([]byte(certPem)) if block == nil || len(block.Bytes) == 0 { @@ -406,7 +383,7 @@ func (b *backend) pathCASignSelfIssued(ctx context.Context, req *logical.Request } var caErr error - signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") + signingBundle, caErr := fetchCAInfo(ctx, b, req, issuerName) if caErr != nil { switch caErr.(type) { case errutil.UserError: @@ -515,15 +492,3 @@ Deletes the root CA key to allow a new one to be generated. const pathDeleteRootHelpDesc = ` See the API documentation for more information. ` - -const pathSignSelfIssuedHelpSyn = ` -Signs another CA's self-issued certificate. -` - -const pathSignSelfIssuedHelpDesc = ` -Signs another CA's self-issued certificate. This is most often used for rolling roots; unless you know you need this you probably want to use sign-intermediate instead. - -Note that this is a very privileged operation and should be extremely restricted in terms of who is allowed to use it. All values will be taken directly from the incoming certificate and only verification that it is self-issued will be performed. - -Configured URLs for CRLs/OCSP/etc. will be copied over and the issuer will be this mount's CA cert. Other than that, all other values will be used verbatim. -` diff --git a/builtin/logical/pki/path_sign_issuers.go b/builtin/logical/pki/path_sign_issuers.go index 40f0ff83df508..c073bce48f2e2 100644 --- a/builtin/logical/pki/path_sign_issuers.go +++ b/builtin/logical/pki/path_sign_issuers.go @@ -77,3 +77,68 @@ another X.509 CA). See the API documentation for more information about required parameters. ` ) + +func pathIssuerSignSelfIssued(b *backend) *framework.Path { + pattern := "issuers/" + framework.GenericNameRegex("ref") + "/sign-self-issued" + return buildPathIssuerSignSelfIssued(b, pattern) +} + +func pathSignSelfIssued(b *backend) *framework.Path { + pattern := "root/sign-self-issued" + return buildPathIssuerSignSelfIssued(b, pattern) +} + +func buildPathIssuerSignSelfIssued(b *backend, pattern string) *framework.Path { + path := &framework.Path{ + Pattern: pattern, + Fields: map[string]*framework.FieldSchema{ + "ref": { + Type: framework.TypeString, + Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, + Default: "default", + }, + "certificate": { + Type: framework.TypeString, + Description: `PEM-format self-issued certificate to be signed.`, + }, + "require_matching_certificate_algorithms": { + Type: framework.TypeBool, + Default: false, + Description: `If true, require the public key algorithm of the signer to match that of the self issued certificate.`, + }, + }, + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.UpdateOperation: b.pathIssuerSignSelfIssued, + }, + + HelpSynopsis: pathIssuerSignSelfIssuedHelpSyn, + HelpDescription: pathIssuerSignSelfIssuedHelpDesc, + } + + return path +} + +const ( + pathIssuerSignSelfIssuedHelpSyn = `Re-issue a self-signed certificate based on the provided certificate.` + pathIssuerSignSelfIssuedHelpDesc = ` +This API endpoint allows for signing the specified self-signed certificate, +effectively allowing cross-signing of external root CAs. This allows for an +alternative validation path, chaining back through this PKI mount. This +endpoint is also useful in a rolling-root scenario, allowing devices to trust +and validate later (or earlier) root certificates and their issued leaves. + +Usually the sign-intermediate operation is preferred to this operation. + +Note that this is a very privileged operation and should be extremely +restricted in terms of who is allowed to use it. All values will be taken +directly from the incoming certificate and only verification that it is +self-issued will be performed. + +Configured URLs for CRLs/OCSP/etc. will be copied over and the issuer will +be this mount's CA cert. Other than that, all other values will be used +verbatim from the given certificate. + +See the API documentation for more information about required parameters. +` +) From 2de0e9cd0c1b9e135a1af0b0d0aae54424b0c14d Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 5 Apr 2022 16:15:40 -0400 Subject: [PATCH 13/76] Add /issuer/:ref/sign-verbatim endpoint This endpoint allows existing issuers to be used to directly sign CSRs. In the process, we've updated the existing /sign-verbatim endpoint to be equivalent to a call to /issuer/:ref/sign-verbatim. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 1 + builtin/logical/pki/fields.go | 6 ++++ builtin/logical/pki/path_issue_sign.go | 46 +++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 707d8b53800a2..cada6802b49e1 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -115,6 +115,7 @@ func Backend(conf *logical.BackendConfig) *backend { pathImportIssuer(&b), pathIssuerSignIntermediate(&b), pathIssuerSignSelfIssued(&b), + pathIssuerSignVerbatim(&b), // Fetch APIs have been lowered to favor the newer issuer API endpoints pathFetchCA(&b), diff --git a/builtin/logical/pki/fields.go b/builtin/logical/pki/fields.go index aafae04dd498b..4593d6d9a7ccb 100644 --- a/builtin/logical/pki/fields.go +++ b/builtin/logical/pki/fields.go @@ -132,6 +132,12 @@ be larger than the role max TTL.`, The value format should be given in UTC format YYYY-MM-ddTHH:MM:SSZ`, } + fields["ref"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, + Default: "default", + } + return fields } diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 7c9e85c350820..11590b5b2222d 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -53,19 +53,30 @@ func pathSign(b *backend) *framework.Path { return ret } +func pathIssuerSignVerbatim(b *backend) *framework.Path { + pattern := "issuers/" + framework.GenericNameRegex("ref") + "/sign-verbatim" + return buildPathIssuerSignVerbatim(b, pattern) +} + func pathSignVerbatim(b *backend) *framework.Path { + pattern := "root/sign-verbatim" + return buildPathIssuerSignVerbatim(b, pattern) +} + +func buildPathIssuerSignVerbatim(b *backend, pattern string) *framework.Path { ret := &framework.Path{ - Pattern: "sign-verbatim" + framework.OptionalParamRegex("role"), + Pattern: pattern, + Fields: map[string]*framework.FieldSchema{}, Callbacks: map[logical.Operation]framework.OperationFunc{ logical.UpdateOperation: b.metricsWrap("sign-verbatim", roleOptional, b.pathSignVerbatim), }, - HelpSynopsis: pathSignHelpSyn, - HelpDescription: pathSignHelpDesc, + HelpSynopsis: pathIssuerSignVerbatimHelpSyn, + HelpDescription: pathIssuerSignVerbatimHelpDesc, } - ret.Fields = addNonCACommonFields(map[string]*framework.FieldSchema{}) + ret.Fields = addNonCACommonFields(ret.Fields) ret.Fields["csr"] = &framework.FieldSchema{ Type: framework.TypeString, @@ -104,6 +115,26 @@ this value to an empty list.`, return ret } +const ( + pathIssuerSignVerbatimHelpSyn = `Issue a certificate directly based on the provided CSR.` + pathIssuerSignVerbatimHelpDesc = ` +This API endpoint allows for directly signing the specified certificate +signing request (CSR) without the typical role-based validation. This +allows for attributes from the CSR to be directly copied to the resulting +certificate. + +Usually the role-based sign operations (/sign and /issue) are preferred to +this operation. + +Note that this is a very privileged operation and should be extremely +restricted in terms of who is allowed to use it. All values will be taken +directly from the incoming CSR. No further verification of attribute are +performed, except as permitted by this endpoint's parameters. + +See the API documentation for more information about required parameters. +` +) + // pathIssue issues a certificate and private key from given parameters, // subject to role restrictions func (b *backend) pathIssue(ctx context.Context, req *logical.Request, data *framework.FieldData, role *roleEntry) (*logical.Response, error) { @@ -167,6 +198,11 @@ func (b *backend) pathIssueSignCert(ctx context.Context, req *logical.Request, d return nil, logical.ErrReadOnly } + issuerName := data.Get("ref").(string) + if len(issuerName) == 0 { + return logical.ErrorResponse("missing issuer reference"), nil + } + format := getFormat(data) if format == "" { return logical.ErrorResponse( @@ -174,7 +210,7 @@ func (b *backend) pathIssueSignCert(ctx context.Context, req *logical.Request, d } var caErr error - signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") + signingBundle, caErr := fetchCAInfo(ctx, b, req, issuerName) if caErr != nil { switch caErr.(type) { case errutil.UserError: From 51ebbbdf9b2ee554df2d75cdbd66395a34d9b2b9 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Wed, 6 Apr 2022 09:38:00 -0400 Subject: [PATCH 14/76] Allow configuration of default issuers Using the new updateDefaultIssuerId(...) from the storage migration PR allows for easy implementation of configuring the default issuer. We restrict callers from setting blank defaults and setting default to default. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 1 + builtin/logical/pki/path_config_ca.go | 66 +++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index cada6802b49e1..6ab418f03bbf0 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -116,6 +116,7 @@ func Backend(conf *logical.BackendConfig) *backend { pathIssuerSignIntermediate(&b), pathIssuerSignSelfIssued(&b), pathIssuerSignVerbatim(&b), + pathConfigIssuers(&b), // Fetch APIs have been lowered to favor the newer issuer API endpoints pathFetchCA(&b), diff --git a/builtin/logical/pki/path_config_ca.go b/builtin/logical/pki/path_config_ca.go index cc6b6383f9b7e..d16137ea3c5bd 100644 --- a/builtin/logical/pki/path_config_ca.go +++ b/builtin/logical/pki/path_config_ca.go @@ -103,6 +103,72 @@ secret key and certificate. For security reasons, the secret key cannot be retrieved later. ` +func pathConfigIssuers(b *backend) *framework.Path { + return &framework.Path{ + Pattern: "config/issuers", + Fields: map[string]*framework.FieldSchema{ + "default": { + Type: framework.TypeString, + Description: `Reference (name or identifier) to the default issuer.`, + }, + }, + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.ReadOperation: b.pathCAIssuersRead, + logical.UpdateOperation: b.pathCAIssuersWrite, + }, + + HelpSynopsis: pathConfigIssuersHelpSyn, + HelpDescription: pathConfigIssuersHelpDesc, + } +} + +func (b *backend) pathCAIssuersRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + config, err := getIssuersConfig(ctx, req.Storage) + if err != nil { + return logical.ErrorResponse("Error loading issuers configuration: " + err.Error()), nil + } + + return &logical.Response{ + Data: map[string]interface{}{ + "default": config.DefaultIssuerId, + }, + }, nil +} + +func (b *backend) pathCAIssuersWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + newDefault := data.Get("default").(string) + if len(newDefault) == 0 || newDefault == "default" { + return logical.ErrorResponse("Invalid issuer specification; must be non-empty and can't be 'default'."), nil + } + + parsedIssuer, err := resolveIssuerReference(ctx, req.Storage, newDefault) + if err != nil { + return logical.ErrorResponse("Error resolving issuer reference: " + err.Error()), nil + } + + err = updateDefaultIssuerId(ctx, req.Storage, parsedIssuer) + if err != nil { + return logical.ErrorResponse("Error updating issuer configuration: " + err.Error()), nil + } + + return &logical.Response{ + Data: map[string]interface{}{ + "default": parsedIssuer, + }, + }, nil +} + +const pathConfigIssuersHelpSyn = `Read and set the default issuer certificate for signing.` + +const pathConfigIssuersHelpDesc = ` +This path allows configuration of issuer parameters. + +Presently, the "default" parameter controls which issuer is the default, +accessible by the existing signing paths (/root/sign-intermediate, +/root/sign-self-issued, /sign-verbatim, /sign/:role, and /issue/:role). +` + const pathConfigCAGenerateHelpSyn = ` Generate a new CA certificate and private key used for signing. ` From 81498a756670c05703414dfdbaad641f0901781e Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Wed, 6 Apr 2022 10:53:50 -0400 Subject: [PATCH 15/76] Fix fetching default issuers After setting a default issuer, one should be able to use the old /ca, /ca_chain, and /cert/{ca,ca_chain} endpoints to fetch the default issuer (and its chain). Update the fetchCertBySerial helper to no longer support fetching the ca and prefer fetchCAInfo for that instead (as we've already updated that to support fetching the new issuer location). Signed-off-by: Alexander Scheel --- builtin/logical/pki/cert_util.go | 9 +++--- builtin/logical/pki/path_fetch.go | 54 ++++++++++++++++++++----------- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index f11d5f2c859ee..c49136c8de194 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -135,7 +135,10 @@ func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRe } // Allows fetching certificates from the backend; it handles the slightly -// separate pathing for CA, CRL, and revoked certificates. +// separate pathing for CRL, and revoked certificates. +// +// Support for fetching CA certificates was removed, due to the new issuers +// changes. func fetchCertBySerial(ctx context.Context, req *logical.Request, prefix, serial string) (*logical.StorageEntry, error) { var path, legacyPath string var err error @@ -145,13 +148,11 @@ func fetchCertBySerial(ctx context.Context, req *logical.Request, prefix, serial colonSerial := strings.Replace(strings.ToLower(serial), "-", ":", -1) switch { - // Revoked goes first as otherwise ca/crl get hardcoded paths which fail if + // Revoked goes first as otherwise crl get hardcoded paths which fail if // we actually want revocation info case strings.HasPrefix(prefix, "revoked/"): legacyPath = "revoked/" + colonSerial path = "revoked/" + hyphenSerial - case serial == "ca": - path = "ca" case serial == "crl": path = "crl" default: diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index 04098403f1b84..7735f178699f2 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -190,7 +190,8 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data goto reply } - if serial == "ca_chain" { + // Prefer fetchCAInfo to fetchCertBySerial for CA certificates. + if serial == "ca_chain" || serial == "ca" { caInfo, err := fetchCAInfo(ctx, b, req, "default") if err != nil { switch err.(type) { @@ -203,27 +204,42 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data } } - caChain := caInfo.GetCAChain() - var certStr string - for _, ca := range caChain { - block := pem.Block{ - Type: "CERTIFICATE", - Bytes: ca.Bytes, + if serial == "ca_chain" { + caChain := caInfo.GetCAChain() + var certStr string + for _, ca := range caChain { + block := pem.Block{ + Type: "CERTIFICATE", + Bytes: ca.Bytes, + } + certStr = strings.Join([]string{certStr, strings.TrimSpace(string(pem.EncodeToMemory(&block)))}, "\n") } - certStr = strings.Join([]string{certStr, strings.TrimSpace(string(pem.EncodeToMemory(&block)))}, "\n") - } - certificate = []byte(strings.TrimSpace(certStr)) - - rawChain := caInfo.GetFullChain() - var chainStr string - for _, ca := range rawChain { - block := pem.Block{ - Type: "CERTIFICATE", - Bytes: ca.Bytes, + certificate = []byte(strings.TrimSpace(certStr)) + + rawChain := caInfo.GetFullChain() + var chainStr string + for _, ca := range rawChain { + block := pem.Block{ + Type: "CERTIFICATE", + Bytes: ca.Bytes, + } + chainStr = strings.Join([]string{chainStr, strings.TrimSpace(string(pem.EncodeToMemory(&block)))}, "\n") + } + fullChain = []byte(strings.TrimSpace(chainStr)) + } else if serial == "ca" { + certificate = caInfo.Certificate.Raw + + if len(pemType) != 0 { + block := pem.Block{ + Type: pemType, + Bytes: certificate, + } + + // This is convoluted on purpose to ensure that we don't have trailing + // newlines via various paths + certificate = []byte(strings.TrimSpace(string(pem.EncodeToMemory(&block)))) } - chainStr = strings.Join([]string{chainStr, strings.TrimSpace(string(pem.EncodeToMemory(&block)))}, "\n") } - fullChain = []byte(strings.TrimSpace(chainStr)) goto reply } From b35f2e62ec4bf6e3161d0055a42d8832e7ed5eb2 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Wed, 6 Apr 2022 11:20:08 -0400 Subject: [PATCH 16/76] Add /issuer/:ref/{sign,issue}/:role This updates the /sign and /issue endpoints, allowing them to take the default issuer (if none is provided by a role) and adding issuer-specific versions of them. Note that at this point in time, the behavior isn't yet ideal (as /sign/:role allows adding the ref=... parameter to override the default issuer); a later change adding role-based issuer specification will fix this incorrect behavior. Signed-off-by: Alexander Scheel --- builtin/logical/pki/path_issue_sign.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 11590b5b2222d..1a485189ecf05 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -15,8 +15,18 @@ import ( ) func pathIssue(b *backend) *framework.Path { + pattern := "issue/" + framework.GenericNameRegex("role") + return buildPathIssue(b, pattern) +} + +func pathIssuerIssue(b *backend) *framework.Path { + pattern := "issuer/" + framework.GenericNameRegex("ref") + "/issue/" + framework.GenericNameRegex("role") + return buildPathIssue(b, pattern) +} + +func buildPathIssue(b *backend, pattern string) *framework.Path { ret := &framework.Path{ - Pattern: "issue/" + framework.GenericNameRegex("role"), + Pattern: pattern, Callbacks: map[logical.Operation]framework.OperationFunc{ logical.UpdateOperation: b.metricsWrap("issue", roleRequired, b.pathIssue), @@ -31,8 +41,18 @@ func pathIssue(b *backend) *framework.Path { } func pathSign(b *backend) *framework.Path { + pattern := "sign/" + framework.GenericNameRegex("role") + return buildPathSign(b, pattern) +} + +func pathIssuerSign(b *backend) *framework.Path { + pattern := "issuer/" + framework.GenericNameRegex("ref") + "/sign/" + framework.GenericNameRegex("role") + return buildPathSign(b, pattern) +} + +func buildPathSign(b *backend, pattern string) *framework.Path { ret := &framework.Path{ - Pattern: "sign/" + framework.GenericNameRegex("role"), + Pattern: pattern, Callbacks: map[logical.Operation]framework.OperationFunc{ logical.UpdateOperation: b.metricsWrap("sign", roleRequired, b.pathSign), From 375005567d416da012221bca31e83ae3598a970a Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Fri, 8 Apr 2022 12:51:59 -0400 Subject: [PATCH 17/76] Add support root issuer generation --- builtin/logical/pki/backend.go | 1 + builtin/logical/pki/backend_test.go | 95 +++++++++++ builtin/logical/pki/ca_util.go | 189 ++++++++++++++++----- builtin/logical/pki/config_util.go | 19 +++ builtin/logical/pki/fields.go | 12 ++ builtin/logical/pki/path_manage_issuers.go | 26 ++- builtin/logical/pki/path_root.go | 21 ++- builtin/logical/pki/storage.go | 53 +++++- builtin/logical/pki/storage_migrations.go | 33 +--- builtin/logical/pki/storage_test.go | 6 +- builtin/logical/pki/util.go | 31 +++- 11 files changed, 397 insertions(+), 89 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 6ab418f03bbf0..98257e0c51ebc 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -116,6 +116,7 @@ func Backend(conf *logical.BackendConfig) *backend { pathIssuerSignIntermediate(&b), pathIssuerSignSelfIssued(&b), pathIssuerSignVerbatim(&b), + pathIssuerGenerateRoot(&b), pathConfigIssuers(&b), // Fetch APIs have been lowered to favor the newer issuer API endpoints diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index 5c0b9b2a839c3..0952a04059af1 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -4637,6 +4637,101 @@ func TestBackend_Roles_KeySizeRegression(t *testing.T) { t.Log(fmt.Sprintf("Key size regression expanded matrix test scenarios: %d", tested)) } +func TestRootWithExistingKey(t *testing.T) { + coreConfig := &vault.CoreConfig{ + LogicalBackends: map[string]logical.Factory{ + "pki": Factory, + }, + } + cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + defer cluster.Cleanup() + + client := cluster.Cores[0].Client + var err error + + mountPKIEndpoint(t, client, "pki-root") + + // Fail requests if type is existing, and we specify the key_type param + ctx := context.Background() + _, err = client.Logical().WriteWithContext(ctx, "pki-root/root/generate/existing", map[string]interface{}{ + "common_name": "root myvault.com", + "key_type": "rsa", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "key_type nor key_bits arguments can be set in this mode") + + // Fail requests if type is existing, and we specify the key_bits param + _, err = client.Logical().WriteWithContext(ctx, "pki-root/root/generate/existing", map[string]interface{}{ + "common_name": "root myvault.com", + "key_bits": "2048", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "key_type nor key_bits arguments can be set in this mode") + + // Fail if the specified key does not exist. + _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/existing", map[string]interface{}{ + "common_name": "root myvault.com", + "id": "my-issuer1", + "key_id": "my-key1", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "unable to find PKI key for reference: my-key1") + + // Create the first CA + resp, err := client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ + "common_name": "root myvault.com", + "key_type": "rsa", + "id": "my-issuer1", + }) + require.NoError(t, err) + require.NotNil(t, resp.Data["certificate"]) + myIssuerId1 := resp.Data["id"] + myKeyId1 := resp.Data["key_id"] + require.NotEmpty(t, myIssuerId1) + require.NotEmpty(t, myKeyId1) + + // Create the second CA + resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ + "common_name": "root myvault.com", + "key_type": "rsa", + "id": "my-issuer2", + }) + require.NoError(t, err) + require.NotNil(t, resp.Data["certificate"]) + myIssuerId2 := resp.Data["id"] + myKeyId2 := resp.Data["key_id"] + require.NotEmpty(t, myIssuerId2) + require.NotEmpty(t, myKeyId2) + + // Create a third CA re-using key from CA 1 + resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/existing", map[string]interface{}{ + "common_name": "root myvault.com", + "id": "my-issuer3", + "key_id": myKeyId1, + }) + require.NoError(t, err) + require.NotNil(t, resp.Data["certificate"]) + myIssuerId3 := resp.Data["id"] + myKeyId3 := resp.Data["key_id"] + require.NotEmpty(t, myIssuerId3) + require.NotEmpty(t, myKeyId3) + + require.NotEqual(t, myIssuerId1, myIssuerId2) + require.NotEqual(t, myIssuerId1, myIssuerId3) + require.NotEqual(t, myKeyId1, myKeyId2) + require.Equal(t, myKeyId1, myKeyId3) + + resp, err = client.Logical().ListWithContext(ctx, "pki-root/issuers") + require.NoError(t, err) + require.Equal(t, 3, len(resp.Data["keys"].([]interface{}))) + require.Contains(t, resp.Data["keys"], myIssuerId1) + require.Contains(t, resp.Data["keys"], myIssuerId2) + require.Contains(t, resp.Data["keys"], myIssuerId3) +} + var ( initTest sync.Once rsaCAKey string diff --git a/builtin/logical/pki/ca_util.go b/builtin/logical/pki/ca_util.go index d1965b7e15e3c..0cd8cbba23a87 100644 --- a/builtin/logical/pki/ca_util.go +++ b/builtin/logical/pki/ca_util.go @@ -2,8 +2,11 @@ package pki import ( "context" + "crypto" "crypto/ecdsa" "crypto/rsa" + "encoding/pem" + "errors" "fmt" "io" "time" @@ -15,18 +18,17 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -func (b *backend) getGenerationParams(ctx context.Context, - data *framework.FieldData, mountPoint string, -) (exported bool, format string, role *roleEntry, errorResp *logical.Response) { +func (b *backend) getGenerationParams(ctx context.Context, data *framework.FieldData, mountPoint string) (exported bool, format string, role *roleEntry, errorResp *logical.Response) { exportedStr := data.Get("exported").(string) switch exportedStr { case "exported": exported = true case "internal": + case "existing": case "kms": default: errorResp = logical.ErrorResponse( - `the "exported" path parameter must be "internal", "exported" or "kms"`) + `the "exported" path parameter must be "internal", "existing", exported" or "kms"`) return } @@ -37,46 +39,10 @@ func (b *backend) getGenerationParams(ctx context.Context, return } - keyType := data.Get("key_type").(string) - keyBits := data.Get("key_bits").(int) - if exportedStr == "kms" { - _, okKeyType := data.Raw["key_type"] - _, okKeyBits := data.Raw["key_bits"] - - if okKeyType || okKeyBits { - errorResp = logical.ErrorResponse( - `invalid parameter for the kms path parameter, key_type nor key_bits arguments can be set in this mode`) - return - } - - keyId, err := getManagedKeyId(data) - if err != nil { - errorResp = logical.ErrorResponse("unable to determine managed key id") - return - } - // Determine key type and key bits from the managed public key - err = withManagedPKIKey(ctx, b, keyId, mountPoint, func(ctx context.Context, key logical.ManagedSigningKey) error { - pubKey, err := key.GetPublicKey(ctx) - if err != nil { - return err - } - switch pubKey.(type) { - case *rsa.PublicKey: - keyType = "rsa" - keyBits = pubKey.(*rsa.PublicKey).Size() * 8 - case *ecdsa.PublicKey: - keyType = "ec" - case *ed25519.PublicKey: - keyType = "ed25519" - default: - return fmt.Errorf("unsupported public key: %#v", pubKey) - } - return nil - }) - if err != nil { - errorResp = logical.ErrorResponse("failed to lookup public key from managed key: %s", err.Error()) - return - } + keyType, keyBits, err := getKeyTypeAndBitsForRole(ctx, b, data, mountPoint) + if err != nil { + errorResp = logical.ErrorResponse(err.Error()) + return } role = &roleEntry{ @@ -102,7 +68,6 @@ func (b *backend) getGenerationParams(ctx context.Context, } *role.AllowWildcardCertificates = true - var err error if role.KeyBits, role.SignatureBits, err = certutil.ValidateDefaultOrValueKeyTypeSignatureLength(role.KeyType, role.KeyBits, role.SignatureBits); err != nil { errorResp = logical.ErrorResponse(err.Error()) } @@ -115,6 +80,14 @@ func generateCABundle(ctx context.Context, b *backend, input *inputBundle, data return generateManagedKeyCABundle(ctx, b, input, data, randomSource) } + if existingKeyRequested(input) { + keyRef, err := getExistingKeyRef(input.apiData) + if err != nil { + return nil, err + } + return certutil.CreateCertificateWithKeyGenerator(data, randomSource, existingGeneratePrivateKey(ctx, input.req.Storage, keyRef)) + } + return certutil.CreateCertificateWithRandomSource(data, randomSource) } @@ -132,3 +105,129 @@ func parseCABundle(ctx context.Context, b *backend, req *logical.Request, bundle } return bundle.ToParsedCertBundle() } + +func getKeyTypeAndBitsForRole(ctx context.Context, b *backend, data *framework.FieldData, mountPoint string) (string, int, error) { + exportedStr := data.Get("exported").(string) + var keyType string + var keyBits int + + switch exportedStr { + case "internal": + fallthrough + case "exported": + keyType = data.Get("key_type").(string) + keyBits = data.Get("key_bits").(int) + return keyType, keyBits, nil + } + + // existing and kms types don't support providing the key_type and key_bits args. + _, okKeyType := data.Raw["key_type"] + _, okKeyBits := data.Raw["key_bits"] + + if okKeyType || okKeyBits { + return "", 0, errors.New("invalid parameter for the kms/existing path parameter, key_type nor key_bits arguments can be set in this mode") + } + + var pubKey crypto.PublicKey + if kmsRequestedFromFieldData(data) { + pubKeyManagedKey, err := getManagedKeyPublicKey(ctx, b, data, mountPoint) + if err != nil { + return "", 0, errors.New("failed to lookup public key from managed key: " + err.Error()) + } + pubKey = pubKeyManagedKey + } + + if existingKeyRequestedFromFieldData(data) { + existingPubKey, err := getExistingPublicKey(ctx, b.storage, data) + if err != nil { + return "", 0, errors.New("failed to lookup public key from existing key: " + err.Error()) + } + pubKey = existingPubKey + } + + return getKeyTypeAndBitsFromPublicKeyForRole(pubKey) +} + +func getExistingPublicKey(ctx context.Context, s logical.Storage, data *framework.FieldData) (crypto.PublicKey, error) { + keyRef, err := getExistingKeyRef(data) + if err != nil { + return nil, err + } + id, err := resolveKeyReference(ctx, s, keyRef) + if err != nil { + return nil, err + } + key, err := fetchKeyById(ctx, s, id) + if err != nil { + return nil, err + } + signer, err := key.GetSigner() + if err != nil { + return nil, err + } + return signer.Public(), nil +} + +func getKeyTypeAndBitsFromPublicKeyForRole(pubKey crypto.PublicKey) (string, int, error) { + var keyType string + var keyBits int + + switch pubKey.(type) { + case *rsa.PublicKey: + keyType = "rsa" + keyBits = certutil.GetPublicKeySize(pubKey) + case *ecdsa.PublicKey: + keyType = "ec" + case *ed25519.PublicKey: + keyType = "ed25519" + default: + return "", 0, fmt.Errorf("unsupported public key: %#v", pubKey) + } + return keyType, keyBits, nil +} + +func getManagedKeyPublicKey(ctx context.Context, b *backend, data *framework.FieldData, mountPoint string) (crypto.PublicKey, error) { + keyId, err := getManagedKeyId(data) + if err != nil { + return nil, errors.New("unable to determine managed key id") + } + // Determine key type and key bits from the managed public key + var pubKey crypto.PublicKey + err = withManagedPKIKey(ctx, b, keyId, mountPoint, func(ctx context.Context, key logical.ManagedSigningKey) error { + pubKey, err = key.GetPublicKey(ctx) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, errors.New("failed to lookup public key from managed key: " + err.Error()) + } + return pubKey, nil +} + +func existingGeneratePrivateKey(ctx context.Context, s logical.Storage, keyRef string) certutil.KeyGenerator { + return func(keyType string, keyBits int, container certutil.ParsedPrivateKeyContainer, _ io.Reader) error { + keyId, err := resolveKeyReference(ctx, s, keyRef) + if err != nil { + return err + } + key, err := fetchKeyById(ctx, s, keyId) + if err != nil { + return err + } + signer, err := key.GetSigner() + if err != nil { + return err + } + privateKeyType := certutil.GetPrivateKeyTypeFromSigner(signer) + if privateKeyType == certutil.UnknownPrivateKey { + return errors.New("unknown private key type loaded from key id: " + keyId.String()) + } + blk, _ := pem.Decode([]byte(key.PrivateKey)) + container.SetParsedPrivateKey(signer, privateKeyType, blk.Bytes) + return nil + } +} + diff --git a/builtin/logical/pki/config_util.go b/builtin/logical/pki/config_util.go index 2ba36fe9d0fd4..830590fbd8b03 100644 --- a/builtin/logical/pki/config_util.go +++ b/builtin/logical/pki/config_util.go @@ -2,10 +2,29 @@ package pki import ( "context" + "strings" "github.com/hashicorp/vault/sdk/logical" ) +func isKeyDefaultSet(ctx context.Context, s logical.Storage) (bool, error) { + config, err := getKeysConfig(ctx, s) + if err != nil { + return false, err + } + + return strings.TrimSpace(config.DefaultKeyId.String()) != "", nil +} + +func isIssuerDefaultSet(ctx context.Context, s logical.Storage) (bool, error) { + config, err := getIssuersConfig(ctx, s) + if err != nil { + return false, err + } + + return strings.TrimSpace(config.DefaultIssuerId.String()) != "", nil +} + func updateDefaultKeyId(ctx context.Context, s logical.Storage, id keyId) error { config, err := getKeysConfig(ctx, s) if err != nil { diff --git a/builtin/logical/pki/fields.go b/builtin/logical/pki/fields.go index 4593d6d9a7ccb..fa94751451b7a 100644 --- a/builtin/logical/pki/fields.go +++ b/builtin/logical/pki/fields.go @@ -314,6 +314,18 @@ SHA-2-512. Defaults to 0 to automatically detect based on key length Value: "rsa", }, } + + fields["key_id"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Reference to a existing key; either "default" +for the configured default key, an identifier or the name assigned +to the key. Note this is only used for the existing generation type.`, + } + + fields["id"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Assign a name to the generated issuer.`, + } return fields } diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index 907f8aa8da4bb..b9ac8bc5740b4 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -11,6 +11,30 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) +func pathIssuerGenerateRoot(b *backend) *framework.Path { + ret := &framework.Path{ + Pattern: "issuers/generate/root/" + framework.GenericNameRegex("exported"), + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathCAGenerateRoot, + // Read more about why these flags are set in backend.go + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + }, + + HelpSynopsis: pathGenerateRootHelpSyn, + HelpDescription: pathGenerateRootHelpDesc, + } + + ret.Fields = addCACommonFields(map[string]*framework.FieldSchema{}) + ret.Fields = addCAKeyGenerationFields(ret.Fields) + ret.Fields = addCAIssueFields(ret.Fields) + + return ret +} + func pathImportIssuer(b *backend) *framework.Path { return &framework.Path{ Pattern: "issuers/import/(cert|bundle)", @@ -88,7 +112,7 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d } for _, certPem := range issuers { - cert, existing, err := importIssuer(ctx, req.Storage, certPem) + cert, existing, err := importIssuer(ctx, req.Storage, certPem, "") if err != nil { return logical.ErrorResponse(err.Error()), nil } diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 75d1dfda002f6..5f84c11523584 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -97,6 +97,12 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, role.MaxPathLength = &maxPathLength } + issuerName := "" + issuerNameIface, ok := data.GetOk("id") + if ok { + issuerName = issuerNameIface.(string) + } + input := &inputBundle{ req: req, apiData: data, @@ -163,14 +169,12 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, } // Store it as the CA bundle - entry, err = logical.StorageEntryJSON("config/ca_bundle", cb) - if err != nil { - return nil, err - } - err = req.Storage.Put(ctx, entry) + myIssuer, myKey, err := writeCaBundle(ctx, req.Storage, cb, issuerName) if err != nil { return nil, err } + resp.Data["id"] = myIssuer.ID + resp.Data["key_id"] = myKey.ID // Also store it as just the certificate identified by serial number, so it // can be revoked @@ -184,9 +188,10 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, // For ease of later use, also store just the certificate at a known // location - entry.Key = "ca" - entry.Value = parsedBundle.CertificateBytes - err = req.Storage.Put(ctx, entry) + err = req.Storage.Put(ctx, &logical.StorageEntry{ + Key: "ca", + Value: parsedBundle.CertificateBytes, + }) if err != nil { return nil, err } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 4b6d5b23930e2..8799caf1112e7 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -306,7 +306,7 @@ func deleteIssuer(ctx context.Context, s logical.Storage, id issuerId) error { return s.Delete(ctx, issuerPrefix+id.String()) } -func importIssuer(ctx context.Context, s logical.Storage, certValue string) (*issuer, bool, error) { +func importIssuer(ctx context.Context, s logical.Storage, certValue string, issuerName string) (*issuer, bool, error) { // importIssuers imports the specified PEM-format certificate (from // certValue) into the new PKI storage format. The first return field is a // reference to the new issuer; the second is whether or not the issuer @@ -349,6 +349,7 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string) (*is // storage. var result issuer result.ID = genIssuerId() + result.Name = issuerName result.Certificate = certValue result.CAChain = []string{certValue} @@ -518,6 +519,56 @@ func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuer return &bundle, nil } +func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.CertBundle, issuerName string) (*issuer, *key, error) { + allKeyIds, err := listKeys(ctx, s) + if err != nil { + return nil, nil, err + } + + allIssuerIds, err := listIssuers(ctx, s) + if err != nil { + return nil, nil, err + } + + myKey, _, err := importKey(ctx, s, caBundle.PrivateKey) + if err != nil { + return nil, nil, err + } + + myIssuer, _, err := importIssuer(ctx, s, caBundle.Certificate, issuerName) + if err != nil { + return nil, nil, err + } + + for _, cert := range caBundle.CAChain { + if _, _, err = importIssuer(ctx, s, cert, ""); err != nil { + return nil, nil, err + } + } + + keyDefaultSet, err := isKeyDefaultSet(ctx, s) + if err != nil { + return nil, nil, err + } + if len(allKeyIds) == 0 || !keyDefaultSet { + if err = updateDefaultKeyId(ctx, s, myKey.ID); err != nil { + return nil, nil, err + } + } + + issuerDefaultSet, err := isIssuerDefaultSet(ctx, s) + if err != nil { + return nil, nil, err + } + if len(allIssuerIds) == 0 || !issuerDefaultSet { + if err = updateDefaultIssuerId(ctx, s, myIssuer.ID); err != nil { + return nil, nil, err + } + } + + return myIssuer, myKey, nil +} + func genIssuerId() issuerId { return issuerId(genUuid()) } diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index 73882f57461f6..c1e05263a77e8 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -50,10 +50,12 @@ func migrateStorage(ctx context.Context, req *logical.InitializationRequest, log logger.Warn("performing PKI migration to new keys/issuers layout") - err = migrateToIssuers(ctx, s, legacyBundle) + anIssuer, aKey, err := writeCaBundle(ctx, s, legacyBundle, "") if err != nil { return err } + logger.Info("Migration generated the following ids and set them as defaults", + "issuer id", anIssuer.ID, "key id", aKey.ID) err = setLegacyBundleMigrationLog(ctx, s, &legacyBundleMigration{ hash: hash, @@ -82,35 +84,6 @@ func computeHashOfLegacyBundle(bundle *certutil.CertBundle) (string, error) { return hex.EncodeToString(hasher.Sum(nil)), nil } -func migrateToIssuers(ctx context.Context, s logical.Storage, bundle *certutil.CertBundle) error { - defaultKey, _, err := importKey(ctx, s, bundle.PrivateKey) - if err != nil { - return err - } - - defaultIssuer, _, err := importIssuer(ctx, s, bundle.Certificate) - if err != nil { - return err - } - - for _, cert := range bundle.CAChain { - if _, _, err = importIssuer(ctx, s, cert); err != nil { - return err - } - } - - if err = updateDefaultKeyId(ctx, s, defaultKey.ID); err != nil { - return err - } - - if err = updateDefaultIssuerId(ctx, s, defaultIssuer.ID); err != nil { - return err - } - - // FIXME: Call function that will recompute the CAChain on issuers here. - return nil -} - type legacyBundleMigration struct { hash string created time.Time diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index f8ec0e8978a4b..b96118a448270 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -113,13 +113,13 @@ func Test_KeysIssuerImport(t *testing.T) { require.Equal(t, key1.PrivateKey, key1_ref1.PrivateKey) require.Equal(t, key1_ref1.ID, key1_ref2.ID) - issuer1_ref1, existing, err := importIssuer(ctx, s, issuer1.Certificate) + issuer1_ref1, existing, err := importIssuer(ctx, s, issuer1.Certificate, "") require.NoError(t, err) require.False(t, existing) require.Equal(t, issuer1.Certificate, issuer1_ref1.Certificate) require.Equal(t, key1_ref1.ID, issuer1_ref1.KeyID) - issuer1_ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate) + issuer1_ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate, "") require.NoError(t, err) require.True(t, existing) require.Equal(t, issuer1.Certificate, issuer1_ref1.Certificate) @@ -132,7 +132,7 @@ func Test_KeysIssuerImport(t *testing.T) { err = writeKey(ctx, s, key2) require.NoError(t, err) - issuer2_ref, existing, err := importIssuer(ctx, s, issuer2.Certificate) + issuer2_ref, existing, err := importIssuer(ctx, s, issuer2.Certificate, "") require.NoError(t, err) require.True(t, existing) require.Equal(t, issuer2.Certificate, issuer2_ref.Certificate) diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 3ed0f14bc6734..f6737bdbd1491 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -23,13 +23,29 @@ func denormalizeSerial(serial string) string { } func kmsRequested(input *inputBundle) bool { - exportedStr, ok := input.apiData.GetOk("exported") + return kmsRequestedFromFieldData(input.apiData) +} + +func kmsRequestedFromFieldData(data *framework.FieldData) bool { + exportedStr, ok := data.GetOk("exported") if !ok { return false } return exportedStr.(string) == "kms" } +func existingKeyRequested(input *inputBundle) bool { + return existingKeyRequestedFromFieldData(input.apiData) +} + +func existingKeyRequestedFromFieldData(data *framework.FieldData) bool { + exportedStr, ok := data.GetOk("exported") + if !ok { + return false + } + return exportedStr.(string) == "existing" +} + type managedKeyId interface { String() string } @@ -63,6 +79,19 @@ func getManagedKeyId(data *framework.FieldData) (managedKeyId, error) { return keyId, nil } +func getExistingKeyRef(data *framework.FieldData) (string, error) { + keyRef, ok := data.GetOk("key_id") + if !ok { + return "", errutil.UserError{Err: fmt.Sprintf("missing argument key_id for existing type")} + } + trimmedKeyRef := strings.TrimSpace(keyRef.(string)) + if len(trimmedKeyRef) == 0 { + return "", errutil.UserError{Err: fmt.Sprintf("missing argument key_id for existing type")} + } + + return trimmedKeyRef, nil +} + func getManagedKeyNameOrUUID(data *framework.FieldData) (name string, UUID string, err error) { getApiData := func(argName string) (string, error) { arg, ok := data.GetOk(argName) From 95c10b89f225882b5bc4823a68c3c7f4a073e3a7 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Fri, 8 Apr 2022 15:38:07 -0400 Subject: [PATCH 18/76] Add support for issuer generate intermediate end-point --- builtin/logical/pki/backend.go | 1 + builtin/logical/pki/backend_test.go | 76 ++++++++++++++++++++++ builtin/logical/pki/ca_util.go | 8 +++ builtin/logical/pki/fields.go | 5 -- builtin/logical/pki/path_intermediate.go | 38 +---------- builtin/logical/pki/path_manage_issuers.go | 45 ++++++++++++- builtin/logical/pki/path_root.go | 22 +------ 7 files changed, 133 insertions(+), 62 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 98257e0c51ebc..da60099b358a7 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -117,6 +117,7 @@ func Backend(conf *logical.BackendConfig) *backend { pathIssuerSignSelfIssued(&b), pathIssuerSignVerbatim(&b), pathIssuerGenerateRoot(&b), + pathIssuerGenerateIntermediate(&b), pathConfigIssuers(&b), // Fetch APIs have been lowered to favor the newer issuer API endpoints diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index 0952a04059af1..f1118259b21eb 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -4732,6 +4732,82 @@ func TestRootWithExistingKey(t *testing.T) { require.Contains(t, resp.Data["keys"], myIssuerId3) } +func TestIntermediateWithExistingKey(t *testing.T) { + coreConfig := &vault.CoreConfig{ + LogicalBackends: map[string]logical.Factory{ + "pki": Factory, + }, + } + cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + defer cluster.Cleanup() + + client := cluster.Cores[0].Client + var err error + + mountPKIEndpoint(t, client, "pki-root") + + // Fail requests if type is existing, and we specify the key_type param + ctx := context.Background() + _, err = client.Logical().WriteWithContext(ctx, "pki-root/intermediate/generate/existing", map[string]interface{}{ + "common_name": "root myvault.com", + "key_type": "rsa", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "key_type nor key_bits arguments can be set in this mode") + + // Fail requests if type is existing, and we specify the key_bits param + _, err = client.Logical().WriteWithContext(ctx, "pki-root/intermediate/generate/existing", map[string]interface{}{ + "common_name": "root myvault.com", + "key_bits": "2048", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "key_type nor key_bits arguments can be set in this mode") + + // Fail if the specified key does not exist. + _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/intermediate/existing", map[string]interface{}{ + "common_name": "root myvault.com", + "key_id": "my-key1", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "unable to find PKI key for reference: my-key1") + + // Create the first intermediate CA + resp, err := client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/intermediate/internal", map[string]interface{}{ + "common_name": "root myvault.com", + "key_type": "rsa", + }) + require.NoError(t, err) + // csr1 := resp.Data["csr"] + myKeyId1 := resp.Data["key_id"] + require.NotEmpty(t, myKeyId1) + + // Create the second intermediate CA + resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/intermediate/internal", map[string]interface{}{ + "common_name": "root myvault.com", + "key_type": "rsa", + }) + require.NoError(t, err) + // csr2 := resp.Data["csr"] + myKeyId2 := resp.Data["key_id"] + require.NotEmpty(t, myKeyId2) + + // Create a third intermediate CA re-using key from intermediate CA 1 + resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/intermediate/existing", map[string]interface{}{ + "common_name": "root myvault.com", + "key_id": myKeyId1, + }) + require.NoError(t, err) + // csr3 := resp.Data["csr"] + myKeyId3 := resp.Data["key_id"] + require.NotEmpty(t, myKeyId3) + + require.NotEqual(t, myKeyId1, myKeyId2) + require.Equal(t, myKeyId1, myKeyId3) +} + var ( initTest sync.Once rsaCAKey string diff --git a/builtin/logical/pki/ca_util.go b/builtin/logical/pki/ca_util.go index 0cd8cbba23a87..e0f62a56b10b8 100644 --- a/builtin/logical/pki/ca_util.go +++ b/builtin/logical/pki/ca_util.go @@ -96,6 +96,14 @@ func generateCSRBundle(ctx context.Context, b *backend, input *inputBundle, data return generateManagedKeyCSRBundle(ctx, b, input, data, addBasicConstraints, randomSource) } + if existingKeyRequested(input) { + keyRef, err := getExistingKeyRef(input.apiData) + if err != nil { + return nil, err + } + return certutil.CreateCSRWithKeyGenerator(data, addBasicConstraints, randomSource, existingGeneratePrivateKey(ctx, input.req.Storage, keyRef)) + } + return certutil.CreateCSRWithRandomSource(data, addBasicConstraints, randomSource) } diff --git a/builtin/logical/pki/fields.go b/builtin/logical/pki/fields.go index fa94751451b7a..5e9d29f8cfb0d 100644 --- a/builtin/logical/pki/fields.go +++ b/builtin/logical/pki/fields.go @@ -321,11 +321,6 @@ SHA-2-512. Defaults to 0 to automatically detect based on key length for the configured default key, an identifier or the name assigned to the key. Note this is only used for the existing generation type.`, } - - fields["id"] = &framework.FieldSchema{ - Type: framework.TypeString, - Description: `Assign a name to the generated issuer.`, - } return fields } diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index 4e1e766eae356..6828c20b0a001 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -12,32 +12,7 @@ import ( ) func pathGenerateIntermediate(b *backend) *framework.Path { - ret := &framework.Path{ - Pattern: "intermediate/generate/" + framework.GenericNameRegex("exported"), - Operations: map[logical.Operation]framework.OperationHandler{ - logical.UpdateOperation: &framework.PathOperation{ - Callback: b.pathGenerateIntermediate, - // Read more about why these flags are set in backend.go - ForwardPerformanceStandby: true, - ForwardPerformanceSecondary: true, - }, - }, - - HelpSynopsis: pathGenerateIntermediateHelpSyn, - HelpDescription: pathGenerateIntermediateHelpDesc, - } - - ret.Fields = addCACommonFields(map[string]*framework.FieldSchema{}) - ret.Fields = addCAKeyGenerationFields(ret.Fields) - ret.Fields["add_basic_constraints"] = &framework.FieldSchema{ - Type: framework.TypeBool, - Description: `Whether to add a Basic Constraints -extension with CA: true. Only needed as a -workaround in some compatibility scenarios -with Active Directory Certificate Services.`, - } - - return ret + return commonGenerateIntermediate(b, "intermediate/generate/"+framework.GenericNameRegex("exported")) } func pathSetSignedIntermediate(b *backend) *framework.Path { @@ -135,18 +110,11 @@ func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Req } } - cb := &certutil.CertBundle{} - cb.PrivateKey = csrb.PrivateKey - cb.PrivateKeyType = csrb.PrivateKeyType - - entry, err := logical.StorageEntryJSON("config/ca_bundle", cb) - if err != nil { - return nil, err - } - err = req.Storage.Put(ctx, entry) + myKey, _, err := importKey(ctx, req.Storage, csrb.PrivateKey) if err != nil { return nil, err } + resp.Data["key_id"] = myKey.ID return resp, nil } diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index b9ac8bc5740b4..58f2a3ade714f 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -12,8 +12,12 @@ import ( ) func pathIssuerGenerateRoot(b *backend) *framework.Path { + return commonGenerateRoot(b, "issuers/generate/root/"+framework.GenericNameRegex("exported")) +} + +func commonGenerateRoot(b *backend, pattern string) *framework.Path { ret := &framework.Path{ - Pattern: "issuers/generate/root/" + framework.GenericNameRegex("exported"), + Pattern: pattern, Operations: map[logical.Operation]framework.OperationHandler{ logical.UpdateOperation: &framework.PathOperation{ @@ -32,6 +36,45 @@ func pathIssuerGenerateRoot(b *backend) *framework.Path { ret.Fields = addCAKeyGenerationFields(ret.Fields) ret.Fields = addCAIssueFields(ret.Fields) + ret.Fields["id"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Assign a name to the generated issuer.`, + } + + return ret +} + +func pathIssuerGenerateIntermediate(b *backend) *framework.Path { + return commonGenerateIntermediate(b, + "issuers/generate/intermediate/"+framework.GenericNameRegex("exported")) +} + +func commonGenerateIntermediate(b *backend, pattern string) *framework.Path { + ret := &framework.Path{ + Pattern: pattern, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathGenerateIntermediate, + // Read more about why these flags are set in backend.go + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + }, + + HelpSynopsis: pathGenerateIntermediateHelpSyn, + HelpDescription: pathGenerateIntermediateHelpDesc, + } + + ret.Fields = addCACommonFields(map[string]*framework.FieldSchema{}) + ret.Fields = addCAKeyGenerationFields(ret.Fields) + ret.Fields["add_basic_constraints"] = &framework.FieldSchema{ + Type: framework.TypeBool, + Description: `Whether to add a Basic Constraints +extension with CA: true. Only needed as a +workaround in some compatibility scenarios +with Active Directory Certificate Services.`, + } + return ret } diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 5f84c11523584..18ee65f96ed6f 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -26,27 +26,7 @@ import ( ) func pathGenerateRoot(b *backend) *framework.Path { - ret := &framework.Path{ - Pattern: "root/generate/" + framework.GenericNameRegex("exported"), - - Operations: map[logical.Operation]framework.OperationHandler{ - logical.UpdateOperation: &framework.PathOperation{ - Callback: b.pathCAGenerateRoot, - // Read more about why these flags are set in backend.go - ForwardPerformanceStandby: true, - ForwardPerformanceSecondary: true, - }, - }, - - HelpSynopsis: pathGenerateRootHelpSyn, - HelpDescription: pathGenerateRootHelpDesc, - } - - ret.Fields = addCACommonFields(map[string]*framework.FieldSchema{}) - ret.Fields = addCAKeyGenerationFields(ret.Fields) - ret.Fields = addCAIssueFields(ret.Fields) - - return ret + return commonGenerateRoot(b, "root/generate/"+framework.GenericNameRegex("exported")) } func pathDeleteRoot(b *backend) *framework.Path { From 5cd5badc72b4a81d0d0d8017eb904b0c668b1c97 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Mon, 11 Apr 2022 10:49:10 -0400 Subject: [PATCH 19/76] Update issuer and key arguments to consistent values - Update all new API endpoints to use the new agreed upon argument names. - issuer_ref & key_ref to refer to existing - issuer_name & key_name for new definitions - Update returned values to always user issuer_id and key_id --- builtin/logical/pki/backend_test.go | 22 +++---- builtin/logical/pki/fields.go | 70 ++++++++++++++++++---- builtin/logical/pki/path_fetch_issuers.go | 38 +++++------- builtin/logical/pki/path_issue_sign.go | 8 +-- builtin/logical/pki/path_manage_issuers.go | 6 -- builtin/logical/pki/path_root.go | 6 +- builtin/logical/pki/path_sign_issuers.go | 8 +-- builtin/logical/pki/util.go | 6 +- 8 files changed, 99 insertions(+), 65 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index f1118259b21eb..6415143741af0 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -4674,8 +4674,8 @@ func TestRootWithExistingKey(t *testing.T) { // Fail if the specified key does not exist. _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/existing", map[string]interface{}{ "common_name": "root myvault.com", - "id": "my-issuer1", - "key_id": "my-key1", + "issuer_name": "my-issuer1", + "key_ref": "my-key1", }) require.Error(t, err) require.Contains(t, err.Error(), "unable to find PKI key for reference: my-key1") @@ -4684,11 +4684,11 @@ func TestRootWithExistingKey(t *testing.T) { resp, err := client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ "common_name": "root myvault.com", "key_type": "rsa", - "id": "my-issuer1", + "issuer_name": "my-issuer1", }) require.NoError(t, err) require.NotNil(t, resp.Data["certificate"]) - myIssuerId1 := resp.Data["id"] + myIssuerId1 := resp.Data["issuer_id"] myKeyId1 := resp.Data["key_id"] require.NotEmpty(t, myIssuerId1) require.NotEmpty(t, myKeyId1) @@ -4697,11 +4697,11 @@ func TestRootWithExistingKey(t *testing.T) { resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ "common_name": "root myvault.com", "key_type": "rsa", - "id": "my-issuer2", + "issuer_name": "my-issuer2", }) require.NoError(t, err) require.NotNil(t, resp.Data["certificate"]) - myIssuerId2 := resp.Data["id"] + myIssuerId2 := resp.Data["issuer_id"] myKeyId2 := resp.Data["key_id"] require.NotEmpty(t, myIssuerId2) require.NotEmpty(t, myKeyId2) @@ -4709,12 +4709,12 @@ func TestRootWithExistingKey(t *testing.T) { // Create a third CA re-using key from CA 1 resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/existing", map[string]interface{}{ "common_name": "root myvault.com", - "id": "my-issuer3", - "key_id": myKeyId1, + "issuer_name": "my-issuer3", + "key_ref": myKeyId1, }) require.NoError(t, err) require.NotNil(t, resp.Data["certificate"]) - myIssuerId3 := resp.Data["id"] + myIssuerId3 := resp.Data["issuer_id"] myKeyId3 := resp.Data["key_id"] require.NotEmpty(t, myIssuerId3) require.NotEmpty(t, myKeyId3) @@ -4769,7 +4769,7 @@ func TestIntermediateWithExistingKey(t *testing.T) { // Fail if the specified key does not exist. _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/intermediate/existing", map[string]interface{}{ "common_name": "root myvault.com", - "key_id": "my-key1", + "key_ref": "my-key1", }) require.Error(t, err) require.Contains(t, err.Error(), "unable to find PKI key for reference: my-key1") @@ -4797,7 +4797,7 @@ func TestIntermediateWithExistingKey(t *testing.T) { // Create a third intermediate CA re-using key from intermediate CA 1 resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/intermediate/existing", map[string]interface{}{ "common_name": "root myvault.com", - "key_id": myKeyId1, + "key_ref": myKeyId1, }) require.NoError(t, err) // csr3 := resp.Data["csr"] diff --git a/builtin/logical/pki/fields.go b/builtin/logical/pki/fields.go index 5e9d29f8cfb0d..d9f23fe58a20f 100644 --- a/builtin/logical/pki/fields.go +++ b/builtin/logical/pki/fields.go @@ -132,11 +132,7 @@ be larger than the role max TTL.`, The value format should be given in UTC format YYYY-MM-ddTHH:MM:SSZ`, } - fields["ref"] = &framework.FieldSchema{ - Type: framework.TypeString, - Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, - Default: "default", - } + fields = addIssuerRefField(fields) return fields } @@ -315,12 +311,8 @@ SHA-2-512. Defaults to 0 to automatically detect based on key length }, } - fields["key_id"] = &framework.FieldSchema{ - Type: framework.TypeString, - Description: `Reference to a existing key; either "default" -for the configured default key, an identifier or the name assigned -to the key. Note this is only used for the existing generation type.`, - } + fields = addKeyRefNameFields(fields) + return fields } @@ -341,5 +333,61 @@ func addCAIssueFields(fields map[string]*framework.FieldSchema) map[string]*fram }, } + fields = addIssuerNameField(fields) + + return fields +} + +func addKeyRefNameFields(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { + fields = addKeyNameField(fields) + fields = addKeyRefField(fields) + return fields +} + +func addIssuerRefNameFields(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { + fields = addIssuerNameField(fields) + fields = addIssuerRefField(fields) + return fields +} + +func addIssuerRefField(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { + fields["issuer_ref"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Reference to a existing issuer; either "default" +for the configured default issuer, an identifier or the name assigned +to the issuer.`, + Default: "default", + } + return fields +} + +func addIssuerNameField(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { + fields["issuer_name"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Provide a name to the generated issuer, the name +must be unique across all issuers and not be the reserved value 'default'`, + } + return fields +} + +func addKeyNameField(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { + fields["key_name"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Provide a name for the key that will be generated, +the name must be unique across all keys and not be the reserved value +'default'`, + } + + return fields +} + +func addKeyRefField(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { + fields["key_ref"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Reference to a existing key; either "default" +for the configured default key, an identifier or the name assigned +to the key.`, + Default: "default", + } return fields } diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 94f34edefa859..a85eb81da457f 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -10,7 +10,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -var nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex("ref") + "$") +var nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex("issuer_ref") + "$") func pathListIssuers(b *backend) *framework.Path { return &framework.Path{ @@ -45,7 +45,7 @@ func (b *backend) pathListIssuersHandler(ctx context.Context, req *logical.Reque responseKeys = append(responseKeys, string(identifier)) responseInfo[string(identifier)] = map[string]interface{}{ - "name": issuer.Name, + "issuer_name": issuer.Name, } } @@ -61,25 +61,17 @@ their identifier and their name (if set). ) func pathGetIssuer(b *backend) *framework.Path { - pattern := "issuer/" + framework.GenericNameRegex("ref") + "(/der|/pem)?" + pattern := "issuer/" + framework.GenericNameRegex("issuer_ref") + "(/der|/pem)?" return buildPathGetIssuer(b, pattern) } func buildPathGetIssuer(b *backend, pattern string) *framework.Path { + fields := map[string]*framework.FieldSchema{} + fields = addIssuerRefNameFields(fields) return &framework.Path{ // Returns a JSON entry. Pattern: pattern, - Fields: map[string]*framework.FieldSchema{ - "ref": { - Type: framework.TypeString, - Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, - Default: "default", - }, - "name": { - Type: framework.TypeString, - Description: `Human-readable name for this issuer.`, - }, - }, + Fields: fields, Callbacks: map[logical.Operation]framework.OperationFunc{ logical.ReadOperation: b.pathGetIssuer, @@ -98,7 +90,7 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data return b.pathGetRawIssuer(ctx, req, data) } - issuerName := data.Get("ref").(string) + issuerName := data.Get("issuer_ref").(string) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } @@ -118,8 +110,8 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data return &logical.Response{ Data: map[string]interface{}{ - "id": issuer.ID, - "name": issuer.Name, + "issuer_id": issuer.ID, + "issuer_name": issuer.Name, "key_id": issuer.KeyID, "certificate": issuer.Certificate, }, @@ -127,12 +119,12 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data } func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - issuerName := data.Get("ref").(string) + issuerName := data.Get("issuer_ref").(string) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } - newName := data.Get("name").(string) + newName := data.Get("issuer_name").(string) if len(newName) > 0 && !nameMatcher.MatchString(newName) { return logical.ErrorResponse("new issuer name outside of valid character limits"), nil } @@ -161,8 +153,8 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da return &logical.Response{ Data: map[string]interface{}{ - "id": issuer.ID, - "name": issuer.Name, + "issuer_id": issuer.ID, + "issuer_name": issuer.Name, "key_id": issuer.KeyID, "certificate": issuer.Certificate, }, @@ -170,7 +162,7 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da } func (b *backend) pathGetRawIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - issuerName := data.Get("ref").(string) + issuerName := data.Get("issuer_ref").(string) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } @@ -217,7 +209,7 @@ func (b *backend) pathGetRawIssuer(ctx context.Context, req *logical.Request, da } func (b *backend) pathDeleteIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - issuerName := data.Get("ref").(string) + issuerName := data.Get("issuer_ref").(string) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 1a485189ecf05..6d72278b4067d 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -20,7 +20,7 @@ func pathIssue(b *backend) *framework.Path { } func pathIssuerIssue(b *backend) *framework.Path { - pattern := "issuer/" + framework.GenericNameRegex("ref") + "/issue/" + framework.GenericNameRegex("role") + pattern := "issuer/" + framework.GenericNameRegex("issuer_ref") + "/issue/" + framework.GenericNameRegex("role") return buildPathIssue(b, pattern) } @@ -46,7 +46,7 @@ func pathSign(b *backend) *framework.Path { } func pathIssuerSign(b *backend) *framework.Path { - pattern := "issuer/" + framework.GenericNameRegex("ref") + "/sign/" + framework.GenericNameRegex("role") + pattern := "issuer/" + framework.GenericNameRegex("issuer_ref") + "/sign/" + framework.GenericNameRegex("role") return buildPathSign(b, pattern) } @@ -74,7 +74,7 @@ func buildPathSign(b *backend, pattern string) *framework.Path { } func pathIssuerSignVerbatim(b *backend) *framework.Path { - pattern := "issuers/" + framework.GenericNameRegex("ref") + "/sign-verbatim" + pattern := "issuers/" + framework.GenericNameRegex("issuer_ref") + "/sign-verbatim" return buildPathIssuerSignVerbatim(b, pattern) } @@ -218,7 +218,7 @@ func (b *backend) pathIssueSignCert(ctx context.Context, req *logical.Request, d return nil, logical.ErrReadOnly } - issuerName := data.Get("ref").(string) + issuerName := data.Get("issuer_ref").(string) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index 58f2a3ade714f..2357352fbf0a9 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -35,12 +35,6 @@ func commonGenerateRoot(b *backend, pattern string) *framework.Path { ret.Fields = addCACommonFields(map[string]*framework.FieldSchema{}) ret.Fields = addCAKeyGenerationFields(ret.Fields) ret.Fields = addCAIssueFields(ret.Fields) - - ret.Fields["id"] = &framework.FieldSchema{ - Type: framework.TypeString, - Description: `Assign a name to the generated issuer.`, - } - return ret } diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 18ee65f96ed6f..8aa25558a859b 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -153,7 +153,7 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, if err != nil { return nil, err } - resp.Data["id"] = myIssuer.ID + resp.Data["issuer_id"] = myIssuer.ID resp.Data["key_id"] = myKey.ID // Also store it as just the certificate identified by serial number, so it @@ -192,7 +192,7 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, func (b *backend) pathIssuerSignIntermediate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { var err error - issuerName := data.Get("ref").(string) + issuerName := data.Get("issuer_ref").(string) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } @@ -341,7 +341,7 @@ func (b *backend) pathIssuerSignIntermediate(ctx context.Context, req *logical.R func (b *backend) pathIssuerSignSelfIssued(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { var err error - issuerName := data.Get("ref").(string) + issuerName := data.Get("issuer_ref").(string) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } diff --git a/builtin/logical/pki/path_sign_issuers.go b/builtin/logical/pki/path_sign_issuers.go index c073bce48f2e2..7ae493c72c223 100644 --- a/builtin/logical/pki/path_sign_issuers.go +++ b/builtin/logical/pki/path_sign_issuers.go @@ -6,7 +6,7 @@ import ( ) func pathIssuerSignIntermediate(b *backend) *framework.Path { - pattern := "issuers/" + framework.GenericNameRegex("ref") + "/sign-intermediate" + pattern := "issuers/" + framework.GenericNameRegex("issuer_ref") + "/sign-intermediate" return pathIssuerSignIntermediateRaw(b, pattern) } @@ -19,7 +19,7 @@ func pathIssuerSignIntermediateRaw(b *backend, pattern string) *framework.Path { path := &framework.Path{ Pattern: pattern, Fields: map[string]*framework.FieldSchema{ - "ref": { + "issuer_ref": { Type: framework.TypeString, Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, Default: "default", @@ -79,7 +79,7 @@ See the API documentation for more information about required parameters. ) func pathIssuerSignSelfIssued(b *backend) *framework.Path { - pattern := "issuers/" + framework.GenericNameRegex("ref") + "/sign-self-issued" + pattern := "issuers/" + framework.GenericNameRegex("issuer_ref") + "/sign-self-issued" return buildPathIssuerSignSelfIssued(b, pattern) } @@ -92,7 +92,7 @@ func buildPathIssuerSignSelfIssued(b *backend, pattern string) *framework.Path { path := &framework.Path{ Pattern: pattern, Fields: map[string]*framework.FieldSchema{ - "ref": { + "issuer_ref": { Type: framework.TypeString, Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, Default: "default", diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index f6737bdbd1491..03413ee6e1b11 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -80,13 +80,13 @@ func getManagedKeyId(data *framework.FieldData) (managedKeyId, error) { } func getExistingKeyRef(data *framework.FieldData) (string, error) { - keyRef, ok := data.GetOk("key_id") + keyRef, ok := data.GetOk("key_ref") if !ok { - return "", errutil.UserError{Err: fmt.Sprintf("missing argument key_id for existing type")} + return "", errutil.UserError{Err: fmt.Sprintf("missing argument key_ref for existing type")} } trimmedKeyRef := strings.TrimSpace(keyRef.(string)) if len(trimmedKeyRef) == 0 { - return "", errutil.UserError{Err: fmt.Sprintf("missing argument key_id for existing type")} + return "", errutil.UserError{Err: fmt.Sprintf("missing argument key_ref for existing type")} } return trimmedKeyRef, nil From 6ddd258fa8f16d53c980db1152a122234bf27f4e Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Mon, 11 Apr 2022 12:49:39 -0400 Subject: [PATCH 20/76] Add utility methods to fetch common ref and name arguments - Add utility methods to fetch the issuer_name, issuer_ref, key_name and key_ref arguments from data fields. - Centralize the logic to clean up these inputs and apply various validations to all of them. --- builtin/logical/pki/backend_test.go | 36 +++++++++ builtin/logical/pki/ca_util.go | 7 +- builtin/logical/pki/path_fetch_issuers.go | 16 ++-- builtin/logical/pki/path_intermediate.go | 6 +- builtin/logical/pki/path_issue_sign.go | 2 +- builtin/logical/pki/path_manage_issuers.go | 2 +- builtin/logical/pki/path_root.go | 17 +++-- builtin/logical/pki/storage.go | 16 ++-- builtin/logical/pki/storage_migrations.go | 2 +- builtin/logical/pki/storage_test.go | 23 +++--- builtin/logical/pki/util.go | 88 ++++++++++++++++++++-- 11 files changed, 171 insertions(+), 44 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index 6415143741af0..6686b4a4b0f1d 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -4680,6 +4680,23 @@ func TestRootWithExistingKey(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "unable to find PKI key for reference: my-key1") + // Fail if the specified key name is default. + _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ + "common_name": "root myvault.com", + "issuer_name": "my-issuer1", + "key_name": "Default", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "reserved keyword 'default' can not be used as key name") + + // Fail if the specified issuer name is default. + _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ + "common_name": "root myvault.com", + "issuer_name": "DEFAULT", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "reserved keyword 'default' can not be used as issuer name") + // Create the first CA resp, err := client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ "common_name": "root myvault.com", @@ -4693,11 +4710,20 @@ func TestRootWithExistingKey(t *testing.T) { require.NotEmpty(t, myIssuerId1) require.NotEmpty(t, myKeyId1) + // Fail if the specified issuer name is re-used. + _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ + "common_name": "root myvault.com", + "issuer_name": "my-issuer1", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "issuer name already used") + // Create the second CA resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ "common_name": "root myvault.com", "key_type": "rsa", "issuer_name": "my-issuer2", + "key_name": "root-key2", }) require.NoError(t, err) require.NotNil(t, resp.Data["certificate"]) @@ -4706,6 +4732,15 @@ func TestRootWithExistingKey(t *testing.T) { require.NotEmpty(t, myIssuerId2) require.NotEmpty(t, myKeyId2) + // Fail if the specified key name is re-used. + _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ + "common_name": "root myvault.com", + "issuer_name": "my-issuer3", + "key_name": "root-key2", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "key name already used") + // Create a third CA re-using key from CA 1 resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/existing", map[string]interface{}{ "common_name": "root myvault.com", @@ -4788,6 +4823,7 @@ func TestIntermediateWithExistingKey(t *testing.T) { resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/intermediate/internal", map[string]interface{}{ "common_name": "root myvault.com", "key_type": "rsa", + "key_name": "interkey1", }) require.NoError(t, err) // csr2 := resp.Data["csr"] diff --git a/builtin/logical/pki/ca_util.go b/builtin/logical/pki/ca_util.go index e0f62a56b10b8..b89cd087bc069 100644 --- a/builtin/logical/pki/ca_util.go +++ b/builtin/logical/pki/ca_util.go @@ -81,7 +81,7 @@ func generateCABundle(ctx context.Context, b *backend, input *inputBundle, data } if existingKeyRequested(input) { - keyRef, err := getExistingKeyRef(input.apiData) + keyRef, err := getKeyRefWithErr(input.apiData) if err != nil { return nil, err } @@ -97,12 +97,13 @@ func generateCSRBundle(ctx context.Context, b *backend, input *inputBundle, data } if existingKeyRequested(input) { - keyRef, err := getExistingKeyRef(input.apiData) + keyRef, err := getKeyRefWithErr(input.apiData) if err != nil { return nil, err } return certutil.CreateCSRWithKeyGenerator(data, addBasicConstraints, randomSource, existingGeneratePrivateKey(ctx, input.req.Storage, keyRef)) } + return certutil.CreateCSRWithRandomSource(data, addBasicConstraints, randomSource) } @@ -157,7 +158,7 @@ func getKeyTypeAndBitsForRole(ctx context.Context, b *backend, data *framework.F } func getExistingPublicKey(ctx context.Context, s logical.Storage, data *framework.FieldData) (crypto.PublicKey, error) { - keyRef, err := getExistingKeyRef(data) + keyRef, err := getKeyRefWithErr(data) if err != nil { return nil, err } diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index a85eb81da457f..28e5f46f3ca40 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -25,7 +25,7 @@ func pathListIssuers(b *backend) *framework.Path { } } -func (b *backend) pathListIssuersHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathListIssuersHandler(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { var responseKeys []string responseInfo := make(map[string]interface{}) @@ -90,7 +90,7 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data return b.pathGetRawIssuer(ctx, req, data) } - issuerName := data.Get("issuer_ref").(string) + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } @@ -119,14 +119,14 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data } func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - issuerName := data.Get("issuer_ref").(string) + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } - newName := data.Get("issuer_name").(string) - if len(newName) > 0 && !nameMatcher.MatchString(newName) { - return logical.ErrorResponse("new issuer name outside of valid character limits"), nil + newName, err := getIssuerName(ctx, req.Storage, data) + if err != nil { + return logical.ErrorResponse(err.Error()), nil } ref, err := resolveIssuerReference(ctx, req.Storage, issuerName) @@ -162,7 +162,7 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da } func (b *backend) pathGetRawIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - issuerName := data.Get("issuer_ref").(string) + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } @@ -209,7 +209,7 @@ func (b *backend) pathGetRawIssuer(ctx context.Context, req *logical.Request, da } func (b *backend) pathDeleteIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - issuerName := data.Get("issuer_ref").(string) + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index 6828c20b0a001..88dbef9c5e646 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -52,6 +52,10 @@ func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Req return errorResp, nil } + keyName, err := getKeyName(ctx, req.Storage, data) + if err != nil { + return logical.ErrorResponse(err.Error()), nil + } var resp *logical.Response input := &inputBundle{ role: role, @@ -110,7 +114,7 @@ func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Req } } - myKey, _, err := importKey(ctx, req.Storage, csrb.PrivateKey) + myKey, _, err := importKey(ctx, req.Storage, csrb.PrivateKey, keyName) if err != nil { return nil, err } diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 6d72278b4067d..4aae8c278cdd4 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -218,7 +218,7 @@ func (b *backend) pathIssueSignCert(ctx context.Context, req *logical.Request, d return nil, logical.ErrReadOnly } - issuerName := data.Get("issuer_ref").(string) + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index 2357352fbf0a9..d4bb435a75807 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -138,7 +138,7 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d for _, keyPem := range keys { // Handle import of private key. - key, existing, err := importKey(ctx, req.Storage, keyPem) + key, existing, err := importKey(ctx, req.Storage, keyPem, "") if err != nil { return logical.ErrorResponse(err.Error()), nil } diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 8aa25558a859b..8b650b78043a7 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -77,10 +77,13 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, role.MaxPathLength = &maxPathLength } - issuerName := "" - issuerNameIface, ok := data.GetOk("id") - if ok { - issuerName = issuerNameIface.(string) + issuerName, err := getIssuerName(ctx, req.Storage, data) + if err != nil { + return logical.ErrorResponse(err.Error()), nil + } + keyName, err := getKeyName(ctx, req.Storage, data) + if err != nil { + return logical.ErrorResponse(err.Error()), nil } input := &inputBundle{ @@ -149,7 +152,7 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, } // Store it as the CA bundle - myIssuer, myKey, err := writeCaBundle(ctx, req.Storage, cb, issuerName) + myIssuer, myKey, err := writeCaBundle(ctx, req.Storage, cb, issuerName, keyName) if err != nil { return nil, err } @@ -192,7 +195,7 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, func (b *backend) pathIssuerSignIntermediate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { var err error - issuerName := data.Get("issuer_ref").(string) + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } @@ -341,7 +344,7 @@ func (b *backend) pathIssuerSignIntermediate(ctx context.Context, req *logical.R func (b *backend) pathIssuerSignSelfIssued(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { var err error - issuerName := data.Get("issuer_ref").(string) + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 8799caf1112e7..17451e68eab46 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -36,6 +36,11 @@ func (p issuerId) String() string { return string(p) } +const ( + IssuerRefNotFound = issuerId("not-found") + KeyRefNotFound = keyId("not-found") +) + type key struct { ID keyId `json:"id" structs:"id" mapstructure:"id"` Name string `json:"name" structs:"name" mapstructure:"name"` @@ -112,7 +117,7 @@ func deleteKey(ctx context.Context, s logical.Storage, id keyId) error { return s.Delete(ctx, keyPrefix+id.String()) } -func importKey(ctx context.Context, s logical.Storage, keyValue string) (*key, bool, error) { +func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName string) (*key, bool, error) { // importKey imports the specified PEM-format key (from keyValue) into // the new PKI storage format. The first return field is a reference to // the new key; the second is whether or not the key already existed @@ -154,6 +159,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string) (*key, b // Haven't found a key, so we've gotta create it and write it into storage. var result key result.ID = genKeyId() + result.Name = keyName result.PrivateKey = keyValue // Extracting the signer is necessary for two reasons: first, to get the @@ -270,7 +276,7 @@ func resolveKeyReference(ctx context.Context, s logical.Storage, reference strin } // Otherwise, we must not have found the key. - return keyId("not-found"), errutil.UserError{Err: fmt.Sprintf("unable to find PKI key for reference: %v", reference)} + return KeyRefNotFound, errutil.UserError{Err: fmt.Sprintf("unable to find PKI key for reference: %v", reference)} } func fetchIssuerById(ctx context.Context, s logical.Storage, issuerId issuerId) (*issuer, error) { @@ -488,7 +494,7 @@ func resolveIssuerReference(ctx context.Context, s logical.Storage, reference st } // Otherwise, we must not have found the issuer. - return issuerId("not-found"), errutil.UserError{Err: fmt.Sprintf("unable to find PKI issuer for reference: %v", reference)} + return IssuerRefNotFound, errutil.UserError{Err: fmt.Sprintf("unable to find PKI issuer for reference: %v", reference)} } // Builds a certutil.CertBundle from the specified issuer identifier, @@ -519,7 +525,7 @@ func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuer return &bundle, nil } -func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.CertBundle, issuerName string) (*issuer, *key, error) { +func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.CertBundle, issuerName string, keyName string) (*issuer, *key, error) { allKeyIds, err := listKeys(ctx, s) if err != nil { return nil, nil, err @@ -530,7 +536,7 @@ func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.Ce return nil, nil, err } - myKey, _, err := importKey(ctx, s, caBundle.PrivateKey) + myKey, _, err := importKey(ctx, s, caBundle.PrivateKey, keyName) if err != nil { return nil, nil, err } diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index c1e05263a77e8..23c1b23997001 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -50,7 +50,7 @@ func migrateStorage(ctx context.Context, req *logical.InitializationRequest, log logger.Warn("performing PKI migration to new keys/issuers layout") - anIssuer, aKey, err := writeCaBundle(ctx, s, legacyBundle, "") + anIssuer, aKey, err := writeCaBundle(ctx, s, legacyBundle, "", "") if err != nil { return err } diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index b96118a448270..b1b4e277b6ac1 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -102,29 +102,32 @@ func Test_KeysIssuerImport(t *testing.T) { issuer1.ID = "" issuer1.KeyID = "" - key1_ref1, existing, err := importKey(ctx, s, key1.PrivateKey) + key1_ref1, existing, err := importKey(ctx, s, key1.PrivateKey, "key1") require.NoError(t, err) require.False(t, existing) require.Equal(t, key1.PrivateKey, key1_ref1.PrivateKey) - key1_ref2, existing, err := importKey(ctx, s, key1.PrivateKey) + key1_ref2, existing, err := importKey(ctx, s, key1.PrivateKey, "ignore-me") require.NoError(t, err) require.True(t, existing) require.Equal(t, key1.PrivateKey, key1_ref1.PrivateKey) require.Equal(t, key1_ref1.ID, key1_ref2.ID) + require.Equal(t, key1_ref1.Name, key1_ref2.Name) - issuer1_ref1, existing, err := importIssuer(ctx, s, issuer1.Certificate, "") + issuer1_ref1, existing, err := importIssuer(ctx, s, issuer1.Certificate, "issuer1") require.NoError(t, err) require.False(t, existing) require.Equal(t, issuer1.Certificate, issuer1_ref1.Certificate) require.Equal(t, key1_ref1.ID, issuer1_ref1.KeyID) + require.Equal(t, "issuer1", issuer1_ref1.Name) - issuer1_ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate, "") + issuer1_ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) require.Equal(t, issuer1.Certificate, issuer1_ref1.Certificate) require.Equal(t, issuer1_ref1.ID, issuer1_ref2.ID) require.Equal(t, key1_ref1.ID, issuer1_ref2.KeyID) + require.Equal(t, issuer1_ref1.Name, issuer1_ref2.Name) err = writeIssuer(ctx, s, &issuer2) require.NoError(t, err) @@ -132,18 +135,20 @@ func Test_KeysIssuerImport(t *testing.T) { err = writeKey(ctx, s, key2) require.NoError(t, err) - issuer2_ref, existing, err := importIssuer(ctx, s, issuer2.Certificate, "") + issuer2_ref, existing, err := importIssuer(ctx, s, issuer2.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) require.Equal(t, issuer2.Certificate, issuer2_ref.Certificate) - require.Equal(t, issuer2_ref.ID, issuer2.ID) - require.Equal(t, issuer2_ref.KeyID, issuer2.KeyID) + require.Equal(t, issuer2.ID, issuer2_ref.ID) + require.Equal(t, "", issuer2_ref.Name) + require.Equal(t, issuer2.KeyID, issuer2_ref.KeyID) - key2_ref, existing, err := importKey(ctx, s, key2.PrivateKey) + key2_ref, existing, err := importKey(ctx, s, key2.PrivateKey, "ignore-me") require.NoError(t, err) require.True(t, existing) require.Equal(t, key2.PrivateKey, key2_ref.PrivateKey) - require.Equal(t, key2_ref.ID, key2.ID) + require.Equal(t, key2.ID, key2_ref.ID) + require.Equal(t, "", key2_ref.Name) } func genIssuerAndKey(t *testing.T, b *backend) (issuer, key) { diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 03413ee6e1b11..5e92134739ac1 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -1,9 +1,12 @@ package pki import ( + "context" "fmt" "strings" + "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/helper/errutil" @@ -79,17 +82,14 @@ func getManagedKeyId(data *framework.FieldData) (managedKeyId, error) { return keyId, nil } -func getExistingKeyRef(data *framework.FieldData) (string, error) { - keyRef, ok := data.GetOk("key_ref") - if !ok { - return "", errutil.UserError{Err: fmt.Sprintf("missing argument key_ref for existing type")} - } - trimmedKeyRef := strings.TrimSpace(keyRef.(string)) - if len(trimmedKeyRef) == 0 { +func getKeyRefWithErr(data *framework.FieldData) (string, error) { + keyRef := getKeyRef(data) + + if len(keyRef) == 0 { return "", errutil.UserError{Err: fmt.Sprintf("missing argument key_ref for existing type")} } - return trimmedKeyRef, nil + return keyRef, nil } func getManagedKeyNameOrUUID(data *framework.FieldData) (name string, UUID string, err error) { @@ -122,3 +122,75 @@ func getManagedKeyNameOrUUID(data *framework.FieldData) (name string, UUID strin return keyName, keyUUID, nil } + +func getIssuerName(ctx context.Context, s logical.Storage, data *framework.FieldData) (string, error) { + issuerName := "" + issuerNameIface, ok := data.GetOk("issuer_name") + if ok { + issuerName = strings.TrimSpace(issuerNameIface.(string)) + + if strings.ToLower(issuerName) == "default" { + return "", errutil.UserError{Err: "reserved keyword 'default' can not be used as issuer name"} + } + + if !nameMatcher.MatchString(issuerName) { + return "", errutil.UserError{Err: "issuer name contained invalid characters"} + } + issuer_id, err := resolveIssuerReference(ctx, s, issuerName) + if err == nil { + return "", errutil.UserError{Err: "issuer name already used."} + } + + if err != nil && issuer_id != IssuerRefNotFound { + return "", errutil.InternalError{Err: err.Error()} + } + } + return issuerName, nil +} + +func getKeyName(ctx context.Context, s logical.Storage, data *framework.FieldData) (string, error) { + keyName := "" + keyNameIface, ok := data.GetOk("key_name") + if ok { + keyName = strings.TrimSpace(keyNameIface.(string)) + + if strings.ToLower(keyName) == "default" { + return "", errutil.UserError{Err: "reserved keyword 'default' can not be used as key name"} + } + + if !nameMatcher.MatchString(keyName) { + return "", errutil.UserError{Err: "key name contained invalid characters"} + } + key_id, err := resolveKeyReference(ctx, s, keyName) + if err == nil { + return "", errutil.UserError{Err: "key name already used."} + } + + if err != nil && key_id != KeyRefNotFound { + return "", errutil.InternalError{Err: err.Error()} + } + } + return keyName, nil +} + +func getIssuerRef(data *framework.FieldData) string { + return extractRef(data, "issuer_ref") +} + +func getKeyRef(data *framework.FieldData) string { + return extractRef(data, "key_ref") +} + +func extractRef(data *framework.FieldData, paramName string) string { + value := "" + issuerNameIface, ok := data.GetOk(paramName) + if ok { + value = strings.TrimSpace(issuerNameIface.(string)) + if strings.ToLower(value) == "default" { + return "default" + } + return value + } + + return value +} From acebaea15f74574dab7a7598cf4269f5d46b6892 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Mon, 11 Apr 2022 14:16:49 -0400 Subject: [PATCH 21/76] Rename common PKI backend handlers - Use the buildPath convention for the function name instead of common... --- builtin/logical/pki/path_intermediate.go | 2 +- builtin/logical/pki/path_manage_issuers.go | 8 ++++---- builtin/logical/pki/path_root.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index 88dbef9c5e646..2783b08cb9d3b 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -12,7 +12,7 @@ import ( ) func pathGenerateIntermediate(b *backend) *framework.Path { - return commonGenerateIntermediate(b, "intermediate/generate/"+framework.GenericNameRegex("exported")) + return buildPathGenerateIntermediate(b, "intermediate/generate/"+framework.GenericNameRegex("exported")) } func pathSetSignedIntermediate(b *backend) *framework.Path { diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index d4bb435a75807..eb04a21ef48aa 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -12,10 +12,10 @@ import ( ) func pathIssuerGenerateRoot(b *backend) *framework.Path { - return commonGenerateRoot(b, "issuers/generate/root/"+framework.GenericNameRegex("exported")) + return buildPathGenerateRoot(b, "issuers/generate/root/"+framework.GenericNameRegex("exported")) } -func commonGenerateRoot(b *backend, pattern string) *framework.Path { +func buildPathGenerateRoot(b *backend, pattern string) *framework.Path { ret := &framework.Path{ Pattern: pattern, @@ -39,11 +39,11 @@ func commonGenerateRoot(b *backend, pattern string) *framework.Path { } func pathIssuerGenerateIntermediate(b *backend) *framework.Path { - return commonGenerateIntermediate(b, + return buildPathGenerateIntermediate(b, "issuers/generate/intermediate/"+framework.GenericNameRegex("exported")) } -func commonGenerateIntermediate(b *backend, pattern string) *framework.Path { +func buildPathGenerateIntermediate(b *backend, pattern string) *framework.Path { ret := &framework.Path{ Pattern: pattern, Operations: map[logical.Operation]framework.OperationHandler{ diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 8b650b78043a7..297393c854c4e 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -26,7 +26,7 @@ import ( ) func pathGenerateRoot(b *backend) *framework.Path { - return commonGenerateRoot(b, "root/generate/"+framework.GenericNameRegex("exported")) + return buildPathGenerateRoot(b, "root/generate/"+framework.GenericNameRegex("exported")) } func pathDeleteRoot(b *backend) *framework.Path { From 9f6731ce5b4f2fa6738e2e89f805b77b6d7dde38 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Mon, 11 Apr 2022 14:35:26 -0400 Subject: [PATCH 22/76] Move setting PKI defaults from writeCaBundle to proper import{keys,issuer} methods - PR feedback, move setting up the default configuration references within the import methods instead of within the writeCaBundle method. This should now cover all use cases of us setting up the defaults properly. --- builtin/logical/pki/storage.go | 56 +++++++++++++++------------------- builtin/logical/pki/util.go | 12 ++------ 2 files changed, 28 insertions(+), 40 deletions(-) diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 17451e68eab46..e0e64ac454590 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -213,6 +213,18 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName } } + // If there was no prior default value set and/or we had no known + // keys when we started, set this key as default. + keyDefaultSet, err := isKeyDefaultSet(ctx, s) + if err != nil { + return nil, false, err + } + if len(knownKeys) == 0 || !keyDefaultSet { + if err = updateDefaultKeyId(ctx, s, result.ID); err != nil { + return nil, false, err + } + } + // All done; return our new key reference. return &result, false, nil } @@ -399,11 +411,23 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu } } - // Finally we can write the issuer to storage. + // We can write the issuer to storage. if err := writeIssuer(ctx, s, &result); err != nil { return nil, false, err } + // If there was no prior default value set and/or we had no known + // issuers when we started, set this issuer as default. + issuerDefaultSet, err := isIssuerDefaultSet(ctx, s) + if err != nil { + return nil, false, err + } + if len(knownIssuers) == 0 || !issuerDefaultSet { + if err = updateDefaultIssuerId(ctx, s, result.ID); err != nil { + return nil, false, err + } + } + // All done; return our new key reference. return &result, false, nil } @@ -526,16 +550,6 @@ func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuer } func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.CertBundle, issuerName string, keyName string) (*issuer, *key, error) { - allKeyIds, err := listKeys(ctx, s) - if err != nil { - return nil, nil, err - } - - allIssuerIds, err := listIssuers(ctx, s) - if err != nil { - return nil, nil, err - } - myKey, _, err := importKey(ctx, s, caBundle.PrivateKey, keyName) if err != nil { return nil, nil, err @@ -552,26 +566,6 @@ func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.Ce } } - keyDefaultSet, err := isKeyDefaultSet(ctx, s) - if err != nil { - return nil, nil, err - } - if len(allKeyIds) == 0 || !keyDefaultSet { - if err = updateDefaultKeyId(ctx, s, myKey.ID); err != nil { - return nil, nil, err - } - } - - issuerDefaultSet, err := isIssuerDefaultSet(ctx, s) - if err != nil { - return nil, nil, err - } - if len(allIssuerIds) == 0 || !issuerDefaultSet { - if err = updateDefaultIssuerId(ctx, s, myIssuer.ID); err != nil { - return nil, nil, err - } - } - return myIssuer, myKey, nil } diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 5e92134739ac1..4c524e1dd0006 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -182,15 +182,9 @@ func getKeyRef(data *framework.FieldData) string { } func extractRef(data *framework.FieldData, paramName string) string { - value := "" - issuerNameIface, ok := data.GetOk(paramName) - if ok { - value = strings.TrimSpace(issuerNameIface.(string)) - if strings.ToLower(value) == "default" { - return "default" - } - return value + value := strings.TrimSpace(data.Get(paramName).(string)) + if strings.ToLower(value) == "default" { + return "default" } - return value } From 5950270d54edb3cdc36a0bce9b5bd0403afb4f95 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Mon, 11 Apr 2022 17:22:28 -0400 Subject: [PATCH 23/76] Introduce constants for issuer_ref, rename isKeyDefaultSet... --- builtin/logical/pki/backend_test.go | 3 +- builtin/logical/pki/config_util.go | 4 +- builtin/logical/pki/fields.go | 30 ++++++----- builtin/logical/pki/path_config_ca.go | 66 +---------------------- builtin/logical/pki/path_fetch_issuers.go | 4 +- builtin/logical/pki/path_issue_sign.go | 6 +-- builtin/logical/pki/path_root.go | 11 ---- builtin/logical/pki/path_sign_issuers.go | 8 +-- builtin/logical/pki/storage.go | 4 +- builtin/logical/pki/storage_test.go | 6 +++ builtin/logical/pki/util.go | 2 +- 11 files changed, 39 insertions(+), 105 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index 6686b4a4b0f1d..fe40ff0328771 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -2275,7 +2275,6 @@ func TestBackend_Root_Idempotency(t *testing.T) { }) cluster.Start() defer cluster.Cleanup() - client := cluster.Cores[0].Client var err error err = client.Sys().Mount("pki", &api.MountInput{ @@ -4841,7 +4840,7 @@ func TestIntermediateWithExistingKey(t *testing.T) { require.NotEmpty(t, myKeyId3) require.NotEqual(t, myKeyId1, myKeyId2) - require.Equal(t, myKeyId1, myKeyId3) + require.Equal(t, myKeyId1, myKeyId3, "our new ca did not seem to reuse the key as we expected.") } var ( diff --git a/builtin/logical/pki/config_util.go b/builtin/logical/pki/config_util.go index 830590fbd8b03..0dddc620ab27e 100644 --- a/builtin/logical/pki/config_util.go +++ b/builtin/logical/pki/config_util.go @@ -7,7 +7,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -func isKeyDefaultSet(ctx context.Context, s logical.Storage) (bool, error) { +func isDefaultKeySet(ctx context.Context, s logical.Storage) (bool, error) { config, err := getKeysConfig(ctx, s) if err != nil { return false, err @@ -16,7 +16,7 @@ func isKeyDefaultSet(ctx context.Context, s logical.Storage) (bool, error) { return strings.TrimSpace(config.DefaultKeyId.String()) != "", nil } -func isIssuerDefaultSet(ctx context.Context, s logical.Storage) (bool, error) { +func isDefaultIssuerSet(ctx context.Context, s logical.Storage) (bool, error) { config, err := getIssuersConfig(ctx, s) if err != nil { return false, err diff --git a/builtin/logical/pki/fields.go b/builtin/logical/pki/fields.go index d9f23fe58a20f..afd8723964538 100644 --- a/builtin/logical/pki/fields.go +++ b/builtin/logical/pki/fields.go @@ -2,6 +2,10 @@ package pki import "github.com/hashicorp/vault/sdk/framework" +const ( + issuerRefParam = "issuer_ref" +) + // addIssueAndSignCommonFields adds fields common to both CA and non-CA issuing // and signing func addIssueAndSignCommonFields(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { @@ -338,20 +342,23 @@ func addCAIssueFields(fields map[string]*framework.FieldSchema) map[string]*fram return fields } -func addKeyRefNameFields(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { - fields = addKeyNameField(fields) - fields = addKeyRefField(fields) - return fields -} - func addIssuerRefNameFields(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { fields = addIssuerNameField(fields) fields = addIssuerRefField(fields) return fields } +func addIssuerNameField(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { + fields["issuer_name"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Provide a name to the generated issuer, the name +must be unique across all issuers and not be the reserved value 'default'`, + } + return fields +} + func addIssuerRefField(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { - fields["issuer_ref"] = &framework.FieldSchema{ + fields[issuerRefParam] = &framework.FieldSchema{ Type: framework.TypeString, Description: `Reference to a existing issuer; either "default" for the configured default issuer, an identifier or the name assigned @@ -361,12 +368,9 @@ to the issuer.`, return fields } -func addIssuerNameField(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { - fields["issuer_name"] = &framework.FieldSchema{ - Type: framework.TypeString, - Description: `Provide a name to the generated issuer, the name -must be unique across all issuers and not be the reserved value 'default'`, - } +func addKeyRefNameFields(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { + fields = addKeyNameField(fields) + fields = addKeyRefField(fields) return fields } diff --git a/builtin/logical/pki/path_config_ca.go b/builtin/logical/pki/path_config_ca.go index d16137ea3c5bd..6b4590344966b 100644 --- a/builtin/logical/pki/path_config_ca.go +++ b/builtin/logical/pki/path_config_ca.go @@ -2,11 +2,8 @@ package pki import ( "context" - "fmt" "github.com/hashicorp/vault/sdk/framework" - "github.com/hashicorp/vault/sdk/helper/certutil" - "github.com/hashicorp/vault/sdk/helper/errutil" "github.com/hashicorp/vault/sdk/logical" ) @@ -22,7 +19,7 @@ secret key and certificate.`, }, Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.pathCAWrite, + logical.UpdateOperation: b.pathImportIssuers, }, HelpSynopsis: pathConfigCAHelpSyn, @@ -30,67 +27,6 @@ secret key and certificate.`, } } -func (b *backend) pathCAWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - pemBundle := data.Get("pem_bundle").(string) - - if pemBundle == "" { - return logical.ErrorResponse("'pem_bundle' was empty"), nil - } - - parsedBundle, err := certutil.ParsePEMBundle(pemBundle) - if err != nil { - switch err.(type) { - case errutil.InternalError: - return nil, err - default: - return logical.ErrorResponse(err.Error()), nil - } - } - - if parsedBundle.PrivateKey == nil { - return logical.ErrorResponse("private key not found in the PEM bundle"), nil - } - - if parsedBundle.PrivateKeyType == certutil.UnknownPrivateKey { - return logical.ErrorResponse("unknown private key found in the PEM bundle"), nil - } - - if parsedBundle.Certificate == nil { - return logical.ErrorResponse("no certificate found in the PEM bundle"), nil - } - - if !parsedBundle.Certificate.IsCA { - return logical.ErrorResponse("the given certificate is not marked for CA use and cannot be used with this backend"), nil - } - - cb, err := parsedBundle.ToCertBundle() - if err != nil { - return nil, fmt.Errorf("error converting raw values into cert bundle: %w", err) - } - - entry, err := logical.StorageEntryJSON("config/ca_bundle", cb) - if err != nil { - return nil, err - } - err = req.Storage.Put(ctx, entry) - if err != nil { - return nil, err - } - - // For ease of later use, also store just the certificate at a known - // location, plus a fresh CRL - entry.Key = "ca" - entry.Value = parsedBundle.CertificateBytes - err = req.Storage.Put(ctx, entry) - if err != nil { - return nil, err - } - - err = buildCRL(ctx, b, req, true) - - return nil, err -} - const pathConfigCAHelpSyn = ` Set the CA certificate and private key used for generated credentials. ` diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 28e5f46f3ca40..6f15761d31fb3 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -10,7 +10,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -var nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex("issuer_ref") + "$") +var nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex(issuerRefParam) + "$") func pathListIssuers(b *backend) *framework.Path { return &framework.Path{ @@ -61,7 +61,7 @@ their identifier and their name (if set). ) func pathGetIssuer(b *backend) *framework.Path { - pattern := "issuer/" + framework.GenericNameRegex("issuer_ref") + "(/der|/pem)?" + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "(/der|/pem)?" return buildPathGetIssuer(b, pattern) } diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 4aae8c278cdd4..e77a983fa61ef 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -20,7 +20,7 @@ func pathIssue(b *backend) *framework.Path { } func pathIssuerIssue(b *backend) *framework.Path { - pattern := "issuer/" + framework.GenericNameRegex("issuer_ref") + "/issue/" + framework.GenericNameRegex("role") + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "/issue/" + framework.GenericNameRegex("role") return buildPathIssue(b, pattern) } @@ -46,7 +46,7 @@ func pathSign(b *backend) *framework.Path { } func pathIssuerSign(b *backend) *framework.Path { - pattern := "issuer/" + framework.GenericNameRegex("issuer_ref") + "/sign/" + framework.GenericNameRegex("role") + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "/sign/" + framework.GenericNameRegex("role") return buildPathSign(b, pattern) } @@ -74,7 +74,7 @@ func buildPathSign(b *backend, pattern string) *framework.Path { } func pathIssuerSignVerbatim(b *backend) *framework.Path { - pattern := "issuers/" + framework.GenericNameRegex("issuer_ref") + "/sign-verbatim" + pattern := "issuers/" + framework.GenericNameRegex(issuerRefParam) + "/sign-verbatim" return buildPathIssuerSignVerbatim(b, pattern) } diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 297393c854c4e..41b2734cea2b9 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -55,17 +55,6 @@ func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, da func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { var err error - entry, err := req.Storage.Get(ctx, "config/ca_bundle") - if err != nil { - return nil, err - } - if entry != nil { - resp := &logical.Response{} - resp.AddWarning(fmt.Sprintf("Refusing to generate a root certificate over an existing root certificate. "+ - "If you really want to destroy the original root certificate, please issue a delete against %s root.", req.MountPoint)) - return resp, nil - } - exported, format, role, errorResp := b.getGenerationParams(ctx, data, req.MountPoint) if errorResp != nil { return errorResp, nil diff --git a/builtin/logical/pki/path_sign_issuers.go b/builtin/logical/pki/path_sign_issuers.go index 7ae493c72c223..7afa1c752f7a5 100644 --- a/builtin/logical/pki/path_sign_issuers.go +++ b/builtin/logical/pki/path_sign_issuers.go @@ -6,7 +6,7 @@ import ( ) func pathIssuerSignIntermediate(b *backend) *framework.Path { - pattern := "issuers/" + framework.GenericNameRegex("issuer_ref") + "/sign-intermediate" + pattern := "issuers/" + framework.GenericNameRegex(issuerRefParam) + "/sign-intermediate" return pathIssuerSignIntermediateRaw(b, pattern) } @@ -19,7 +19,7 @@ func pathIssuerSignIntermediateRaw(b *backend, pattern string) *framework.Path { path := &framework.Path{ Pattern: pattern, Fields: map[string]*framework.FieldSchema{ - "issuer_ref": { + issuerRefParam: { Type: framework.TypeString, Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, Default: "default", @@ -79,7 +79,7 @@ See the API documentation for more information about required parameters. ) func pathIssuerSignSelfIssued(b *backend) *framework.Path { - pattern := "issuers/" + framework.GenericNameRegex("issuer_ref") + "/sign-self-issued" + pattern := "issuers/" + framework.GenericNameRegex(issuerRefParam) + "/sign-self-issued" return buildPathIssuerSignSelfIssued(b, pattern) } @@ -92,7 +92,7 @@ func buildPathIssuerSignSelfIssued(b *backend, pattern string) *framework.Path { path := &framework.Path{ Pattern: pattern, Fields: map[string]*framework.FieldSchema{ - "issuer_ref": { + issuerRefParam: { Type: framework.TypeString, Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, Default: "default", diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index e0e64ac454590..e341f1fb8817c 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -215,7 +215,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName // If there was no prior default value set and/or we had no known // keys when we started, set this key as default. - keyDefaultSet, err := isKeyDefaultSet(ctx, s) + keyDefaultSet, err := isDefaultKeySet(ctx, s) if err != nil { return nil, false, err } @@ -418,7 +418,7 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu // If there was no prior default value set and/or we had no known // issuers when we started, set this issuer as default. - issuerDefaultSet, err := isIssuerDefaultSet(ctx, s) + issuerDefaultSet, err := isDefaultIssuerSet(ctx, s) if err != nil { return nil, false, err } diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index b1b4e277b6ac1..a47a324c3c417 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -107,6 +107,8 @@ func Test_KeysIssuerImport(t *testing.T) { require.False(t, existing) require.Equal(t, key1.PrivateKey, key1_ref1.PrivateKey) + // Make sure if we attempt to re-import the same private key, no import/updates occur. + // So the existing flag should be set to true and we do not update the existing Name field. key1_ref2, existing, err := importKey(ctx, s, key1.PrivateKey, "ignore-me") require.NoError(t, err) require.True(t, existing) @@ -121,6 +123,8 @@ func Test_KeysIssuerImport(t *testing.T) { require.Equal(t, key1_ref1.ID, issuer1_ref1.KeyID) require.Equal(t, "issuer1", issuer1_ref1.Name) + // Make sure if we attempt to re-import the same issuer, no import/updates occur. + // So the existing flag should be set to true and we do not update the existing Name field. issuer1_ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) @@ -135,6 +139,7 @@ func Test_KeysIssuerImport(t *testing.T) { err = writeKey(ctx, s, key2) require.NoError(t, err) + // Same double import tests as above, but make sure if the previous was created through writeIssuer not importIssuer. issuer2_ref, existing, err := importIssuer(ctx, s, issuer2.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) @@ -143,6 +148,7 @@ func Test_KeysIssuerImport(t *testing.T) { require.Equal(t, "", issuer2_ref.Name) require.Equal(t, issuer2.KeyID, issuer2_ref.KeyID) + // Same double import tests as above, but make sure if the previous was created through writeKey not importKey. key2_ref, existing, err := importKey(ctx, s, key2.PrivateKey, "ignore-me") require.NoError(t, err) require.True(t, existing) diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 4c524e1dd0006..cabf9f0b5adae 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -174,7 +174,7 @@ func getKeyName(ctx context.Context, s logical.Storage, data *framework.FieldDat } func getIssuerRef(data *framework.FieldData) string { - return extractRef(data, "issuer_ref") + return extractRef(data, issuerRefParam) } func getKeyRef(data *framework.FieldData) string { From a9264528ea0d9722d9230823820b2f7250700d85 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Mon, 11 Apr 2022 15:18:31 -0400 Subject: [PATCH 24/76] Fix legacy PKI sign-verbatim api path - Addresses some test failures due to an incorrect refactoring of a legacy api path /sign-verbatim within PKI --- builtin/logical/pki/path_issue_sign.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index e77a983fa61ef..9993cd5eeb675 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -79,7 +79,7 @@ func pathIssuerSignVerbatim(b *backend) *framework.Path { } func pathSignVerbatim(b *backend) *framework.Path { - pattern := "root/sign-verbatim" + pattern := "sign-verbatim" + framework.OptionalParamRegex("role") return buildPathIssuerSignVerbatim(b, pattern) } From 651133386218597f9b7c5623ff6484c4fd3f11e6 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 11 Apr 2022 15:36:46 -0400 Subject: [PATCH 25/76] Use import code to handle intermediate, config/ca The existing bundle import code will satisfy the intermediate import; use it instead of the old ca_bundle import logic. Additionally, update /config/ca to use the new import code as well. While testing, a panic was discovered: > reflect.Value.SetMapIndex: value of type string is not assignable to type pki.keyId This was caused by returning a map with type issuerId->keyId; instead switch to returning string->string maps so the audit log can properly HMAC them. Signed-off-by: Alexander Scheel --- builtin/logical/pki/path_intermediate.go | 101 +-------------------- builtin/logical/pki/path_manage_issuers.go | 37 ++++++-- 2 files changed, 30 insertions(+), 108 deletions(-) diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index 2783b08cb9d3b..2ab87f985a100 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/hashicorp/vault/sdk/framework" - "github.com/hashicorp/vault/sdk/helper/certutil" "github.com/hashicorp/vault/sdk/helper/errutil" "github.com/hashicorp/vault/sdk/logical" ) @@ -25,12 +24,13 @@ func pathSetSignedIntermediate(b *backend) *framework.Path { Description: `PEM-format certificate. This must be a CA certificate with a public key matching the previously-generated key from the generation -endpoint.`, +endpoint. Additional parent CAs may be optionally +appended to the bundle.`, }, }, Operations: map[logical.Operation]framework.OperationHandler{ logical.UpdateOperation: &framework.PathOperation{ - Callback: b.pathSetSignedIntermediate, + Callback: b.pathImportIssuers, // Read more about why these flags are set in backend.go ForwardPerformanceStandby: true, ForwardPerformanceSecondary: true, @@ -123,101 +123,6 @@ func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Req return resp, nil } -func (b *backend) pathSetSignedIntermediate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - cert := data.Get("certificate").(string) - - if cert == "" { - return logical.ErrorResponse("no certificate provided in the \"certificate\" parameter"), nil - } - - inputBundle, err := certutil.ParsePEMBundle(cert) - if err != nil { - switch err.(type) { - case errutil.InternalError: - return nil, err - default: - return logical.ErrorResponse(err.Error()), nil - } - } - - if inputBundle.Certificate == nil { - return logical.ErrorResponse("supplied certificate could not be successfully parsed"), nil - } - - cb := &certutil.CertBundle{} - entry, err := req.Storage.Get(ctx, "config/ca_bundle") - if err != nil { - return nil, err - } - if entry == nil { - return logical.ErrorResponse("could not find any existing entry with a private key"), nil - } - - err = entry.DecodeJSON(cb) - if err != nil { - return nil, err - } - - if len(cb.PrivateKey) == 0 || cb.PrivateKeyType == "" { - return logical.ErrorResponse("could not find an existing private key"), nil - } - - parsedCB, err := parseCABundle(ctx, b, req, cb) - if err != nil { - return nil, err - } - if parsedCB.PrivateKey == nil { - return nil, fmt.Errorf("saved key could not be parsed successfully") - } - - inputBundle.PrivateKey = parsedCB.PrivateKey - inputBundle.PrivateKeyType = parsedCB.PrivateKeyType - inputBundle.PrivateKeyBytes = parsedCB.PrivateKeyBytes - - if !inputBundle.Certificate.IsCA { - return logical.ErrorResponse("the given certificate is not marked for CA use and cannot be used with this backend"), nil - } - - if err := inputBundle.Verify(); err != nil { - return nil, fmt.Errorf("verification of parsed bundle failed: %w", err) - } - - cb, err = inputBundle.ToCertBundle() - if err != nil { - return nil, fmt.Errorf("error converting raw values into cert bundle: %w", err) - } - - entry, err = logical.StorageEntryJSON("config/ca_bundle", cb) - if err != nil { - return nil, err - } - err = req.Storage.Put(ctx, entry) - if err != nil { - return nil, err - } - - entry.Key = "certs/" + normalizeSerial(cb.SerialNumber) - entry.Value = inputBundle.CertificateBytes - err = req.Storage.Put(ctx, entry) - if err != nil { - return nil, err - } - - // For ease of later use, also store just the certificate at a known - // location - entry.Key = "ca" - entry.Value = inputBundle.CertificateBytes - err = req.Storage.Put(ctx, entry) - if err != nil { - return nil, err - } - - // Build a fresh CRL - err = buildCRL(ctx, b, req, true) - - return nil, err -} - const pathGenerateIntermediateHelpSyn = ` Generate a new CSR and private key used for signing. ` diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index eb04a21ef48aa..229b4927b00df 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -93,16 +93,33 @@ secret-key (optional) and certificates.`, } func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - keysAllowed := strings.HasSuffix(req.Path, "bundle") + keysAllowed := strings.HasSuffix(req.Path, "bundle") || req.Path == "config/ca" + + var pemBundle string + var certificate string + rawPemBundle, bundleOk := data.GetOk("pem_bundle") + rawCertificate, certOk := data.GetOk("certificate") + if bundleOk { + pemBundle = rawPemBundle.(string) + } + if certOk { + certificate = rawCertificate.(string) + } - pemBundle := data.Get("pem_bundle").(string) - if len(pemBundle) == 0 { - return logical.ErrorResponse("'pem_bundle' parameter was empty"), nil + if len(pemBundle) == 0 && len(certificate) == 0 { + return logical.ErrorResponse("'pem_bundle' and 'certificate' parameters were empty"), nil + } + if len(pemBundle) > 0 && len(certificate) > 0 { + return logical.ErrorResponse("'pem_bundle' and 'certificate' parameters were both provided"), nil + } + if len(certificate) > 0 { + keysAllowed = false + pemBundle = certificate } - var createdKeys []keyId - var createdIssuers []issuerId - issuerKeyMap := make(map[issuerId]keyId) + var createdKeys []string + var createdIssuers []string + issuerKeyMap := make(map[string]string) // Rather than using certutil.ParsePEMBundle (which restricts the // construction of the PEM bundle), we manually parse the bundle instead. @@ -144,7 +161,7 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d } if !existing { - createdKeys = append(createdKeys, key.ID) + createdKeys = append(createdKeys, key.ID.String()) } } @@ -154,9 +171,9 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d return logical.ErrorResponse(err.Error()), nil } - issuerKeyMap[cert.ID] = cert.KeyID + issuerKeyMap[cert.ID.String()] = cert.KeyID.String() if !existing { - createdIssuers = append(createdIssuers, cert.ID) + createdIssuers = append(createdIssuers, cert.ID.String()) } } From 0a30e5094a8d50cbe8b64391c8d440e7ca2efd6a Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 11 Apr 2022 15:47:02 -0400 Subject: [PATCH 26/76] Clarify error message on missing defaults When the default issuer and key are missing (and haven't yet been specified), we should clarify that error message. Signed-off-by: Alexander Scheel --- builtin/logical/pki/storage.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index e341f1fb8817c..bf78660661ada 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -85,6 +85,10 @@ func listKeys(ctx context.Context, s logical.Storage) ([]keyId, error) { } func fetchKeyById(ctx context.Context, s logical.Storage, keyId keyId) (*key, error) { + if len(keyId) == 0 { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki key: empty key identifier")} + } + keyEntry, err := s.Get(ctx, keyPrefix+keyId.String()) if err != nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki key: %v", err)} @@ -259,6 +263,9 @@ func resolveKeyReference(ctx context.Context, s logical.Storage, reference strin if err != nil { return keyId("config-error"), err } + if len(config.DefaultKeyId) == 0 { + return KeyRefNotFound, fmt.Errorf("no default key currently configured") + } return config.DefaultKeyId, nil } @@ -292,6 +299,10 @@ func resolveKeyReference(ctx context.Context, s logical.Storage, reference strin } func fetchIssuerById(ctx context.Context, s logical.Storage, issuerId issuerId) (*issuer, error) { + if len(issuerId) == 0 { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki issuer: empty issuer identifier")} + } + issuerEntry, err := s.Get(ctx, issuerPrefix+issuerId.String()) if err != nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki issuer: %v", err)} @@ -489,6 +500,9 @@ func resolveIssuerReference(ctx context.Context, s logical.Storage, reference st if err != nil { return issuerId("config-error"), err } + if len(config.DefaultIssuerId) == 0 { + return IssuerRefNotFound, fmt.Errorf("no default issuer currently configured") + } return config.DefaultIssuerId, nil } From bfd41cf84ca4afd0fe76839a4b7b4be08f2cec4b Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 11 Apr 2022 17:00:19 -0400 Subject: [PATCH 27/76] Update test semantics for new changes This makes two minor changes to the existing test suite: 1. Importing partial bundles should now succeed, where they'd previously error. 2. fetchCertBySerial no longer handles CA certificates. Signed-off-by: Alexander Scheel --- builtin/logical/pki/ca_test.go | 18 +++++++++--------- builtin/logical/pki/cert_util_test.go | 7 ------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/builtin/logical/pki/ca_test.go b/builtin/logical/pki/ca_test.go index c1ba77cbde419..b74026b131a72 100644 --- a/builtin/logical/pki/ca_test.go +++ b/builtin/logical/pki/ca_test.go @@ -257,13 +257,13 @@ func runSteps(t *testing.T, rootB, intB *backend, client *api.Client, rootName, // Load CA cert/key in and ensure we can fetch it back in various formats, // unauthenticated { - // Attempt import but only provide one the cert + // Attempt import but only provide one the cert; this should work. { _, err := client.Logical().Write(rootName+"config/ca", map[string]interface{}{ "pem_bundle": caCert, }) - if err == nil { - t.Fatal("expected error") + if err != nil { + t.Fatalf("unexpected error: %v", err) } } @@ -272,18 +272,18 @@ func runSteps(t *testing.T, rootB, intB *backend, client *api.Client, rootName, _, err := client.Logical().Write(rootName+"config/ca", map[string]interface{}{ "pem_bundle": caKey, }) - if err == nil { - t.Fatal("expected error") + if err != nil { + t.Fatalf("unexpected error: %v", err) } } - // Import CA bundle + // Import entire CA bundle; this should work as well { _, err := client.Logical().Write(rootName+"config/ca", map[string]interface{}{ "pem_bundle": strings.Join([]string{caKey, caCert}, "\n"), }) if err != nil { - t.Fatal(err) + t.Fatalf("unexpected error: %v", err) } } @@ -464,8 +464,8 @@ func runSteps(t *testing.T, rootB, intB *backend, client *api.Client, rootName, if err != nil { t.Fatal(err) } - if resp != nil { - t.Fatal("expected nil response") + if resp == nil { + t.Fatal("nil response") } } diff --git a/builtin/logical/pki/cert_util_test.go b/builtin/logical/pki/cert_util_test.go index 2d8dd04dd2419..b3507613302e8 100644 --- a/builtin/logical/pki/cert_util_test.go +++ b/builtin/logical/pki/cert_util_test.go @@ -92,13 +92,6 @@ func TestPki_FetchCertBySerial(t *testing.T) { Prefix string Serial string }{ - "ca": { - &logical.Request{ - Storage: storage, - }, - "", - "ca", - }, "crl": { &logical.Request{ Storage: storage, From 547012f9d9751a45c1896a1903ec4e66bd2cd70b Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 12 Apr 2022 11:09:49 -0400 Subject: [PATCH 28/76] Add support for deleting all keys, issuers The old DELETE /root code must now delete all keys and issuers for backwards compatibility. We strongly suggest calling individual delete methods (DELETE /key/:key_ref or DELETE /issuer/:issuer_ref) instead, for finer control. In the process, we detect whether the deleted key/issuers was set as the default. This will allow us to warn (from the single key/deletion issuer code) whether or not the default was deleted (while allowing the operation to succeed). Signed-off-by: Alexander Scheel --- builtin/logical/pki/path_fetch_issuers.go | 14 ++++++++- builtin/logical/pki/path_root.go | 34 +++++++++++++++++++- builtin/logical/pki/storage.go | 38 ++++++++++++++++++++--- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 6f15761d31fb3..77650b43f8c6a 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -3,6 +3,7 @@ package pki import ( "context" "encoding/pem" + "fmt" "regexp" "strings" @@ -222,7 +223,18 @@ func (b *backend) pathDeleteIssuer(ctx context.Context, req *logical.Request, da return logical.ErrorResponse("unable to resolve issuer id for reference: " + issuerName), nil } - return nil, deleteIssuer(ctx, req.Storage, ref) + wasDefault, err := deleteIssuer(ctx, req.Storage, ref) + if err != nil { + return nil, err + } + + var response *logical.Response + if wasDefault { + response = &logical.Response{} + response.AddWarning(fmt.Sprintf("Deleted issuer %v (via issuer_ref %v); this was configured as the default issuer. Operations without an explicit issuer will not work until a new default is configured.", ref, issuerName)) + } + + return response, nil } const ( diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 41b2734cea2b9..fedefc9ef56ad 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -49,7 +49,39 @@ func pathDeleteRoot(b *backend) *framework.Path { } func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - return nil, req.Storage.Delete(ctx, "config/ca_bundle") + issuers, err := listIssuers(ctx, req.Storage) + if err != nil { + return nil, err + } + + keys, err := listKeys(ctx, req.Storage) + if err != nil { + return nil, err + } + + // Delete all issuers and keys. Ignore deleting the default since we're + // explicitly deleting everything. + for _, issuer := range issuers { + if _, err = deleteIssuer(ctx, req.Storage, issuer); err != nil { + return nil, err + } + } + for _, key := range keys { + if _, err = deleteKey(ctx, req.Storage, key); err != nil { + return nil, err + } + } + + // Delete legacy CA bundle; but don't error if it doesn't exist. + if err := req.Storage.Delete(ctx, legacyCertBundlePath); err != nil { + return nil, err + } + + // Return a warning about preferring to delete issuers and keys + // explicitly versus deleting everything. + resp := &logical.Response{} + resp.AddWarning("DELETE /root deletes all keys and issuers; prefer the new DELETE /key/:key_ref and DELETE /issuer/:issuer_ref for finer granularity, unless removal of all keys and issuers is desired.") + return resp, nil } func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index bf78660661ada..60185cf5c2fb3 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -117,8 +117,23 @@ func writeKey(ctx context.Context, s logical.Storage, key key) error { return s.Put(ctx, json) } -func deleteKey(ctx context.Context, s logical.Storage, id keyId) error { - return s.Delete(ctx, keyPrefix+id.String()) +func deleteKey(ctx context.Context, s logical.Storage, id keyId) (bool, error) { + wasDefault := false + + config, err := getKeysConfig(ctx, s) + if err != nil { + return wasDefault, err + } + + if config.DefaultKeyId == id { + wasDefault = true + config.DefaultKeyId = keyId("") + if err := setKeysConfig(ctx, s, config); err != nil { + return wasDefault, err + } + } + + return wasDefault, s.Delete(ctx, keyPrefix+id.String()) } func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName string) (*key, bool, error) { @@ -331,8 +346,23 @@ func writeIssuer(ctx context.Context, s logical.Storage, issuer *issuer) error { return s.Put(ctx, json) } -func deleteIssuer(ctx context.Context, s logical.Storage, id issuerId) error { - return s.Delete(ctx, issuerPrefix+id.String()) +func deleteIssuer(ctx context.Context, s logical.Storage, id issuerId) (bool, error) { + wasDefault := false + + config, err := getIssuersConfig(ctx, s) + if err != nil { + return wasDefault, err + } + + if config.DefaultIssuerId == id { + wasDefault = true + config.DefaultIssuerId = issuerId("") + if err := setIssuersConfig(ctx, s, config); err != nil { + return wasDefault, err + } + } + + return wasDefault, s.Delete(ctx, issuerPrefix+id.String()) } func importIssuer(ctx context.Context, s logical.Storage, certValue string, issuerName string) (*issuer, bool, error) { From f34f024d62a8437cce59507a2e62f61ba5d2fe53 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Tue, 12 Apr 2022 09:50:08 -0400 Subject: [PATCH 29/76] Introduce defaultRef constant within PKI - Replace hardcoded "default" references with a constant to easily identify various usages. - Use the addIssuerRefField function instead of redefining the field in various locations. --- builtin/logical/pki/backend_test.go | 2 +- builtin/logical/pki/crl_util.go | 4 +-- builtin/logical/pki/fields.go | 4 +-- builtin/logical/pki/path_config_ca.go | 2 +- builtin/logical/pki/path_fetch.go | 2 +- builtin/logical/pki/path_fetch_issuers.go | 3 -- builtin/logical/pki/path_sign_issuers.go | 40 +++++++++-------------- builtin/logical/pki/storage.go | 4 +-- builtin/logical/pki/util.go | 12 ++++--- 9 files changed, 32 insertions(+), 41 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index fe40ff0328771..d3f4a8b670a0a 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -2584,7 +2584,7 @@ func TestBackend_SignSelfIssued(t *testing.T) { t.Fatal(err) } - signingBundle, err := fetchCAInfo(context.Background(), b, &logical.Request{Storage: storage}, "default") + signingBundle, err := fetchCAInfo(context.Background(), b, &logical.Request{Storage: storage}, defaultRef) if err != nil { t.Fatal(err) } diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index 27b72270eb416..2b656f58fc63d 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -32,7 +32,7 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st return nil, nil } - signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") + signingBundle, caErr := fetchCAInfo(ctx, b, req, defaultRef) if caErr != nil { switch caErr.(type) { case errutil.UserError: @@ -223,7 +223,7 @@ func buildCRL(ctx context.Context, b *backend, req *logical.Request, forceNew bo } WRITE: - signingBundle, caErr := fetchCAInfo(ctx, b, req, "default") + signingBundle, caErr := fetchCAInfo(ctx, b, req, defaultRef) if caErr != nil { switch caErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/fields.go b/builtin/logical/pki/fields.go index afd8723964538..09a6eace39308 100644 --- a/builtin/logical/pki/fields.go +++ b/builtin/logical/pki/fields.go @@ -363,7 +363,7 @@ func addIssuerRefField(fields map[string]*framework.FieldSchema) map[string]*fra Description: `Reference to a existing issuer; either "default" for the configured default issuer, an identifier or the name assigned to the issuer.`, - Default: "default", + Default: defaultRef, } return fields } @@ -391,7 +391,7 @@ func addKeyRefField(fields map[string]*framework.FieldSchema) map[string]*framew Description: `Reference to a existing key; either "default" for the configured default key, an identifier or the name assigned to the key.`, - Default: "default", + Default: defaultRef, } return fields } diff --git a/builtin/logical/pki/path_config_ca.go b/builtin/logical/pki/path_config_ca.go index 6b4590344966b..b70292d8e77ed 100644 --- a/builtin/logical/pki/path_config_ca.go +++ b/builtin/logical/pki/path_config_ca.go @@ -74,7 +74,7 @@ func (b *backend) pathCAIssuersRead(ctx context.Context, req *logical.Request, d func (b *backend) pathCAIssuersWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { newDefault := data.Get("default").(string) - if len(newDefault) == 0 || newDefault == "default" { + if len(newDefault) == 0 || newDefault == defaultRef { return logical.ErrorResponse("Invalid issuer specification; must be non-empty and can't be 'default'."), nil } diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index 7735f178699f2..c7a5d8d0e2e21 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -192,7 +192,7 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data // Prefer fetchCAInfo to fetchCertBySerial for CA certificates. if serial == "ca_chain" || serial == "ca" { - caInfo, err := fetchCAInfo(ctx, b, req, "default") + caInfo, err := fetchCAInfo(ctx, b, req, defaultRef) if err != nil { switch err.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 77650b43f8c6a..86d5aa49c0300 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -4,15 +4,12 @@ import ( "context" "encoding/pem" "fmt" - "regexp" "strings" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/logical" ) -var nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex(issuerRefParam) + "$") - func pathListIssuers(b *backend) *framework.Path { return &framework.Path{ Pattern: "issuers/?$", diff --git a/builtin/logical/pki/path_sign_issuers.go b/builtin/logical/pki/path_sign_issuers.go index 7afa1c752f7a5..e54e45276e8f3 100644 --- a/builtin/logical/pki/path_sign_issuers.go +++ b/builtin/logical/pki/path_sign_issuers.go @@ -16,16 +16,10 @@ func pathSignIntermediate(b *backend) *framework.Path { } func pathIssuerSignIntermediateRaw(b *backend, pattern string) *framework.Path { + fields := addIssuerRefField(map[string]*framework.FieldSchema{}) path := &framework.Path{ Pattern: pattern, - Fields: map[string]*framework.FieldSchema{ - issuerRefParam: { - Type: framework.TypeString, - Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, - Default: "default", - }, - }, - + Fields: fields, Callbacks: map[logical.Operation]framework.OperationFunc{ logical.UpdateOperation: b.pathIssuerSignIntermediate, }, @@ -89,25 +83,21 @@ func pathSignSelfIssued(b *backend) *framework.Path { } func buildPathIssuerSignSelfIssued(b *backend, pattern string) *framework.Path { + fields := map[string]*framework.FieldSchema{ + "certificate": { + Type: framework.TypeString, + Description: `PEM-format self-issued certificate to be signed.`, + }, + "require_matching_certificate_algorithms": { + Type: framework.TypeBool, + Default: false, + Description: `If true, require the public key algorithm of the signer to match that of the self issued certificate.`, + }, + } + fields = addIssuerRefField(fields) path := &framework.Path{ Pattern: pattern, - Fields: map[string]*framework.FieldSchema{ - issuerRefParam: { - Type: framework.TypeString, - Description: `Reference to issuer; either "default" for the configured default issuer, an identifier of an issuer, or the name assigned to the issuer.`, - Default: "default", - }, - "certificate": { - Type: framework.TypeString, - Description: `PEM-format self-issued certificate to be signed.`, - }, - "require_matching_certificate_algorithms": { - Type: framework.TypeBool, - Default: false, - Description: `If true, require the public key algorithm of the signer to match that of the self issued certificate.`, - }, - }, - + Fields: fields, Callbacks: map[logical.Operation]framework.OperationFunc{ logical.UpdateOperation: b.pathIssuerSignSelfIssued, }, diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 60185cf5c2fb3..4c39ef723ba90 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -272,7 +272,7 @@ func listIssuers(ctx context.Context, s logical.Storage) ([]issuerId, error) { } func resolveKeyReference(ctx context.Context, s logical.Storage, reference string) (keyId, error) { - if reference == "default" { + if reference == defaultRef { // Handle fetching the default key. config, err := getKeysConfig(ctx, s) if err != nil { @@ -524,7 +524,7 @@ func getIssuersConfig(ctx context.Context, s logical.Storage) (*issuerConfig, er } func resolveIssuerReference(ctx context.Context, s logical.Storage, reference string) (issuerId, error) { - if reference == "default" { + if reference == defaultRef { // Handle fetching the default issuer. config, err := getIssuersConfig(ctx, s) if err != nil { diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index cabf9f0b5adae..62fc1d0e3c641 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -3,6 +3,7 @@ package pki import ( "context" "fmt" + "regexp" "strings" "github.com/hashicorp/vault/sdk/logical" @@ -15,8 +16,11 @@ import ( const ( managedKeyNameArg = "managed_key_name" managedKeyIdArg = "managed_key_id" + defaultRef = "default" ) +var nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex(issuerRefParam) + "$") + func normalizeSerial(serial string) string { return strings.Replace(strings.ToLower(serial), ":", "-", -1) } @@ -129,7 +133,7 @@ func getIssuerName(ctx context.Context, s logical.Storage, data *framework.Field if ok { issuerName = strings.TrimSpace(issuerNameIface.(string)) - if strings.ToLower(issuerName) == "default" { + if strings.ToLower(issuerName) == defaultRef { return "", errutil.UserError{Err: "reserved keyword 'default' can not be used as issuer name"} } @@ -154,7 +158,7 @@ func getKeyName(ctx context.Context, s logical.Storage, data *framework.FieldDat if ok { keyName = strings.TrimSpace(keyNameIface.(string)) - if strings.ToLower(keyName) == "default" { + if strings.ToLower(keyName) == defaultRef { return "", errutil.UserError{Err: "reserved keyword 'default' can not be used as key name"} } @@ -183,8 +187,8 @@ func getKeyRef(data *framework.FieldData) string { func extractRef(data *framework.FieldData, paramName string) string { value := strings.TrimSpace(data.Get(paramName).(string)) - if strings.ToLower(value) == "default" { - return "default" + if strings.ToLower(value) == defaultRef { + return defaultRef } return value } From 705d64f33e774399d5c432a6c98bda0364dbfd03 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Tue, 12 Apr 2022 18:19:27 -0400 Subject: [PATCH 30/76] Rework PKI test TestBackend_Root_Idempotency - Validate that generate/root calls are no longer idempotent, but the bundle importing does not generate new keys/issuers - As before make sure that the delete root api resets everything - Address a bug within the storage that we bombed when we had multiple different key types within storage. --- builtin/logical/pki/backend_test.go | 137 +++++++++++++++------------ builtin/logical/pki/storage.go | 4 +- sdk/helper/certutil/certutil_test.go | 67 +++++++++++++ sdk/helper/certutil/helpers.go | 16 +++- 4 files changed, 161 insertions(+), 63 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index d3f4a8b670a0a..f9152dfdcc328 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -2276,90 +2276,107 @@ func TestBackend_Root_Idempotency(t *testing.T) { cluster.Start() defer cluster.Cleanup() client := cluster.Cores[0].Client - var err error - err = client.Sys().Mount("pki", &api.MountInput{ - Type: "pki", - Config: api.MountConfigInput{ - DefaultLeaseTTL: "16h", - MaxLeaseTTL: "32h", - }, - }) - if err != nil { - t.Fatal(err) - } + mountPKIEndpoint(t, client, "pki") + + // This is a change within 1.11, we are no longer idempotent across generate/internal calls. resp, err := client.Logical().Write("pki/root/generate/internal", map[string]interface{}{ "common_name": "myvault.com", }) - if err != nil { - t.Fatal(err) - } - if resp == nil { - t.Fatal("expected ca info") - } + require.NoError(t, err) + require.NotNil(t, resp, "expected ca info") + keyId1 := resp.Data["key_id"] + issuerId1 := resp.Data["issuer_id"] + resp, err = client.Logical().Read("pki/cert/ca_chain") - if err != nil { - t.Fatalf("error reading ca_chain: %v", err) - } + require.NoError(t, err, "error reading ca_chain: %v", err) r1Data := resp.Data - // Try again, make sure it's a 204 and same CA + // Calling generate/internal should generate a new CA as well. resp, err = client.Logical().Write("pki/root/generate/internal", map[string]interface{}{ "common_name": "myvault.com", }) - if err != nil { - t.Fatal(err) - } - if resp == nil { - t.Fatal("expected a warning") - } - if resp.Data != nil || len(resp.Warnings) == 0 { - t.Fatalf("bad response: %#v", *resp) - } + require.NoError(t, err) + require.NotNil(t, resp, "expected ca info") + keyId2 := resp.Data["key_id"] + issuerId2 := resp.Data["issuer_id"] + + // Make sure that we actually generated different issuer and key values + require.NotEqual(t, keyId1, keyId2) + require.NotEqual(t, issuerId1, issuerId2) + + // Now because the issued CA's have no links, the call to ca_chain should return the same data (ca chain from default) resp, err = client.Logical().Read("pki/cert/ca_chain") - if err != nil { - t.Fatalf("error reading ca_chain: %v", err) - } + require.NoError(t, err, "error reading ca_chain: %v", err) + r2Data := resp.Data if !reflect.DeepEqual(r1Data, r2Data) { t.Fatal("got different ca certs") } + // Now let's validate that the import bundle is idempotent. + pemBundleRootCA := string(cluster.CACertPEM) + string(cluster.CAKeyPEM) + resp, err = client.Logical().Write("pki/config/ca", map[string]interface{}{ + "pem_bundle": pemBundleRootCA, + }) + require.NoError(t, err) + require.NotNil(t, resp, "expected ca info") + firstImportedKeys := resp.Data["imported_keys"].([]interface{}) + firstImportedIssuers := resp.Data["imported_issuers"].([]interface{}) + + require.NotContains(t, firstImportedKeys, keyId1) + require.NotContains(t, firstImportedKeys, keyId2) + require.NotContains(t, firstImportedIssuers, issuerId1) + require.NotContains(t, firstImportedIssuers, issuerId2) + + // Performing this again should result in no key/issuer ids being imported/generated. + resp, err = client.Logical().Write("pki/config/ca", map[string]interface{}{ + "pem_bundle": pemBundleRootCA, + }) + require.NoError(t, err) + require.NotNil(t, resp, "expected ca info") + secondImportedKeys := resp.Data["imported_keys"] + secondImportedIssuers := resp.Data["imported_issuers"] + + require.Nil(t, secondImportedKeys) + require.Nil(t, secondImportedIssuers) + resp, err = client.Logical().Delete("pki/root") - if err != nil { - t.Fatal(err) - } - if resp != nil { - t.Fatal("expected nil response") - } - // Make sure it behaves the same + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, 1, len(resp.Warnings)) + + // Make sure we can delete twice... resp, err = client.Logical().Delete("pki/root") - if err != nil { - t.Fatal(err) - } - if resp != nil { - t.Fatal("expected nil response") - } + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, 1, len(resp.Warnings)) _, err = client.Logical().Read("pki/cert/ca_chain") - if err == nil { - t.Fatal("expected error") - } + require.Error(t, err, "expected an error fetching deleted ca_chain") - resp, err = client.Logical().Write("pki/root/generate/internal", map[string]interface{}{ - "common_name": "myvault.com", + // We should be able to import the same ca bundle as before and get a different key/issuer ids + resp, err = client.Logical().Write("pki/config/ca", map[string]interface{}{ + "pem_bundle": pemBundleRootCA, }) - if err != nil { - t.Fatal(err) - } - if resp == nil { - t.Fatal("expected ca info") - } + require.NoError(t, err) + require.NotNil(t, resp, "expected ca info") + postDeleteImportedKeys := resp.Data["imported_keys"] + postDeleteImportedIssuers := resp.Data["imported_issuers"] - _, err = client.Logical().Read("pki/cert/ca_chain") - if err != nil { - t.Fatal(err) + // Make sure that we actually generated different issuer and key values, then the previous import + require.NotNil(t, postDeleteImportedKeys) + require.NotNil(t, postDeleteImportedIssuers) + require.NotEqual(t, postDeleteImportedKeys, firstImportedKeys) + require.NotEqual(t, postDeleteImportedIssuers, firstImportedIssuers) + + resp, err = client.Logical().Read("pki/cert/ca_chain") + require.NoError(t, err) + + caChainPostDelete := resp.Data + if reflect.DeepEqual(r1Data, caChainPostDelete) { + t.Fatal("ca certs from ca_chain were the same post delete, should have changed.") } } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 4c39ef723ba90..0d5d1dbef0a98 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -217,7 +217,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName return nil, false, err } - equal, err := certutil.ComparePublicKeys(cert.PublicKey, keyPublic) + equal, err := certutil.ComparePublicKeysAndType(cert.PublicKey, keyPublic) if err != nil { return nil, false, err } @@ -437,7 +437,7 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu return nil, false, err } - equal, err := certutil.ComparePublicKeys(issuerCert.PublicKey, signer.Public()) + equal, err := certutil.ComparePublicKeysAndType(issuerCert.PublicKey, signer.Public()) if err != nil { return nil, false, err } diff --git a/sdk/helper/certutil/certutil_test.go b/sdk/helper/certutil/certutil_test.go index 83d32988678b6..0b5b74ddb0df2 100644 --- a/sdk/helper/certutil/certutil_test.go +++ b/sdk/helper/certutil/certutil_test.go @@ -2,6 +2,7 @@ package certutil import ( "bytes" + "crypto" "crypto/ecdsa" "crypto/ed25519" "crypto/elliptic" @@ -853,6 +854,72 @@ func setCerts() { issuingCaChainPem = []string{intCertPEM, caCertPEM} } +func TestComparePublicKeysAndType(t *testing.T) { + rsa1 := genRsaKey(t).Public() + rsa2 := genRsaKey(t).Public() + eddsa1 := genEdDSA(t).Public() + eddsa2 := genEdDSA(t).Public() + ed25519_1, _ := genEd25519Key(t) + ed25519_2, _ := genEd25519Key(t) + + type args struct { + key1Iface crypto.PublicKey + key2Iface crypto.PublicKey + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + {name: "RSA_Equal", args: args{key1Iface: rsa1, key2Iface: rsa1}, want: true, wantErr: false}, + {name: "RSA_NotEqual", args: args{key1Iface: rsa1, key2Iface: rsa2}, want: false, wantErr: false}, + {name: "EDDSA_Equal", args: args{key1Iface: eddsa1, key2Iface: eddsa1}, want: true, wantErr: false}, + {name: "EDDSA_NotEqual", args: args{key1Iface: eddsa1, key2Iface: eddsa2}, want: false, wantErr: false}, + {name: "ED25519_Equal", args: args{key1Iface: ed25519_1, key2Iface: ed25519_1}, want: true, wantErr: false}, + {name: "ED25519_NotEqual", args: args{key1Iface: ed25519_1, key2Iface: ed25519_2}, want: false, wantErr: false}, + {name: "Mismatched_RSA", args: args{key1Iface: rsa1, key2Iface: ed25519_2}, want: false, wantErr: false}, + {name: "Mismatched_EDDSA", args: args{key1Iface: ed25519_1, key2Iface: rsa1}, want: false, wantErr: false}, + {name: "Mismatched_ED25519", args: args{key1Iface: ed25519_1, key2Iface: rsa1}, want: false, wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ComparePublicKeysAndType(tt.args.key1Iface, tt.args.key2Iface) + if (err != nil) != tt.wantErr { + t.Errorf("ComparePublicKeysAndType() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("ComparePublicKeysAndType() got = %v, want %v", got, tt.want) + } + }) + } +} + +func genRsaKey(t *testing.T) *rsa.PrivateKey { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + return key +} + +func genEdDSA(t *testing.T) *ecdsa.PrivateKey { + key, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + t.Fatal(err) + } + return key +} + +func genEd25519Key(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) { + key, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return key, priv +} + var ( initTest sync.Once privRSA8KeyPem string diff --git a/sdk/helper/certutil/helpers.go b/sdk/helper/certutil/helpers.go index 6d415bbadfd34..819d96bb30f89 100644 --- a/sdk/helper/certutil/helpers.go +++ b/sdk/helper/certutil/helpers.go @@ -352,7 +352,21 @@ func generateSerialNumber(randReader io.Reader) (*big.Int, error) { return serial, nil } -// ComparePublicKeys compares two public keys and returns true if they match +// ComparePublicKeysAndType compares two public keys and returns true if they match, +// false if their types or contents differ, and an error on unsupported key types. +func ComparePublicKeysAndType(key1Iface, key2Iface crypto.PublicKey) (bool, error) { + equal, err := ComparePublicKeys(key1Iface, key2Iface) + if err != nil { + if strings.Contains(err.Error(), "key types do not match:") { + return false, nil + } + } + + return equal, err +} + +// ComparePublicKeys compares two public keys and returns true if they match, +// returns an error if public key types are mismatched, or they are an unsupported key type. func ComparePublicKeys(key1Iface, key2Iface crypto.PublicKey) (bool, error) { switch key1Iface.(type) { case *rsa.PublicKey: From 12131b3254bf3ec9901ba3ffa9690218e8261eac Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Wed, 13 Apr 2022 10:46:53 -0400 Subject: [PATCH 31/76] Assign Name=current to migrated key and issuer - Detail I missed from the RFC was to assign the Name field as "current" for migrated key and issuer. --- builtin/logical/pki/storage_migrations.go | 2 +- builtin/logical/pki/storage_migrations_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index 23c1b23997001..2d2e9b818180e 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -50,7 +50,7 @@ func migrateStorage(ctx context.Context, req *logical.InitializationRequest, log logger.Warn("performing PKI migration to new keys/issuers layout") - anIssuer, aKey, err := writeCaBundle(ctx, s, legacyBundle, "", "") + anIssuer, aKey, err := writeCaBundle(ctx, s, legacyBundle, "current", "current") if err != nil { return err } diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index 1129319b4fd9b..bda3fb70cc951 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -60,9 +60,11 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { keyId := keyIds[0] issuer, err := fetchIssuerById(ctx, s, issuerId) require.NoError(t, err) + require.Equal(t, "current", issuer.Name) // RFC says we should import with Name=current key, err := fetchKeyById(ctx, s, keyId) require.NoError(t, err) + require.Equal(t, "current", key.Name) // RFC says we should import with Name=current require.Equal(t, issuerId, issuer.ID) require.Equal(t, bundle.SerialNumber, issuer.SerialNumber) From 15460156fc7be0c0bc28b2fb4c11b02ad96a6247 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Wed, 13 Apr 2022 11:28:42 -0400 Subject: [PATCH 32/76] Build CRL upon PKI intermediary set-signed api called - Add a call to buildCRL if we created an issuer within pathImportIssuers - Augment existing FullCAChain to verify we have a proper CRL post set-signed api call - Remove a code block writing out "ca" storage entry that is no longer used. --- builtin/logical/pki/backend_test.go | 48 +++++++++------------- builtin/logical/pki/path_manage_issuers.go | 7 ++++ builtin/logical/pki/path_root.go | 10 ----- 3 files changed, 27 insertions(+), 38 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index f9152dfdcc328..9dd2850676e39 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -3855,25 +3855,7 @@ func TestBackend_RevokePlusTidy_Intermediate(t *testing.T) { // Get CRL and ensure the tidied cert is still in the list after the tidy // operation since it's not past the NotAfter (ttl) value yet. - req := client.NewRequest("GET", "/v1/pki/crl") - resp, err := client.RawRequest(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - crlBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatalf("err: %s", err) - } - if len(crlBytes) == 0 { - t.Fatalf("expected CRL in response body") - } - - crl, err := x509.ParseDERCRL(crlBytes) - if err != nil { - t.Fatal(err) - } + crl := getParsedCrl(t, client, "pki") revokedCerts := crl.TBSCertList.RevokedCertificates if len(revokedCerts) == 0 { @@ -3986,14 +3968,24 @@ func TestBackend_RevokePlusTidy_Intermediate(t *testing.T) { } } - req = client.NewRequest("GET", "/v1/pki/crl") - resp, err = client.RawRequest(req) + crl = getParsedCrl(t, client, "pki") + + revokedCerts = crl.TBSCertList.RevokedCertificates + if len(revokedCerts) != 0 { + t.Fatal("expected CRL to be empty") + } +} + +func getParsedCrl(t *testing.T, client *api.Client, mountPoint string) *pkix.CertificateList { + path := fmt.Sprintf("/v1/%s/crl", mountPoint) + req := client.NewRequest("GET", path) + resp, err := client.RawRequest(req) if err != nil { t.Fatal(err) } defer resp.Body.Close() - crlBytes, err = ioutil.ReadAll(resp.Body) + crlBytes, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatalf("err: %s", err) } @@ -4001,15 +3993,11 @@ func TestBackend_RevokePlusTidy_Intermediate(t *testing.T) { t.Fatalf("expected CRL in response body") } - crl, err = x509.ParseDERCRL(crlBytes) + crl, err := x509.ParseDERCRL(crlBytes) if err != nil { t.Fatal(err) } - - revokedCerts = crl.TBSCertList.RevokedCertificates - if len(revokedCerts) != 0 { - t.Fatal("expected CRL to be empty") - } + return crl } func TestBackend_Root_FullCAChain(t *testing.T) { @@ -4141,6 +4129,10 @@ func runFullCAChainTest(t *testing.T, keyType string) { t.Fatal("expected intermediate chain information") } + // Verify we have a proper CRL now + crl := getParsedCrl(t, client, "pki-intermediate") + require.Equal(t, 0, len(crl.TBSCertList.RevokedCertificates)) + fullChain = resp.Data["ca_chain"].(string) if !strings.Contains(fullChain, intermediateCert) { t.Fatal("expected full chain to contain intermediate certificate") diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index 229b4927b00df..1bd6575c9791f 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -177,6 +177,13 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d } } + if len(createdIssuers) > 0 { + err := buildCRL(ctx, b, req, true) + if err != nil { + return nil, err + } + } + return &logical.Response{ Data: map[string]interface{}{ "mapping": issuerKeyMap, diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index fedefc9ef56ad..ae1a6369820f7 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -190,16 +190,6 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, return nil, fmt.Errorf("unable to store certificate locally: %w", err) } - // For ease of later use, also store just the certificate at a known - // location - err = req.Storage.Put(ctx, &logical.StorageEntry{ - Key: "ca", - Value: parsedBundle.CertificateBytes, - }) - if err != nil { - return nil, err - } - // Build a fresh CRL err = buildCRL(ctx, b, req, true) if err != nil { From d9c7f2a02a525a0e2b2765813356b8f8c8533fc5 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Fri, 15 Apr 2022 09:48:01 -0400 Subject: [PATCH 33/76] Identify which certificate or key failed When importing complex chains, we should identify in which certificate or key the failure occurred. Signed-off-by: Alexander Scheel --- builtin/logical/pki/path_manage_issuers.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index 1bd6575c9791f..f0b8c3f5e86c4 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/pem" + "fmt" "strings" "github.com/hashicorp/vault/sdk/framework" @@ -153,11 +154,11 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d return logical.ErrorResponse("private keys found in the PEM bundle but not allowed by the path; use /issuers/import/bundle"), nil } - for _, keyPem := range keys { + for keyIndex, keyPem := range keys { // Handle import of private key. key, existing, err := importKey(ctx, req.Storage, keyPem, "") if err != nil { - return logical.ErrorResponse(err.Error()), nil + return logical.ErrorResponse(fmt.Sprintf("Error parsing key %v: %v", keyIndex, err)), nil } if !existing { @@ -165,10 +166,10 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d } } - for _, certPem := range issuers { + for certIndex, certPem := range issuers { cert, existing, err := importIssuer(ctx, req.Storage, certPem, "") if err != nil { - return logical.ErrorResponse(err.Error()), nil + return logical.ErrorResponse(fmt.Sprintf("Error parsing issuer %v: %v\n%v", certIndex, err, certPem)), nil } issuerKeyMap[cert.ID.String()] = cert.KeyID.String() From fe3303794d500a904ae1652052ea23d69b1551e4 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Tue, 19 Apr 2022 10:32:29 -0400 Subject: [PATCH 34/76] PKI migration writes out empty migration log entry - Since the elements of the struct were not exported we serialized an empty migration log to disk and would re-run the migration --- builtin/logical/pki/storage_migrations.go | 26 +++++++++---------- .../logical/pki/storage_migrations_test.go | 18 +++++++++++++ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index 2d2e9b818180e..6082af90e5b68 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -16,6 +16,12 @@ import ( // and we need to perform it again... const latestMigrationVersion = 1 +type legacyBundleMigration struct { + Hash string `json:"hash" structs:"hash" mapstructure:"hash"` + Created time.Time `json:"created" structs:"created" mapstructure:"created"` + MigrationVersion int `json:"migrationVersion" structs:"migrationVersion" mapstructure:"migrationVersion"` +} + func migrateStorage(ctx context.Context, req *logical.InitializationRequest, logger log.Logger) error { s := req.Storage legacyBundle, err := getLegacyCertBundle(ctx, s) @@ -40,27 +46,27 @@ func migrateStorage(ctx context.Context, req *logical.InitializationRequest, log if migrationEntry != nil { // At this point we have already migrated something previously. - if migrationEntry.hash == hash && - migrationEntry.migrationVersion == latestMigrationVersion { + if migrationEntry.Hash == hash && + migrationEntry.MigrationVersion == latestMigrationVersion { // The hashes are the same, no need to try and re-import... logger.Debug("existing migration hash found and matched legacy bundle, skipping migration.") return nil } } - logger.Warn("performing PKI migration to new keys/issuers layout") + logger.Info("performing PKI migration to new keys/issuers layout") anIssuer, aKey, err := writeCaBundle(ctx, s, legacyBundle, "current", "current") if err != nil { return err } - logger.Info("Migration generated the following ids and set them as defaults", + logger.Debug("Migration generated the following ids and set them as defaults", "issuer id", anIssuer.ID, "key id", aKey.ID) err = setLegacyBundleMigrationLog(ctx, s, &legacyBundleMigration{ - hash: hash, - created: time.Now(), - migrationVersion: latestMigrationVersion, + Hash: hash, + Created: time.Now(), + MigrationVersion: latestMigrationVersion, }) if err != nil { return err @@ -84,12 +90,6 @@ func computeHashOfLegacyBundle(bundle *certutil.CertBundle) (string, error) { return hex.EncodeToString(hasher.Sum(nil)), nil } -type legacyBundleMigration struct { - hash string - created time.Time - migrationVersion int -} - func getLegacyBundleMigrationLog(ctx context.Context, s logical.Storage) (*legacyBundleMigration, error) { entry, err := s.Get(ctx, legacyMigrationBundleLogKey) if err != nil { diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index bda3fb70cc951..02936180eeeeb 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -2,7 +2,9 @@ package pki import ( "context" + "strings" "testing" + "time" "github.com/hashicorp/vault/sdk/logical" "github.com/stretchr/testify/require" @@ -30,6 +32,7 @@ func Test_migrateStorageEmptyStorage(t *testing.T) { } func Test_migrateStorageSimpleBundle(t *testing.T) { + startTime := time.Now() ctx := context.Background() b, s := createBackendWithStorage(t) @@ -55,6 +58,11 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { logEntry, err := getLegacyBundleMigrationLog(ctx, s) require.NoError(t, err) require.NotNil(t, logEntry) + require.Equal(t, latestMigrationVersion, logEntry.MigrationVersion) + require.True(t, len(strings.TrimSpace(logEntry.Hash)) > 0, + "Hash value (%s) should not have been empty", logEntry.Hash) + require.True(t, startTime.Before(logEntry.Created), + "created log entry time (%v) was before our start time(%v)?", logEntry.Created, startTime) issuerId := issuerIds[0] keyId := keyIds[0] @@ -89,4 +97,14 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { issuersConfig, err := getIssuersConfig(ctx, s) require.NoError(t, err) require.Equal(t, &issuerConfig{DefaultIssuerId: issuerId}, issuersConfig) + + // Make sure if we attempt to re-run the migration nothing happens... + err = migrateStorage(ctx, request, b.Logger()) + require.NoError(t, err) + logEntry2, err := getLegacyBundleMigrationLog(ctx, s) + require.NoError(t, err) + require.NotNil(t, logEntry2) + + require.Equal(t, logEntry.Created, logEntry2.Created) + require.Equal(t, logEntry.Hash, logEntry2.Hash) } From a25845ec9dc561575d7f11e3086804ceabd1ac68 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 12 Apr 2022 08:50:35 -0400 Subject: [PATCH 35/76] Add chain-building logic to PKI issuers path With the one-entry-per-issuer approach, CA Chains become implicitly constructed from the pool of issuers. This roughly matches the existing expectations from /config/ca (wherein a chain could be provided) and /intemediate/set-signed (where a chain may be provided). However, in both of those cases, we simply accepted a chain. Here, we need to be able to reconstruct the chain from parts on disk. However, with potential rotation of roots, we need to be aware of disparate chains. Simply concating together all issuers isn't sufficient. Thus we need to be able to parse a certificate's Issuer and Subject field and reconstruct valid (and potentially parallel) parent<->child mappings. This attempts to handle roots, intermediates, cross-signed intermediates, cross-signed roots, and rotated keys (wherein one might not have a valid signature due to changed key material with the same subject). Signed-off-by: Alexander Scheel --- builtin/logical/pki/chain_util.go | 1249 +++++++++++++++++++++++++++++ builtin/logical/pki/storage.go | 11 +- 2 files changed, 1257 insertions(+), 3 deletions(-) create mode 100644 builtin/logical/pki/chain_util.go diff --git a/builtin/logical/pki/chain_util.go b/builtin/logical/pki/chain_util.go new file mode 100644 index 0000000000000..c870339c09c55 --- /dev/null +++ b/builtin/logical/pki/chain_util.go @@ -0,0 +1,1249 @@ +package pki + +import ( + "bytes" + "context" + "crypto/x509" + "fmt" + "sort" + + "github.com/hashicorp/vault/sdk/logical" +) + +func prettyIssuer(issuerIdEntryMap map[issuerId]*issuer, issuer issuerId) string { + if entry, ok := issuerIdEntryMap[issuer]; ok && len(entry.Name) > 0 { + return "[id:" + string(issuer) + "/name:" + entry.Name + "]" + } + + return "[" + string(issuer) + "]" +} + +func rebuildIssuersChains(ctx context.Context, s logical.Storage, referenceCert *issuer /* optional */) error { + // This function rebuilds the CAChain field of all known issuers. This + // function should usually be invoked when a new issuer is added to the + // pool of issuers. + // + // In addition to the context and storage, we take an optional + // referenceCert parameter -- an issuer certificate that we should write + // to storage once done, but which might not be persisted yet (either due + // to new values on it or due to it not yet existing in the list). This is + // helpful when calling e.g., importIssuer(...) (from storage.go), to allow + // the newly imported issuer to have its CAChain field computed, but + // without writing and re-reading it from storage (potentially failing in + // the process if chain building failed). + // + // Our contract guarantees that, if referenceCert is provided, we'll write + // it to storage. Further, we guarantee that (given the issuers haven't + // changed), the results will be stable on multiple calls to rebuild the + // chain. + // + // Note that at no point in time do we fetch the private keys associated + // with any issuers. It is sufficient to merely look at the issuers + // themselves. + // + // To begin, we fetch all known issuers from disk. + issuers, err := listIssuers(ctx, s) + if err != nil { + return fmt.Errorf("unable to list issuers to build chain: %v", err) + } + + // Fast path: no issuers means we can set the reference cert's value, if + // provided, to itself. + if len(issuers) == 0 { + if referenceCert == nil { + // Nothing to do; no reference cert was provided. + return nil + } + + // Otherwise, the only entry in the chain (that we know about) is the + // certificate itself. + referenceCert.CAChain = []string{referenceCert.Certificate} + return writeIssuer(ctx, s, referenceCert) + } + + // Our provided reference cert might not be in the list of issuers. In + // that case, add it manually. + if referenceCert != nil { + missing := true + for _, issuer := range issuers { + if issuer == referenceCert.ID { + missing = false + break + } + } + + if missing { + issuers = append(issuers, referenceCert.ID) + } + } + + // Now call a stable sorting algorithm here. We want to ensure the results + // are the same across multiple calls to rebuildIssuersChains with the same + // input data. + sort.SliceStable(issuers, func(i, j int) bool { + return issuers[i] < issuers[j] + }) + + // We expect each of these maps to be the size of the number of issuers + // we have (as we're mapping from issuers to other values). + // + // The first caches the storage entry for the issuer, the second caches + // the parsed *x509.Certificate of the issuer itself, and the third and + // fourth maps that certificate back to the other issuers with that + // subject (note the keyword _other_: we'll exclude self-loops here) -- + // either via a parent or child relationship. + issuerIdEntryMap := make(map[issuerId]*issuer, len(issuers)) + issuerIdCertMap := make(map[issuerId]*x509.Certificate, len(issuers)) + issuerIdParentsMap := make(map[issuerId][]issuerId, len(issuers)) + issuerIdChildrenMap := make(map[issuerId][]issuerId, len(issuers)) + + // For every known issuer, we map that subject back to the id of issuers + // containing that subject. This lets us build our issuerId -> parents + // mapping efficiently. Worst case we'll have a single linear chain where + // every entry has a distinct subject. + subjectIssuerIdsMap := make(map[string][]issuerId, len(issuers)) + + // First, read every issuer entry from storage. We'll propagate entries + // to three of the maps here: all but issuerIdParentsMap and + // issuerIdChildrenMap, which we'll do in a second pass. + for _, identifier := range issuers { + var stored *issuer + + // When the reference issuer is provided and matches this identifier, + // prefer the updated reference copy instead. + if referenceCert != nil && identifier == referenceCert.ID { + stored = referenceCert + } else { + // Otherwise, fetch it from disk. + stored, err = fetchIssuerById(ctx, s, identifier) + if err != nil { + return fmt.Errorf("unable to fetch issuer %v to build chain: %v", identifier, err) + } + } + + if stored == nil || len(stored.Certificate) == 0 { + return fmt.Errorf("bad issuer while building chain: missing certificate entry: %v", identifier) + } + + issuerIdEntryMap[identifier] = stored + cert, err := stored.GetCertificate() + if err != nil { + return fmt.Errorf("unable to parse issuer %v to certificate to build chain: %v", identifier, err) + } + + issuerIdCertMap[identifier] = cert + subjectIssuerIdsMap[string(cert.RawSubject)] = append(subjectIssuerIdsMap[string(cert.RawSubject)], identifier) + } + + // Now that we have the subj->issuer map built, we can build the parent + // and child mappings. We iterate over all issuers and build it one step + // at a time. + // + // This is worst case O(n^2) because all of the issuers could have the + // same name and be self-signed certs with different keys. That makes the + // chain building (below) fast as they've all got empty parents/children + // maps. + // + // Note that the order of iteration is stable. Why? We've built + // subjectIssuerIdsMap from the (above) sorted issuers by appending the + // next entry to the present list; since they're already sorted, that + // lookup will also be sorted. Thus, each of these iterations are also + // in sorted order, so the resulting map entries (of ids) are also sorted. + // Thus, the graph structure is in sorted order and thus the toposort + // below will be stable. + for _, child := range issuers { + // Fetch the certificate as we'll need it later. + childCert := issuerIdCertMap[child] + + parentSubject := string(issuerIdCertMap[child].RawIssuer) + parentCerts, ok := subjectIssuerIdsMap[parentSubject] + if !ok { + // When the issuer isn't known to Vault, the lookup by the issuer + // will be empty. This most commonly occurs when intermediates are + // directly added (via intermediate/set-signed) without providing + // the root. + continue + } + + // Now, iterate over all possible parents and assign the child/parent + // relationship. + for _, parent := range parentCerts { + // Skip self-references to the exact same certificate. + if child == parent { + continue + } + + // While we could use Subject/Authority Key Identifier (SKI/AKI) + // as a heuristic for whether or not this relationship is valid, + // this is insufficient as otherwise valid CA certificates could + // elide this information. That means its best to actually validate + // the signature (e.g., call child.CheckSignatureFrom(parent)) + // instead. + parentCert := issuerIdCertMap[parent] + if err := childCert.CheckSignatureFrom(parentCert); err != nil { + // We cannot return an error here as it could be that this + // signature is entirely valid -- but just for a different + // key. Instead, skip adding the parent->child and + // child->parent link. + continue + } + + // Otherwise, we can append it to the map, allowing us to walk the + // issuer->parent mapping. + issuerIdParentsMap[child] = append(issuerIdParentsMap[child], parent) + + // Also cross-add the child relationship step at the same time. + issuerIdChildrenMap[parent] = append(issuerIdChildrenMap[parent], child) + } + } + + // Finally, we consult RFC 8446 Section 4.4.2 for creating an algorithm for + // building the chain: + // + // > ... The sender's certificate MUST come in the first + // > CertificateEntry in the list. Each following certificate SHOULD + // > directly certify the one immediately preceding it. Because + // > certificate validation requires that trust anchors be distributed + // > independently, a certificate that specifies a trust anchor MAY be + // > omitted from the chain, provided that supported peers are known to + // > possess any omitted certificates. + // > + // > Note: Prior to TLS 1.3, "certificate_list" ordering required each + // > certificate to certify the one immediately preceding it; however, + // > some implementations allowed some flexibility. Servers sometimes + // > send both a current and deprecated intermediate for transitional + // > purposes, and others are simply configured incorrectly, but these + // > cases can nonetheless be validated properly. For maximum + // > compatibility, all implementations SHOULD be prepared to handle + // > potentially extraneous certificates and arbitrary orderings from any + // > TLS version, with the exception of the end-entity certificate which + // > MUST be first. + // + // So, we take this to mean we should build chains via DFS: each issuer is + // explored until an empty parent pointer (i.e., self-loop) is reached and + // then the last most recently seen duplicate parent link is then explored. + // + // However, we don't actually need to do a DFS (per issuer) here. We can + // simply invert the (pseudo-)directed graph, i.e., topologically sort it. + // Some number of certs (roots without cross-signing) lack parent issuers. + // These are already "done" from the PoV of chain building. We can thus + // iterating through the parent mapping to find entries without parents to + // start the sort. After processing, we can add all children and visit them + // if all parents have been processed. + // + // Note though, that while topographical sorting is equivalent to the DFS, + // we have to take care to make it a pseudo-DAG. This means handling the + // most common 2-star (2-clique) sub-graphs of reissued certificates, + // manually building their chain prior to starting the topographical sort. + // + // This thus runs in O(|V| + |E|) -> O(n^2) in the number of issuers. + processedIssuers := make(map[issuerId]bool, len(issuers)) + toVisit := make([]issuerId, 0, len(issuers)) + + // Setup the toVisit queue. + for _, candidate := range issuers { + parentCerts, ok := issuerIdParentsMap[candidate] + if ok && len(parentCerts) > 0 { + // Assumption: no self-loops in the parent mapping, so if there's + // a non-empty parent mapping it means we can skip this node as + // it can't be processed yet. + continue + } + + // Because this candidate has no known parent issuers; update the + // list. + toVisit = append(toVisit, candidate) + } + + // If the queue is empty (and we know we have issuers), trigger the + // clique/cycle detection logic so we aren't starved for nodes. + if len(toVisit) == 0 { + toVisit, err = processAnyCliqueOrCycle(issuers, processedIssuers, toVisit, issuerIdEntryMap, issuerIdCertMap, issuerIdParentsMap, issuerIdChildrenMap, subjectIssuerIdsMap) + if err != nil { + return err + } + } + + // Now actually build the CAChain entries... Use a safety mechanism to + // ensure we don't accidentally infinite-loop (if we introduce a bug). + maxVisitCount := len(issuers)*len(issuers)*len(issuers) + 100 + for len(toVisit) > 0 && maxVisitCount >= 0 { + var issuer issuerId + issuer, toVisit = toVisit[0], toVisit[1:] + + // If (and only if) we're presently starved for next nodes to visit, + // attempt to resolve cliques and cycles again to fix that. This is + // because all-cycles cycle detection is at least as costly as + // traversing the entire graph a couple of times. + // + // Additionally, we do this immediately after popping a node from the + // queue as we wish to ensure we never become starved for nodes. + if len(toVisit) == 0 { + toVisit, err = processAnyCliqueOrCycle(issuers, processedIssuers, toVisit, issuerIdEntryMap, issuerIdCertMap, issuerIdParentsMap, issuerIdChildrenMap, subjectIssuerIdsMap) + if err != nil { + return err + } + } + + // Self-loops and cross-signing might lead to this node already being + // processed; skip it on the second pass. + if processed, ok := processedIssuers[issuer]; ok && processed { + continue + } + + // Check our parent certs now; if they are all processed, we can + // process this node. Otherwise, we'll re-add this to the queue + // when the last parent is processed (and we re-add its children). + parentCerts, ok := issuerIdParentsMap[issuer] + if ok && len(parentCerts) > 0 { + // For each parent, validate that we've processed it. + mustSkip := false + for _, parentCert := range parentCerts { + if processed, ok := processedIssuers[parentCert]; !ok || !processed { + mustSkip = true + break + } + } + + if mustSkip { + // Skip this node for now, we'll come back to it later. + continue + } + } + + // Now we can build the chain. Start with the current cert... + entry := issuerIdEntryMap[issuer] + entry.CAChain = []string{entry.Certificate} + + // ...and add all parents into it. Note that we have to tell if + // that parent was already visited or not. + if ok && len(parentCerts) > 0 { + includedParentCerts := make(map[string]bool, len(parentCerts)+1) + includedParentCerts[entry.Certificate] = true + for _, parentCert := range parentCerts { + // See discussion of the algorithm above as to why this is + // in the correct order. However, note that we do need to + // exclude duplicate certs, hence the map above. + // + // Assumption: issuerIdEntryMap and issuerIdParentsMap is well + // constructed. + parent := issuerIdEntryMap[parentCert] + for _, parentChainCert := range parent.CAChain { + addToChainIfNotExisting(includedParentCerts, entry, parentChainCert) + } + } + } + + // Now, mark this node as processed and go and visit all of its + // children. + processedIssuers[issuer] = true + + childrenCerts, ok := issuerIdChildrenMap[issuer] + if ok && len(childrenCerts) > 0 { + toVisit = append(toVisit, childrenCerts...) + } + } + + // Assumption: no nodes left unprocessed. They should've either been + // reached through the parent->child addition or they should've been + // self-loops. + var msg string + for _, issuer := range issuers { + if visited, ok := processedIssuers[issuer]; !ok || !visited { + pretty := prettyIssuer(issuerIdEntryMap, issuer) + msg += fmt.Sprintf("[failed to build chain correctly: unprocessed issuer %v: ok: %v; visited: %v]\n", pretty, ok, visited) + } + } + if len(msg) > 0 { + return fmt.Errorf(msg) + } + + // Finally, write all issuers to disk. + for _, issuer := range issuers { + entry := issuerIdEntryMap[issuer] + + err := writeIssuer(ctx, s, entry) + if err != nil { + pretty := prettyIssuer(issuerIdEntryMap, issuer) + return fmt.Errorf("failed to persist issuer (%v) chain to disk: %v", pretty, err) + } + } + + // Everything worked \o/ + return nil +} + +func addToChainIfNotExisting(includedParentCerts map[string]bool, entry *issuer, certToAdd string) { + included, ok := includedParentCerts[certToAdd] + if ok && included { + return + } + + entry.CAChain = append(entry.CAChain, certToAdd) + includedParentCerts[certToAdd] = true +} + +func processAnyCliqueOrCycle( + issuers []issuerId, + processedIssuers map[issuerId]bool, + toVisit []issuerId, + issuerIdEntryMap map[issuerId]*issuer, + issuerIdCertMap map[issuerId]*x509.Certificate, + issuerIdParentsMap map[issuerId][]issuerId, + issuerIdChildrenMap map[issuerId][]issuerId, + subjectIssuerIdsMap map[string][]issuerId, +) ([]issuerId /* toVisit */, error) { + // Topological sort really only works on directed acyclic graphs (DAGs). + // But a pool of arbitrary (issuer) certificates are actually neither! + // This pool could contain both cliques and cycles. Because this could + // block chain construction, we need to handle these cases. + // + // Within the helper for rebuildIssuersChains, we realize that we might + // have certain pathological cases where cliques and cycles might _mix_. + // This warrants handling them outside of the topo-sort code, effectively + // acting as a node-collapsing technique (turning many nodes into one). + // In reality, we just special-case this and handle the processing of + // these nodes manually, fixing their CAChain value and then skipping + // them. + // + // Since clique detection is (in this case) cheap (at worst O(n) on the + // size of the graph), we favor it over the cycle detection logic. The + // order (in the case of mixed cliques+cycles) doesn't matter, as the + // discovery of the clique will lead to the cycle. We additionally find + // all (unprocessed) cliques first, so our cycle detection code can avoid + // falling into cliques. + // + // We need to be able to handle cliques adjacent to cycles. This is + // necessary because a cross-signed cert (with same subject and key as + // the clique, but different issuer) could be part of a cycle; this cycle + // loop forms a parent chain (that topo-sort can't resolve) -- AND the + // clique itself mixes with this, so resolving one or the other isn't + // sufficient (as the reissued clique plus the cross-signed cert + // effectively acts as a single node in the cycle). Oh, and there might + // be multiple cycles. :-) + // + // We also might just have cycles, separately from reissued cliques. + // + // The nice thing about both cliques and cycles is that, as long as you + // deduplicate your certs, all issuers in the collection (including the + // mixed collection) have the same chain entries, just in different + // orders (preferring the cycle and appending the remaining clique + // entries afterwards). + + // To begin, cache all cliques that we know about. + allCliques, issuerIdCliqueMap, allCliqueNodes, err := findAllCliques(issuerIdCertMap, subjectIssuerIdsMap, issuers) + if err != nil { + // Found a clique that is too large; exit with an error. + return nil, err + } + + for _, issuer := range issuers { + // This first branch is finding cliques. However, finding a clique is + // not sufficient as discussed above -- we also need to find any + // incident cycle as this cycle is a parent and child to the clique, + // which means the cycle nodes _must_ include the clique _and_ the + // clique must include the cycle (in the CA Chain computation). + // However, its not sufficient to just do one and then the other: + // we need the closure of all cliques (and their incident cycles). + // Finally -- it isn't enough to consider this chain in isolation + // either. We need to consider _all_ parents and ensure they've been + // processed before processing this closure. + + var cliques [][]issuerId + var cycles [][]issuerId + closure := make(map[issuerId]bool) + + var cliquesToProcess []issuerId + cliquesToProcess = append(cliquesToProcess, issuer) + + for len(cliquesToProcess) > 0 { + var node issuerId + node, cliquesToProcess = cliquesToProcess[0], cliquesToProcess[1:] + + // Skip potential clique nodes which have already been processed + // (either by the topo-sort or by this clique-finding code). + if processed, ok := processedIssuers[node]; ok && processed { + continue + } + if nodeInClosure, ok := closure[node]; ok && nodeInClosure { + continue + } + + // Check if we have a clique for this node from our computed + // collection of cliques. + cliqueId, ok := issuerIdCliqueMap[node] + if !ok { + continue + } + cliqueNodes := allCliques[cliqueId] + + // Add our discovered clique. Note that we avoid duplicate cliques by + // the skip logic above. Additionally, we know that cliqueNodes must + // be unique and not duplicated with any existing nodes so we can add + // all nodes to closure. + cliques = append(cliques, cliqueNodes) + for _, node := range cliqueNodes { + closure[node] = true + } + + // Try and expand the clique to see if there's common cycles around + // it. We exclude _all_ clique nodes from the expansion path, because + // it will unnecessarily bloat the detected cycles AND we know that + // we'll find them again from the neighborhood search. + // + // Additionally, note that, detection of cycles should be independent + // of cliques: cliques form under reissuance, and cycles form via + // cross-signing chains; the latter ensures that any cliques can be + // strictly bypassed from cycles (but the chain construction later + // ensures we pull in the cliques into the cycles). + foundCycles, err := findCyclesNearClique(processedIssuers, issuerIdChildrenMap, allCliqueNodes) + if err != nil { + // Cycle is too large. + return toVisit, err + } + + // Assumption: each cycle in foundCycles is in canonical order (see note + // below about canonical ordering). Deduplicate these against already + // existing cycles and add them to the closure nodes. + for _, cycle := range foundCycles { + cycles = appendCycleIfNotExisting(cycles, cycle) + + // Now, for each cycle node, we need to find all adjacent cliques. + // We do this by finding each child of the cycle and adding it to + // the queue. If these nodes aren't on cliques, we'll skip them + // fairly quickly since the cliques were pre-computed. + for _, cycleNode := range cycle { + children, ok := issuerIdChildrenMap[cycleNode] + if !ok { + continue + } + + for _, child := range children { + cliquesToProcess = append(cliquesToProcess, child) + } + + // While we're here, add this cycle node to the closure. + closure[cycleNode] = true + } + } + } + + // Before we begin, we need to compute the _parents_ of the nodes in + // these cliques and cycles and ensure they've all been processed (if + // they're not already part of the closure). + parents, ok := computeParentsFromClosure(processedIssuers, issuerIdParentsMap, closure) + if !ok { + // At least one parent wasn't processed; skip this cliques and + // cycles group for now until they have all been processed. + continue + } + + // Ok, we've computed the closure. Now we can build CA nodes and mark + // everything as processed, growing the toVisit queue in the process. + // For every node we've found... + for node := range closure { + // Before we begin, mark this node as processed (so we can continue + // later) and add children to toVisit. + processedIssuers[node] = true + childrenCerts, ok := issuerIdChildrenMap[node] + if ok && len(childrenCerts) > 0 { + toVisit = append(toVisit, childrenCerts...) + } + + // It can either be part of a clique or a cycle. We wish to add + // the nodes of whatever grouping + foundNode := false + for _, clique := range cliques { + inClique := false + for _, cliqueNode := range clique { + if cliqueNode == node { + inClique = true + break + } + } + + if inClique { + foundNode = true + + // Compute this node's CAChain. Note order doesn't matter + // (within the clique), but we'll preserve the relative + // order of associated cycles. + entry := issuerIdEntryMap[node] + entry.CAChain = []string{entry.Certificate} + + includedParentCerts := make(map[string]bool, len(closure)+1) + includedParentCerts[entry.Certificate] = true + + // First add nodes from this clique, then all cycles, and then + // all other cliques. + addNodeCertsToEntry(issuerIdEntryMap, issuerIdChildrenMap, includedParentCerts, entry, clique) + addNodeCertsToEntry(issuerIdEntryMap, issuerIdChildrenMap, includedParentCerts, entry, cycles...) + addNodeCertsToEntry(issuerIdEntryMap, issuerIdChildrenMap, includedParentCerts, entry, cliques...) + addParentChainsToEntry(issuerIdEntryMap, includedParentCerts, entry, parents) + + break + } + } + + // Otherwise, it must be part of a cycle. + for _, cycle := range cycles { + inCycle := false + offsetInCycle := 0 + for index, cycleNode := range cycle { + if cycleNode == node { + inCycle = true + offsetInCycle = index + break + } + } + + if inCycle { + foundNode = true + + // Compute this node's CAChain. Note that order within cycles + // matters, but we'll preserve the relative order. + entry := issuerIdEntryMap[node] + entry.CAChain = []string{entry.Certificate} + + includedParentCerts := make(map[string]bool, len(closure)+1) + includedParentCerts[entry.Certificate] = true + + // First add nodes from this cycle, then all cliques, then all + // other cycles, and finally from parents. + orderedCycle := append(cycle[offsetInCycle:], cycle[0:offsetInCycle]...) + addNodeCertsToEntry(issuerIdEntryMap, issuerIdChildrenMap, includedParentCerts, entry, orderedCycle) + addNodeCertsToEntry(issuerIdEntryMap, issuerIdChildrenMap, includedParentCerts, entry, cliques...) + addNodeCertsToEntry(issuerIdEntryMap, issuerIdChildrenMap, includedParentCerts, entry, cycles...) + addParentChainsToEntry(issuerIdEntryMap, includedParentCerts, entry, parents) + + break + } + } + + if !foundNode { + // Unable to find node; return an error. This shouldn't happen + // generally. + pretty := prettyIssuer(issuerIdEntryMap, issuer) + return nil, fmt.Errorf("Unable to find node (%v) in closure (%v) but not in cycles (%v) or cliques (%v)", pretty, closure, cycles, cliques) + } + } + } + + // We might also have cycles without having associated cliques. We assume + // that any cliques (if they existed and were relevant for the remaining + // cycles) were processed at this point. However, we might still have + // unprocessed cliques (and related cycles) at this point _if_ an + // unrelated cycle is the parent to that clique+cycle group. + for _, issuer := range issuers { + // Skip this node if it is already processed. + if processed, ok := processedIssuers[issuer]; ok && processed { + continue + } + + // Cliques should've been processed by now, if they were necessary + // for processable cycles, so ignore them from here to avoid + // bloating our search paths. + cycles, err := findAllCyclesWithNode(processedIssuers, issuerIdChildrenMap, issuer, allCliqueNodes) + if err != nil { + // To large of cycle. + return nil, err + } + + closure := make(map[issuerId]bool) + for _, cycle := range cycles { + for _, node := range cycle { + closure[node] = true + } + } + + // Before we begin, we need to compute the _parents_ of the nodes in + // these cycles and ensure they've all been processed (if they're not + // part of the closure). + parents, ok := computeParentsFromClosure(processedIssuers, issuerIdParentsMap, closure) + if !ok { + // At least one parent wasn't processed; skip this cycle + // group for now until they have all been processed. + continue + } + + // Finally, for all detected cycles, build the CAChain for nodes in + // cycles. Since they all share a common parent, they must all contain + // each other. + for _, cycle := range cycles { + // For each node in each cycle + for nodeIndex, node := range cycle { + // If the node is processed already, skip it. + if processed, ok := processedIssuers[node]; ok && processed { + continue + } + + // Otherwise, build its CAChain. + entry := issuerIdEntryMap[node] + entry.CAChain = []string{entry.Certificate} + + // No indication as to size of chain here + includedParentCerts := make(map[string]bool) + includedParentCerts[entry.Certificate] = true + + // First add nodes from this cycle, then all other cycles, and + // finally from parents. + orderedCycle := append(cycle[nodeIndex:], cycle[0:nodeIndex]...) + addNodeCertsToEntry(issuerIdEntryMap, issuerIdChildrenMap, includedParentCerts, entry, orderedCycle) + addNodeCertsToEntry(issuerIdEntryMap, issuerIdChildrenMap, includedParentCerts, entry, cycles...) + addParentChainsToEntry(issuerIdEntryMap, includedParentCerts, entry, parents) + + // Finally, mark the node as processed and add the remaining + // children to toVisit. + processedIssuers[node] = true + childrenCerts, ok := issuerIdChildrenMap[node] + if ok && len(childrenCerts) > 0 { + toVisit = append(toVisit, childrenCerts...) + } + } + } + } + + return toVisit, nil +} + +func findAllCliques( + issuerIdCertMap map[issuerId]*x509.Certificate, + subjectIssuerIdsMap map[string][]issuerId, + issuers []issuerId, +) ([][]issuerId, map[issuerId]int, []issuerId, error) { + var allCliques [][]issuerId + issuerIdCliqueMap := make(map[issuerId]int) + var allCliqueNodes []issuerId + + for _, node := range issuers { + // Check if the node has already been visited... + if _, ok := issuerIdCliqueMap[node]; ok { + // ...if so it must be on another clique; skip the clique finding + // so we don't get duplicated cliques. + continue + } + + // See if this is a node on a clique and find that clique. + cliqueNodes, err := isOnReissuedClique(issuerIdCertMap, subjectIssuerIdsMap, node) + if err != nil { + // Clique is too large. + return nil, nil, nil, err + } + + // Skip nodes which really aren't a clique. + if len(cliqueNodes) <= 1 { + continue + } + + // Add this clique and update the mapping. A given node can only be in one + // clique. + cliqueId := len(allCliques) + allCliques = append(allCliques, cliqueNodes) + allCliqueNodes = append(allCliqueNodes, cliqueNodes...) + for _, cliqueNode := range cliqueNodes { + issuerIdCliqueMap[cliqueNode] = cliqueId + } + } + + return allCliques, issuerIdCliqueMap, allCliqueNodes, nil +} + +func isOnReissuedClique( + issuerIdCertMap map[issuerId]*x509.Certificate, + subjectIssuerIdsMap map[string][]issuerId, + node issuerId, +) ([]issuerId, error) { + // Finding max cliques in arbitrary graphs is a nearly pathological + // problem, usually left to the realm of SAT solvers and NP-Complete + // theoretical. + // + // We're not dealing with arbitrary graphs though. We're dealing with + // a highly regular, highly structured constructed graph. + // + // Reissued cliques form in certificate chains when two conditions hold: + // + // 1. The Subject of the certificate matches the Issuer. + // 2. The underlying public key is the same, resulting in the signature + // validating for any pair of certs. + // + // This follows from the definition of a reissued certificate (same key + // material, subject, and issuer but with a different serial number and + // a different validity period). The structure means that the graph is + // highly regular: given a partial or self-clique, if any candidate node + // can satisfy this relation with any node of the existing clique, it must + // mean it must form a larger clique and satisfy this relationship with + // all other nodes in the existing clique. + // + // (Aside: this is not the only type of clique, but it is the only type + // of 3+ node clique. A 2-star is emitted from certain graphs, but we + // chose to handle that case in the cycle detection code rather than + // under this reissued clique detection code). + // + // What does this mean for our algorithm? A simple greedy search is + // sufficient. If we index our certificates by subject -> issuerId + // (and cache its value across calls, which we've already done for + // building the parent/child relationship), we can find all other issuers + // with the same public key and subject as the existing node fairly + // easily. + // + // However, we should also set some reasonable bounds on clique size. + // Let's limit it to 6 nodes. + maxCliqueSize := 6 + + // Per assumptions of how we've built the graph, these map lookups should + // both exist. + cert := issuerIdCertMap[node] + subject := string(cert.RawSubject) + issuer := string(cert.RawIssuer) + candidates := subjectIssuerIdsMap[subject] + + // If the given node doesn't have the same subject and issuer, it isn't + // a valid clique node. + if subject != issuer { + return nil, nil + } + + // We have two choices here for validating that the two keys are the same: + // perform a cheap ASN.1 encoding comparison of the public keys, which + // _should_ be the same but may not be, or perform a more costly (but + // which should definitely be correct) signature verification. We prefer + // cheap and call it good enough. + spki := cert.RawSubjectPublicKeyInfo + + // We know candidates has everything satisfying _half_ of the first + // condition (the subject half), so validate they match the other half + // (the issuer half) and the second condition. For node (which is + // included in candidates), the condition should vacuously hold. + var clique []issuerId + for _, candidate := range candidates { + candidateCert := issuerIdCertMap[candidate] + hasRightKey := bytes.Equal(candidateCert.RawSubjectPublicKeyInfo, spki) + hasMatchingIssuer := string(candidateCert.RawIssuer) == issuer + + if hasRightKey && hasMatchingIssuer { + clique = append(clique, candidate) + } + } + + // Clique is invalid if it contains zero or one nodes. + if len(clique) <= 1 { + return nil, nil + } + + // Validate it is within the acceptable clique size. + if len(clique) > maxCliqueSize { + return clique, fmt.Errorf("error building issuer chains: excessively reissued certificate: %v entries", len(clique)) + } + + // Must be a valid clique. + return clique, nil +} + +func containsIssuer(collection []issuerId, target issuerId) bool { + if len(collection) == 0 { + return false + } + + for _, needle := range collection { + if needle == target { + return true + } + } + + return false +} + +func appendCycleIfNotExisting(knownCycles [][]issuerId, candidate []issuerId) [][]issuerId { + // There's two ways to do cycle detection: canonicalize the cycles, + // rewriting them to have the least (or max) element first or just + // brute force the detection. + // + // Canonicalizing them is faster and easier to write (just compare + // canonical forms) so do that instead. + canonicalized := canonicalizeCycle(candidate) + + found := false + for _, existing := range knownCycles { + if len(existing) != len(canonicalized) { + continue + } + + equivalent := true + for index, node := range canonicalized { + if node != existing[index] { + equivalent = false + break + } + } + + if equivalent { + found = true + break + } + } + + if !found { + return append(knownCycles, canonicalized) + } + + return knownCycles +} + +func canonicalizeCycle(cycle []issuerId) []issuerId { + // Find the minimum value and put it at the head, keeping the relative + // ordering the same. + minIndex := 0 + for index, entry := range cycle { + if entry < cycle[minIndex] { + minIndex = index + } + } + + ret := append(cycle[minIndex:], cycle[0:minIndex]...) + if len(ret) != len(cycle) { + panic("ABORT") + } + + return ret +} + +func findCyclesNearClique( + processedIssuers map[issuerId]bool, + issuerIdChildrenMap map[issuerId][]issuerId, + cliqueNodes []issuerId, +) ([][]issuerId, error) { + // When we have a reissued clique, we need to find all cycles next to it. + // Presumably, because they all have non-empty parents, they should not + // have been visited yet. We further know that (because we're exploring + // the children path), any processed check would be unnecessary as all + // children shouldn't have been processed yet (since their parents aren't + // either). + // + // So, we can explore each of the children of any one clique node and + // find all cycles using that node, until we come back to the starting + // node, excluding the clique and other cycles. + cliqueNode := cliqueNodes[0] + + // Copy the clique nodes as excluded nodes; we'll avoid exploring cycles + // which have parents that have been already explored. + excludeNodes := cliqueNodes[:] + var knownCycles [][]issuerId + + // We know the node has at least one child, since the clique is non-empty. + for _, child := range issuerIdChildrenMap[cliqueNode] { + // Skip children that are part of the clique. + if containsIssuer(excludeNodes, child) { + continue + } + + // Find cycles containing this node. + newCycles, err := findAllCyclesWithNode(processedIssuers, issuerIdChildrenMap, child, excludeNodes) + if err != nil { + // Found too large of a cycle + return nil, err + } + + // Add all cycles into the known cycles list. + for _, cycle := range newCycles { + knownCycles = appendCycleIfNotExisting(knownCycles, cycle) + } + + // Exclude only the current child. Adding everything in the cycles + // results might prevent discovery of other valid cycles. + excludeNodes = append(excludeNodes, child) + } + + return knownCycles, nil +} + +func findAllCyclesWithNode( + processedIssuers map[issuerId]bool, + issuerIdChildrenMap map[issuerId][]issuerId, + source issuerId, + exclude []issuerId, +) ([][]issuerId, error) { + // We wish to find all cycles involving this particular node and report + // the corresponding paths. This is a full-graph traversal (excluding + // certain paths) as we're not just checking if a cycle occurred, but + // instead returning all of cycles with that node. + // + // Set some limit on max cycle size. + maxCycleSize := 8 + + // Whether we've visited any given node. + cycleVisited := make(map[issuerId]bool) + visitCounts := make(map[issuerId]int) + parentCounts := make(map[issuerId]map[issuerId]bool) + + // Paths to the specified node. Some of these might be cycles. + pathsTo := make(map[issuerId][][]issuerId) + + // Nodes to visit. + var visitQueue []issuerId + + // Add the source node to start. In order to set up the paths to a + // given node, we seed pathsTo with the single path involving just + // this node + visitQueue = append(visitQueue, source) + pathsTo[source] = [][]issuerId{{source}} + + // Begin building paths. + // + // Loop invariant: + // pathTo[x] contains valid paths to reach this node, from source. + for len(visitQueue) > 0 { + var current issuerId + current, visitQueue = visitQueue[0], visitQueue[1:] + + // If we've already processed this node, we have a cycle. Skip this + // node for now; we'll build cycles later. + if processed, ok := cycleVisited[current]; ok && processed { + continue + } + + // Mark this node as visited for next time. + cycleVisited[current] = true + if _, ok := visitCounts[current]; !ok { + visitCounts[current] = 0 + } + visitCounts[current] += 1 + + // For every child of this node... + children, ok := issuerIdChildrenMap[current] + if !ok { + // Node has no children, nothing else we can do. + continue + } + + for _, child := range children { + // Ensure we can visit this child; exclude processedIssuers and + // exclude lists. + if childProcessed, ok := processedIssuers[child]; ok && childProcessed { + continue + } + + skipNode := false + for _, excluded := range exclude { + if excluded == child { + skipNode = true + break + } + } + + if skipNode { + continue + } + + // Track this parent->child relationship to know when to exit. + setOfParents, ok := parentCounts[child] + if !ok { + setOfParents = make(map[issuerId]bool) + parentCounts[child] = setOfParents + } + _, existingParent := setOfParents[current] + setOfParents[current] = true + + // Since we know that we can visit this node, we should now build + // all destination paths using this node, from our current node. + // + // Since these are all starting at a single path from source, + // if we have any cycles back to source, we'll find them here. + // + // Only add this if it is a net-new path that doesn't repeat + // (either internally -- indicating an internal cycle -- or + // externally with an existing path). + addedPath := false + if _, ok := pathsTo[child]; !ok { + pathsTo[child] = make([][]issuerId, 0) + } + for _, path := range pathsTo[current] { + if child != source { + // We only care about source->source cycles. If this + // cycles, but isn't a source->source cycle, don't add + // this path. + foundSelf := false + for _, node := range path { + if child == node { + foundSelf = true + break + } + } + if foundSelf { + // Skip this path. + continue + } + } + + // Make sure to deep copy the path. + newPath := make([]issuerId, 0, len(path)+1) + newPath = append(newPath, path...) + newPath = append(newPath, child) + + isSamePath := false + for _, childPath := range pathsTo[child] { + if len(childPath) != len(newPath) { + continue + } + + isSamePath = true + for index, node := range childPath { + if newPath[index] != node { + isSamePath = false + break + } + } + + if isSamePath { + break + } + } + + if !isSamePath { + pathsTo[child] = append(pathsTo[child], newPath) + addedPath = true + } + } + + // Visit this child next. + visitQueue = append(visitQueue, child) + + // If there's a new parent or we found a new path, then we should + // revisit this child, to update _its_ children and see if there's + // another new path. Eventually the paths will stabilize and we'll + // end up with no new parents or paths. + if !existingParent || addedPath { + cycleVisited[child] = false + } + } + } + + // Ok, we've now exited from our loop. Any cycles would've been detected + // and their paths recorded in pathsTo. Now we can iterate over these + // (starting a source), clean them up and validate them. + var cycles [][]issuerId + for _, cycle := range pathsTo[source] { + // Skip the trivial cycle. + if len(cycle) == 1 && cycle[0] == source { + continue + } + + // Validate cycle starts and ends with source. + if cycle[0] != source { + return nil, fmt.Errorf("cycle (%v) unexpectedly starts with node %v; expected to start with %v", cycle, cycle[0], source) + } + + // If the cycle doesn't start/end with the source, + // skip it. + if cycle[len(cycle)-1] != source { + continue + } + + truncatedCycle := cycle[0 : len(cycle)-1] + if len(truncatedCycle) >= maxCycleSize { + return nil, fmt.Errorf("cycle (%v) exceeds max size: %v > %v", cycle, len(cycle), maxCycleSize) + } + + // Now one last thing: our cycle was built via parent->child + // traversal, but we want child->parent ordered cycles. So, + // just reverse it. + reversed := reversedCycle(truncatedCycle) + cycles = appendCycleIfNotExisting(cycles, reversed) + } + + return cycles, nil +} + +func reversedCycle(cycle []issuerId) []issuerId { + var result []issuerId + for index := len(cycle) - 1; index >= 0; index-- { + result = append(result, cycle[index]) + } + + return result +} + +func computeParentsFromClosure( + processedIssuers map[issuerId]bool, + issuerIdParentsMap map[issuerId][]issuerId, + closure map[issuerId]bool, +) (map[issuerId]bool, bool) { + parents := make(map[issuerId]bool) + for node := range closure { + nodeParents, ok := issuerIdParentsMap[node] + if !ok { + continue + } + + for _, parent := range nodeParents { + if nodeInClosure, ok := closure[parent]; ok && nodeInClosure { + continue + } + + parents[parent] = true + if processed, ok := processedIssuers[parent]; ok && processed { + continue + } + + return nil, false + } + } + + return parents, true +} + +func addNodeCertsToEntry( + issuerIdEntryMap map[issuerId]*issuer, + issuerIdChildrenMap map[issuerId][]issuerId, + includedParentCerts map[string]bool, + entry *issuer, + issuersCollection ...[]issuerId, +) { + for _, collection := range issuersCollection { + // Find a starting point into this collection such that it verifies + // something in the existing collection. + offset := 0 + for index, issuer := range collection { + children, ok := issuerIdChildrenMap[issuer] + if !ok { + continue + } + + foundChild := false + for _, child := range children { + childEntry := issuerIdEntryMap[child] + if inChain, ok := includedParentCerts[childEntry.Certificate]; ok && inChain { + foundChild = true + break + } + } + + if foundChild { + offset = index + break + } + } + + // Assumption: collection is in child -> parent order. For cliques, + // this is trivially true because everyone can validate each other, + // but for cycles we have to ensure that in findAllCyclesWithNode. + // This allows us to build the chain in the correct order. + for _, issuer := range append(collection[offset:], collection[0:offset]...) { + nodeEntry := issuerIdEntryMap[issuer] + addToChainIfNotExisting(includedParentCerts, entry, nodeEntry.Certificate) + } + } +} + +func addParentChainsToEntry( + issuerIdEntryMap map[issuerId]*issuer, + includedParentCerts map[string]bool, + entry *issuer, + parents map[issuerId]bool, +) { + for parent := range parents { + nodeEntry := issuerIdEntryMap[parent] + for _, cert := range nodeEntry.CAChain { + addToChainIfNotExisting(includedParentCerts, entry, cert) + } + } +} diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 0d5d1dbef0a98..ea2b0a96cff84 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -410,7 +410,6 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu result.ID = genIssuerId() result.Name = issuerName result.Certificate = certValue - result.CAChain = []string{certValue} // Extracting the certificate is necessary for two reasons: first, it lets // us fetch the serial number; second, for the public key comparison with @@ -420,6 +419,11 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu return nil, false, err } + // Ensure this certificate is a usable as a CA certificate. + if !issuerCert.BasicConstraintsValid || !issuerCert.IsCA { + return nil, false, errutil.UserError{Err: "Refusing to import non-CA certificate"} + } + result.SerialNumber = strings.TrimSpace(certutil.GetHexFormatted(issuerCert.SerialNumber.Bytes(), ":")) // Now, for each key, try and compute the issuer<->key link. We delay @@ -452,8 +456,9 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu } } - // We can write the issuer to storage. - if err := writeIssuer(ctx, s, &result); err != nil { + // Finally, rebuild the chains. In this process, because the provided + // reference issuer is non-nil, we'll save this issuer to storage. + if err := rebuildIssuersChains(ctx, s, &result); err != nil { return nil, false, err } From e7d5258443019284c2a3fc53d610bc4943b941a9 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Wed, 13 Apr 2022 11:22:30 -0400 Subject: [PATCH 36/76] Return CA Chain when fetching issuers This returns the CA Chain attribute of an issuer, showing its computed chain based on other issuers in the database, when fetching a specific issuer. Signed-off-by: Alexander Scheel --- builtin/logical/pki/path_fetch_issuers.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 86d5aa49c0300..57b9aab7d6852 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -112,6 +112,7 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data "issuer_name": issuer.Name, "key_id": issuer.KeyID, "certificate": issuer.Certificate, + "ca_chain": issuer.CAChain, }, }, nil } @@ -155,6 +156,7 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da "issuer_name": issuer.Name, "key_id": issuer.KeyID, "certificate": issuer.Certificate, + "ca_chain": issuer.CAChain, }, }, nil } From ade2b496d59d45aaadf0aaf3c8f2c715be81ba78 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Wed, 13 Apr 2022 15:09:46 -0400 Subject: [PATCH 37/76] Add testing for chain building Using the issuance infrastructure, we generate new certificates (either roots or intermediates), positing that this is roughly equivalent to importing an external bundle (minus error handling during partial imports). This allows us to incrementally construct complex chains, creating reissuance cliques and cross-signing cycles. By using ECDSA certificates, we avoid high signature verification and key generation times. Signed-off-by: Alexander Scheel --- builtin/logical/pki/chain_test.go | 897 ++++++++++++++++++++++++++++++ 1 file changed, 897 insertions(+) create mode 100644 builtin/logical/pki/chain_test.go diff --git a/builtin/logical/pki/chain_test.go b/builtin/logical/pki/chain_test.go new file mode 100644 index 0000000000000..f44b0116f9adf --- /dev/null +++ b/builtin/logical/pki/chain_test.go @@ -0,0 +1,897 @@ +package pki + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "strings" + "testing" + + "github.com/hashicorp/vault/api" + vaulthttp "github.com/hashicorp/vault/http" + "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/vault" +) + +// For speed, all keys are ECDSA. +type CBGenerateKey struct { + Name string +} + +func (c CBGenerateKey) Run(t *testing.T, client *api.Client, mount string, knownKeys map[string]string, knownCerts map[string]string) { + resp, err := client.Logical().Write(mount+"/keys/generate/exported", map[string]interface{}{ + "name": c.Name, + "algo": "ec", + "bits": 256, + }) + if err != nil { + t.Fatalf("failed to provision key (%v): %v", c.Name, err) + } + knownKeys[c.Name] = resp.Data["private"].(string) +} + +// Generate a root. +type CBGenerateRoot struct { + Key string + Existing bool + Name string + CommonName string + ErrorMessage string +} + +func (c CBGenerateRoot) Run(t *testing.T, client *api.Client, mount string, knownKeys map[string]string, knownCerts map[string]string) { + url := mount + "/issuers/generate/root/" + data := make(map[string]interface{}) + + if c.Existing { + url += "existing" + data["key_ref"] = c.Key + } else { + url += "exported" + data["key_type"] = "ec" + data["key_bits"] = 256 + data["key_name"] = c.Key + } + + data["issuer_name"] = c.Name + data["common_name"] = c.Name + if len(c.CommonName) > 0 { + data["common_name"] = c.CommonName + } + + resp, err := client.Logical().Write(url, data) + if err != nil { + if len(c.ErrorMessage) > 0 { + if !strings.Contains(err.Error(), c.ErrorMessage) { + t.Fatalf("failed to generate root cert for issuer (%v): expected (%v) in error message but got %v", c.Name, c.ErrorMessage, err) + } + return + } + t.Fatalf("failed to provision issuer (%v): %v / body: %v", c.Name, err, data) + } else if len(c.ErrorMessage) > 0 { + t.Fatalf("expected to fail generation of issuer (%v) with error message containing (%v)", c.Name, c.ErrorMessage) + } + + if !c.Existing { + knownKeys[c.Key] = resp.Data["private_key"].(string) + } + + knownCerts[c.Name] = resp.Data["certificate"].(string) +} + +// Generate an intermediate. Might not really be an intermediate; might be +// a cross-signed cert. +type CBGenerateIntermediate struct { + Key string + Existing bool + Name string + CommonName string + Parent string + ImportErrorMessage string +} + +func (c CBGenerateIntermediate) Run(t *testing.T, client *api.Client, mount string, knownKeys map[string]string, knownCerts map[string]string) { + // Build CSR + url := mount + "/issuers/generate/intermediate/" + data := make(map[string]interface{}) + + if c.Existing { + url += "existing" + data["key_ref"] = c.Key + } else { + url += "exported" + data["key_type"] = "ec" + data["key_bits"] = 256 + data["key_name"] = c.Key + } + + resp, err := client.Logical().Write(url, data) + if err != nil { + t.Fatalf("failed to generate CSR for issuer (%v): %v / body: %v", c.Name, err, data) + } + + if !c.Existing { + knownKeys[c.Key] = resp.Data["private_key"].(string) + } + + csr := resp.Data["csr"].(string) + + // Sign CSR + url = fmt.Sprintf(mount+"/issuers/%s/sign-intermediate", c.Parent) + data = make(map[string]interface{}) + data["csr"] = csr + data["common_name"] = c.Name + if len(c.CommonName) > 0 { + data["common_name"] = c.CommonName + } + resp, err = client.Logical().Write(url, data) + if err != nil { + t.Fatalf("failed to sign CSR for issuer (%v): %v / body: %v", c.Name, err, data) + } + + knownCerts[c.Name] = strings.TrimSpace(resp.Data["certificate"].(string)) + + // Set the signed intermediate + url = mount + "/intermediate/set-signed" + data = make(map[string]interface{}) + data["certificate"] = knownCerts[c.Name] + data["issuer_name"] = c.Name + + resp, err = client.Logical().Write(url, data) + if err != nil { + if len(c.ImportErrorMessage) > 0 { + if !strings.Contains(err.Error(), c.ImportErrorMessage) { + t.Fatalf("failed to import signed cert for issuer (%v): expected (%v) in error message but got %v", c.Name, c.ImportErrorMessage, err) + } + return + } + + t.Fatalf("failed to import signed cert for issuer (%v): %v / body: %v", c.Name, err, data) + } else if len(c.ImportErrorMessage) > 0 { + t.Fatalf("expected to fail import (with error %v) of cert for issuer (%v) but was success: response: %v", c.ImportErrorMessage, c.Name, resp) + } + + // Update the name since set-signed doesn't actually take an issuer name + // parameter. + rawNewCerts := resp.Data["imported_issuers"].([]interface{}) + if len(rawNewCerts) != 1 { + t.Fatalf("Expected a single new certificate during import of signed cert for %v: got %v\nresp: %v", c.Name, len(rawNewCerts), resp) + } + + newCertId := rawNewCerts[0].(string) + _, err = client.Logical().Write(mount+"/issuer/"+newCertId, map[string]interface{}{ + "issuer_name": c.Name, + }) + if err != nil { + t.Fatalf("failed to update name for issuer (%v/%v): %v", c.Name, newCertId, err) + } +} + +// Delete an issuer; breaks chains. +type CBDeleteIssuer struct { + Issuer string +} + +func (c CBDeleteIssuer) Run(t *testing.T, client *api.Client, mount string, knownKeys map[string]string, knownCerts map[string]string) { + url := fmt.Sprintf(mount+"/issuer/%v", c.Issuer) + _, err := client.Logical().Delete(url) + if err != nil { + t.Fatalf("failed to delete issuer (%v): %v", c.Issuer, err) + } + + delete(knownCerts, c.Issuer) +} + +// Validate the specified chain exists, by name. +type CBValidateChain struct { + Chains map[string][]string + Aliases map[string]string +} + +func (c CBValidateChain) ChainToPEMs(t *testing.T, parent string, chain []string, knownCerts map[string]string) []string { + var result []string + for entryIndex, entry := range chain { + var chainEntry string + modifiedEntry := entry + if entryIndex == 0 && entry == "self" { + modifiedEntry = parent + } + for pattern, replacement := range c.Aliases { + modifiedEntry = strings.ReplaceAll(modifiedEntry, pattern, replacement) + } + for _, issuer := range strings.Split(modifiedEntry, ",") { + cert, ok := knownCerts[issuer] + if !ok { + t.Fatalf("Unknown issuer %v in chain for %v: %v", issuer, parent, chain) + } + + chainEntry += cert + } + result = append(result, chainEntry) + } + + return result +} + +func (c CBValidateChain) FindNameForCert(t *testing.T, cert string, knownCerts map[string]string) string { + for issuer, known := range knownCerts { + if strings.TrimSpace(known) == strings.TrimSpace(cert) { + return issuer + } + } + + t.Fatalf("Unable to find cert:\n[%v]\nin known map:\n%v\n", cert, knownCerts) + return "" +} + +func (c CBValidateChain) PrettyChain(t *testing.T, chain []string, knownCerts map[string]string) []string { + var prettyChain []string + for _, cert := range chain { + prettyChain = append(prettyChain, c.FindNameForCert(t, cert, knownCerts)) + } + + return prettyChain +} + +func (c CBValidateChain) ToCertificate(t *testing.T, cert string) *x509.Certificate { + block, _ := pem.Decode([]byte(cert)) + if block == nil { + t.Fatalf("Unable to parse certificate: nil PEM block\n[%v]\n", cert) + } + + ret, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("Unable to parse certificate: %v\n[%v]\n", err, cert) + } + + return ret +} + +func (c CBValidateChain) Run(t *testing.T, client *api.Client, mount string, knownKeys map[string]string, knownCerts map[string]string) { + for issuer, chain := range c.Chains { + resp, err := client.Logical().Read(mount + "/issuer/" + issuer) + if err != nil { + t.Fatalf("failed to get chain for issuer (%v): %v", issuer, err) + } + + rawCurrentChain := resp.Data["ca_chain"].([]interface{}) + var currentChain []string + for _, entry := range rawCurrentChain { + currentChain = append(currentChain, strings.TrimSpace(entry.(string))) + } + + // Ensure the issuer cert is always first. + if currentChain[0] != knownCerts[issuer] { + pretty := c.FindNameForCert(t, currentChain[0], knownCerts) + t.Fatalf("expected certificate at index 0 to be self:\n[%v]\n[pretty: %v]\nis not the issuer's cert:\n[%v]\n[pretty: %v]", currentChain[0], pretty, knownCerts[issuer], issuer) + } + + // Validate it against the expected chain. + expectedChain := c.ChainToPEMs(t, issuer, chain, knownCerts) + if len(currentChain) != len(expectedChain) { + prettyCurrentChain := c.PrettyChain(t, currentChain, knownCerts) + t.Fatalf("Lengths of chains for issuer %v mismatched: got %v vs expected %v:\n[%v]\n[pretty: %v]\n[%v]\n[pretty: %v]", issuer, len(currentChain), len(expectedChain), currentChain, prettyCurrentChain, expectedChain, chain) + } + + for currentIndex, currentCert := range currentChain { + // Chains might be forked so we may not be able to strictly validate + // the chain against a single value. Instead, use strings.Contains + // to validate the current cert is in the list of allowed + // possibilities. + if !strings.Contains(expectedChain[currentIndex], currentCert) { + pretty := c.FindNameForCert(t, currentCert, knownCerts) + t.Fatalf("chain mismatch at index %v for issuer %v: got cert:\n[%v]\n[pretty: %v]\nbut expected one of\n[%v]\n[pretty: %v]\n", currentIndex, issuer, currentCert, pretty, expectedChain[currentIndex], chain[currentIndex]) + } + } + + // Due to alternate paths, the above doesn't ensure ensure each cert + // in the chain is only used once. Validate that now. + for thisIndex, thisCert := range currentChain { + for otherIndex, otherCert := range currentChain[thisIndex+1:] { + if thisCert == otherCert { + thisPretty := c.FindNameForCert(t, thisCert, knownCerts) + otherPretty := c.FindNameForCert(t, otherCert, knownCerts) + otherIndex += thisIndex + 1 + t.Fatalf("cert reused in chain for %v:\n[%v]\n[pretty: %v / index: %v]\n[%v]\n[pretty: %v / index: %v]\n", issuer, thisCert, thisPretty, thisIndex, otherCert, otherPretty, otherIndex) + } + } + } + + // Finally, validate that all certs verify something that came before + // it. In the linear chain sense, this should strictly mean that the + // parent comes before the child. + for thisIndex, thisCertPem := range currentChain[1:] { + thisIndex += 1 // Absolute index. + parentCert := c.ToCertificate(t, thisCertPem) + + // Iterate backwards; prefer the most recent cert to the older + // certs. + foundCert := false + for otherIndex := thisIndex - 1; otherIndex >= 0; otherIndex-- { + otherCertPem := currentChain[otherIndex] + childCert := c.ToCertificate(t, otherCertPem) + + if err := childCert.CheckSignatureFrom(parentCert); err == nil { + foundCert = true + } + } + + if !foundCert { + pretty := c.FindNameForCert(t, thisCertPem, knownCerts) + t.Fatalf("malformed test scenario: certificate at chain index %v when validating %v does not validate any previous certificates:\n[%v]\n[pretty: %v]\n", thisIndex, issuer, thisCertPem, pretty) + } + } + } +} + +type CBTestStep interface { + Run(t *testing.T, client *api.Client, mount string, knownKeys map[string]string, knownCerts map[string]string) +} + +type CBTestScenario struct { + Steps []CBTestStep +} + +func Test_CAChainBuilding(t *testing.T) { + coreConfig := &vault.CoreConfig{ + LogicalBackends: map[string]logical.Factory{ + "pki": Factory, + }, + } + cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + defer cluster.Cleanup() + + client := cluster.Cores[0].Client + + testCases := []CBTestScenario{ + { + // This test builds up two cliques lined by a cycle, dropping into + // a single intermediate. + Steps: []CBTestStep{ + // Create a reissued certificate using the same key. These + // should validate themselves. + CBGenerateRoot{ + Key: "key-root-old", + Name: "root-old-a", + CommonName: "root-old", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self"}, + }, + }, + // After adding the second root using the same key and common + // name, there should now be two certs in each chain. + CBGenerateRoot{ + Key: "key-root-old", + Existing: true, + Name: "root-old-b", + CommonName: "root-old", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self", "root-old-b"}, + "root-old-b": {"self", "root-old-a"}, + }, + }, + // After adding a third root, there are now two possibilities for + // each later chain entry. + CBGenerateRoot{ + Key: "key-root-old", + Existing: true, + Name: "root-old-c", + CommonName: "root-old", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self", "root-old-bc", "root-old-bc"}, + "root-old-b": {"self", "root-old-ac", "root-old-ac"}, + "root-old-c": {"self", "root-old-ab", "root-old-ab"}, + }, + Aliases: map[string]string{ + "root-old-ac": "root-old-a,root-old-c", + "root-old-ab": "root-old-a,root-old-b", + "root-old-bc": "root-old-b,root-old-c", + }, + }, + // If we generate an unrelated issuer, it shouldn't affect either + // chain. + CBGenerateRoot{ + Key: "key-root-new", + Name: "root-new-a", + CommonName: "root-new", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self", "root-old-bc", "root-old-bc"}, + "root-old-b": {"self", "root-old-ac", "root-old-ac"}, + "root-old-c": {"self", "root-old-ab", "root-old-ab"}, + "root-new-a": {"self"}, + }, + Aliases: map[string]string{ + "root-old-ac": "root-old-a,root-old-c", + "root-old-ab": "root-old-a,root-old-b", + "root-old-bc": "root-old-b,root-old-c", + }, + }, + // Reissuing this new root should form another clique. + CBGenerateRoot{ + Key: "key-root-new", + Existing: true, + Name: "root-new-b", + CommonName: "root-new", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self", "root-old-bc", "root-old-bc"}, + "root-old-b": {"self", "root-old-ac", "root-old-ac"}, + "root-old-c": {"self", "root-old-ab", "root-old-ab"}, + "root-new-a": {"self", "root-new-b"}, + "root-new-b": {"self", "root-new-a"}, + }, + Aliases: map[string]string{ + "root-old-ac": "root-old-a,root-old-c", + "root-old-ab": "root-old-a,root-old-b", + "root-old-bc": "root-old-b,root-old-c", + }, + }, + // Generating a cross-signed cert from old->new should result + // in all old clique certs showing up in the new root's paths. + // This does not form a cycle. + CBGenerateIntermediate{ + // In order to validate the existing root-new clique, we + // have to reuse the key and common name here for + // cross-signing. + Key: "key-root-new", + Existing: true, + Name: "cross-old-new", + CommonName: "root-new", + // Which old issuer is used here doesn't matter as they have + // the same CN and key. + Parent: "root-old-a", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self", "root-old-bc", "root-old-bc"}, + "root-old-b": {"self", "root-old-ac", "root-old-ac"}, + "root-old-c": {"self", "root-old-ab", "root-old-ab"}, + "cross-old-new": {"self", "root-old-abc", "root-old-abc", "root-old-abc"}, + "root-new-a": {"self", "root-new-b", "cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + "root-new-b": {"self", "root-new-a", "cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + }, + Aliases: map[string]string{ + "root-old-ac": "root-old-a,root-old-c", + "root-old-ab": "root-old-a,root-old-b", + "root-old-bc": "root-old-b,root-old-c", + "root-old-abc": "root-old-a,root-old-b,root-old-c", + }, + }, + // If we create a new intermediate off of the root-new, we should + // simply add to the existing chain. + CBGenerateIntermediate{ + Key: "key-inter-a-root-new", + Name: "inter-a-root-new", + Parent: "root-new-a", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self", "root-old-bc", "root-old-bc"}, + "root-old-b": {"self", "root-old-ac", "root-old-ac"}, + "root-old-c": {"self", "root-old-ab", "root-old-ab"}, + "cross-old-new": {"self", "root-old-abc", "root-old-abc", "root-old-abc"}, + "root-new-a": {"self", "root-new-b", "cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + "root-new-b": {"self", "root-new-a", "cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + // If we find cross-old-new first, the old clique will be ahead + // of the new clique; otherwise the new clique will appear first. + "inter-a-root-new": {"self", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle"}, + }, + Aliases: map[string]string{ + "root-old-ac": "root-old-a,root-old-c", + "root-old-ab": "root-old-a,root-old-b", + "root-old-bc": "root-old-b,root-old-c", + "root-old-abc": "root-old-a,root-old-b,root-old-c", + "full-cycle": "root-old-a,root-old-b,root-old-c,cross-old-new,root-new-a,root-new-b", + }, + }, + // Now, if we cross-sign back from new to old, we should + // form cycle with multiple reissued cliques. This means + // all nodes will have the same chain. + CBGenerateIntermediate{ + // In order to validate the existing root-old clique, we + // have to reuse the key and common name here for + // cross-signing. + Key: "key-root-old", + Existing: true, + Name: "cross-new-old", + CommonName: "root-old", + // Which new issuer is used here doesn't matter as they have + // the same CN and key. + Parent: "root-new-a", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self", "root-old-bc", "root-old-bc", "both-cross-old-new", "both-cross-old-new", "root-new-ab", "root-new-ab"}, + "root-old-b": {"self", "root-old-ac", "root-old-ac", "both-cross-old-new", "both-cross-old-new", "root-new-ab", "root-new-ab"}, + "root-old-c": {"self", "root-old-ab", "root-old-ab", "both-cross-old-new", "both-cross-old-new", "root-new-ab", "root-new-ab"}, + "cross-old-new": {"self", "cross-new-old", "both-cliques", "both-cliques", "both-cliques", "both-cliques", "both-cliques"}, + "cross-new-old": {"self", "cross-old-new", "both-cliques", "both-cliques", "both-cliques", "both-cliques", "both-cliques"}, + "root-new-a": {"self", "root-new-b", "both-cross-old-new", "both-cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + "root-new-b": {"self", "root-new-a", "both-cross-old-new", "both-cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + "inter-a-root-new": {"self", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle"}, + }, + Aliases: map[string]string{ + "root-old-ac": "root-old-a,root-old-c", + "root-old-ab": "root-old-a,root-old-b", + "root-old-bc": "root-old-b,root-old-c", + "root-old-abc": "root-old-a,root-old-b,root-old-c", + "root-new-ab": "root-new-a,root-new-b", + "both-cross-old-new": "cross-old-new,cross-new-old", + "both-cliques": "root-old-a,root-old-b,root-old-c,root-new-a,root-new-b", + "full-cycle": "root-old-a,root-old-b,root-old-c,cross-old-new,cross-new-old,root-new-a,root-new-b", + }, + }, + }, + }, + { + // Here we're testing our chain capacity. First we'll create a + // bunch of unique roots to form a cycle of length 10. + Steps: []CBTestStep{ + CBGenerateRoot{ + Key: "key-root-a", + Name: "root-a", + CommonName: "root-a", + }, + CBGenerateRoot{ + Key: "key-root-b", + Name: "root-b", + CommonName: "root-b", + }, + CBGenerateRoot{ + Key: "key-root-c", + Name: "root-c", + CommonName: "root-c", + }, + CBGenerateRoot{ + Key: "key-root-d", + Name: "root-d", + CommonName: "root-d", + }, + CBGenerateRoot{ + Key: "key-root-e", + Name: "root-e", + CommonName: "root-e", + }, + // They should all be disjoint to start. + CBValidateChain{ + Chains: map[string][]string{ + "root-a": {"self"}, + "root-b": {"self"}, + "root-c": {"self"}, + "root-d": {"self"}, + "root-e": {"self"}, + }, + }, + // Start the cross-signing chains. These are all linear, so there's + // no error expected; they're just long. + CBGenerateIntermediate{ + Key: "key-root-b", + Existing: true, + Name: "cross-a-b", + CommonName: "root-b", + Parent: "root-a", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-a": {"self"}, + "cross-a-b": {"self", "root-a"}, + "root-b": {"self", "cross-a-b", "root-a"}, + "root-c": {"self"}, + "root-d": {"self"}, + "root-e": {"self"}, + }, + }, + CBGenerateIntermediate{ + Key: "key-root-c", + Existing: true, + Name: "cross-b-c", + CommonName: "root-c", + Parent: "root-b", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-a": {"self"}, + "cross-a-b": {"self", "root-a"}, + "root-b": {"self", "cross-a-b", "root-a"}, + "cross-b-c": {"self", "b-or-cross", "b-chained-cross", "b-chained-cross"}, + "root-c": {"self", "cross-b-c", "b-or-cross", "b-chained-cross", "b-chained-cross"}, + "root-d": {"self"}, + "root-e": {"self"}, + }, + Aliases: map[string]string{ + "b-or-cross": "root-b,cross-a-b", + "b-chained-cross": "root-b,cross-a-b,root-a", + }, + }, + CBGenerateIntermediate{ + Key: "key-root-d", + Existing: true, + Name: "cross-c-d", + CommonName: "root-d", + Parent: "root-c", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-a": {"self"}, + "cross-a-b": {"self", "root-a"}, + "root-b": {"self", "cross-a-b", "root-a"}, + "cross-b-c": {"self", "b-or-cross", "b-chained-cross", "b-chained-cross"}, + "root-c": {"self", "cross-b-c", "b-or-cross", "b-chained-cross", "b-chained-cross"}, + "cross-c-d": {"self", "c-or-cross", "c-chained-cross", "c-chained-cross", "c-chained-cross", "c-chained-cross"}, + "root-d": {"self", "cross-c-d", "c-or-cross", "c-chained-cross", "c-chained-cross", "c-chained-cross", "c-chained-cross"}, + "root-e": {"self"}, + }, + Aliases: map[string]string{ + "b-or-cross": "root-b,cross-a-b", + "b-chained-cross": "root-b,cross-a-b,root-a", + "c-or-cross": "root-c,cross-b-c", + "c-chained-cross": "root-c,cross-b-c,root-b,cross-a-b,root-a", + }, + }, + CBGenerateIntermediate{ + Key: "key-root-e", + Existing: true, + Name: "cross-d-e", + CommonName: "root-e", + Parent: "root-d", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-a": {"self"}, + "cross-a-b": {"self", "root-a"}, + "root-b": {"self", "cross-a-b", "root-a"}, + "cross-b-c": {"self", "b-or-cross", "b-chained-cross", "b-chained-cross"}, + "root-c": {"self", "cross-b-c", "b-or-cross", "b-chained-cross", "b-chained-cross"}, + "cross-c-d": {"self", "c-or-cross", "c-chained-cross", "c-chained-cross", "c-chained-cross", "c-chained-cross"}, + "root-d": {"self", "cross-c-d", "c-or-cross", "c-chained-cross", "c-chained-cross", "c-chained-cross", "c-chained-cross"}, + "cross-d-e": {"self", "d-or-cross", "d-chained-cross", "d-chained-cross", "d-chained-cross", "d-chained-cross", "d-chained-cross", "d-chained-cross"}, + "root-e": {"self", "cross-d-e", "d-or-cross", "d-chained-cross", "d-chained-cross", "d-chained-cross", "d-chained-cross", "d-chained-cross", "d-chained-cross"}, + }, + Aliases: map[string]string{ + "b-or-cross": "root-b,cross-a-b", + "b-chained-cross": "root-b,cross-a-b,root-a", + "c-or-cross": "root-c,cross-b-c", + "c-chained-cross": "root-c,cross-b-c,root-b,cross-a-b,root-a", + "d-or-cross": "root-d,cross-c-d", + "d-chained-cross": "root-d,cross-c-d,root-c,cross-b-c,root-b,cross-a-b,root-a", + }, + }, + // Importing the new e->a cross fails because the cycle + // it builds is too long. + CBGenerateIntermediate{ + Key: "key-root-a", + Existing: true, + Name: "cross-e-a", + CommonName: "root-a", + Parent: "root-e", + ImportErrorMessage: "exceeds max size", + }, + // Deleting any root and one of its crosses (either a->b or b->c) + // should fix this. + CBDeleteIssuer{"root-b"}, + CBDeleteIssuer{"cross-b-c"}, + // Importing the new e->a cross fails because the cycle + // it builds is too long. + CBGenerateIntermediate{ + Key: "key-root-a", + Existing: true, + Name: "cross-e-a", + CommonName: "root-a", + Parent: "root-e", + }, + }, + }, + { + // Here we're testing our clique capacity. First we'll create a + // bunch of unique roots to form a cycle of length 10. + Steps: []CBTestStep{ + CBGenerateRoot{ + Key: "key-root", + Name: "root-a", + CommonName: "root", + }, + CBGenerateRoot{ + Key: "key-root", + Existing: true, + Name: "root-b", + CommonName: "root", + }, + CBGenerateRoot{ + Key: "key-root", + Existing: true, + Name: "root-c", + CommonName: "root", + }, + CBGenerateRoot{ + Key: "key-root", + Existing: true, + Name: "root-d", + CommonName: "root", + }, + CBGenerateRoot{ + Key: "key-root", + Existing: true, + Name: "root-e", + CommonName: "root", + }, + CBGenerateRoot{ + Key: "key-root", + Existing: true, + Name: "root-f", + CommonName: "root", + }, + // Seventh reissuance fails. + CBGenerateRoot{ + Key: "key-root", + Existing: true, + Name: "root-g", + CommonName: "root", + ErrorMessage: "excessively reissued certificate", + }, + // Deleting one and trying again should succeed. + CBDeleteIssuer{"root-a"}, + CBGenerateRoot{ + Key: "key-root", + Existing: true, + Name: "root-g", + CommonName: "root", + }, + }, + }, + { + // There's one more pathological case here: we have a cycle + // which validates a clique/cycle via cross-signing. We call + // the parent cycle new roots and the child cycle/clique the + // old roots. + Steps: []CBTestStep{ + // New Cycle + CBGenerateRoot{ + Key: "key-root-new-a", + Name: "root-new-a", + }, + CBGenerateRoot{ + Key: "key-root-new-b", + Name: "root-new-b", + }, + CBGenerateIntermediate{ + Key: "key-root-new-b", + Existing: true, + Name: "cross-root-new-b-sig-a", + CommonName: "root-new-b", + Parent: "root-new-a", + }, + CBGenerateIntermediate{ + Key: "key-root-new-a", + Existing: true, + Name: "cross-root-new-a-sig-b", + CommonName: "root-new-a", + Parent: "root-new-b", + }, + // Old Cycle + Clique + CBGenerateRoot{ + Key: "key-root-old-a", + Name: "root-old-a", + }, + CBGenerateRoot{ + Key: "key-root-old-a", + Existing: true, + Name: "root-old-a-reissued", + CommonName: "root-old-a", + }, + CBGenerateRoot{ + Key: "key-root-old-b", + Name: "root-old-b", + }, + CBGenerateRoot{ + Key: "key-root-old-b", + Existing: true, + Name: "root-old-b-reissued", + CommonName: "root-old-b", + }, + CBGenerateIntermediate{ + Key: "key-root-old-b", + Existing: true, + Name: "cross-root-old-b-sig-a", + CommonName: "root-old-b", + Parent: "root-old-a", + }, + CBGenerateIntermediate{ + Key: "key-root-old-a", + Existing: true, + Name: "cross-root-old-a-sig-b", + CommonName: "root-old-a", + Parent: "root-old-b", + }, + // Validate the chains are separate before linking them. + CBValidateChain{ + Chains: map[string][]string{ + // New stuff + "root-new-a": {"self", "cross-root-new-a-sig-b", "root-new-b-or-cross", "root-new-b-or-cross"}, + "root-new-b": {"self", "cross-root-new-b-sig-a", "root-new-a-or-cross", "root-new-a-or-cross"}, + "cross-root-new-b-sig-a": {"self", "any-root-new", "any-root-new", "any-root-new"}, + "cross-root-new-a-sig-b": {"self", "any-root-new", "any-root-new", "any-root-new"}, + + // Old stuff + "root-old-a": {"self", "root-old-a-reissued", "cross-root-old-a-sig-b", "cross-root-old-b-sig-a", "both-root-old-b", "both-root-old-b"}, + "root-old-a-reissued": {"self", "root-old-a", "cross-root-old-a-sig-b", "cross-root-old-b-sig-a", "both-root-old-b", "both-root-old-b"}, + "root-old-b": {"self", "root-old-b-reissued", "cross-root-old-b-sig-a", "cross-root-old-a-sig-b", "both-root-old-a", "both-root-old-a"}, + "root-old-b-reissued": {"self", "root-old-b", "cross-root-old-b-sig-a", "cross-root-old-a-sig-b", "both-root-old-a", "both-root-old-a"}, + "cross-root-old-b-sig-a": {"self", "all-root-old", "all-root-old", "all-root-old", "all-root-old", "all-root-old"}, + "cross-root-old-a-sig-b": {"self", "all-root-old", "all-root-old", "all-root-old", "all-root-old", "all-root-old"}, + }, + Aliases: map[string]string{ + "root-new-a-or-cross": "root-new-a,cross-root-new-a-sig-b", + "root-new-b-or-cross": "root-new-b,cross-root-new-b-sig-a", + "both-root-new": "root-new-a,root-new-b", + "any-root-new": "root-new-a,cross-root-new-a-sig-b,root-new-b,cross-root-new-b-sig-a", + "both-root-old-a": "root-old-a,root-old-a-reissued", + "both-root-old-b": "root-old-b,root-old-b-reissued", + "all-root-old": "root-old-a,root-old-a-reissued,root-old-b,root-old-b-reissued,cross-root-old-b-sig-a,cross-root-old-a-sig-b", + }, + }, + // Finally, generate an intermediate to link new->old. We + // link root-new-a into root-old-a. + CBGenerateIntermediate{ + Key: "key-root-old-a", + Existing: true, + Name: "cross-root-old-a-sig-root-new-a", + CommonName: "root-old-a", + Parent: "root-new-a", + }, + CBValidateChain{ + Chains: map[string][]string{ + // New stuff should be unchanged. + "root-new-a": {"self", "cross-root-new-a-sig-b", "root-new-b-or-cross", "root-new-b-or-cross"}, + "root-new-b": {"self", "cross-root-new-b-sig-a", "root-new-a-or-cross", "root-new-a-or-cross"}, + "cross-root-new-b-sig-a": {"self", "any-root-new", "any-root-new", "any-root-new"}, + "cross-root-new-a-sig-b": {"self", "any-root-new", "any-root-new", "any-root-new"}, + + // Old stuff + "root-old-a": {"self", "root-old-a-reissued", "cross-root-old-a-sig-b", "cross-root-old-b-sig-a", "both-root-old-b", "both-root-old-b", "cross-root-old-a-sig-root-new-a", "any-root-new", "any-root-new", "any-root-new", "any-root-new"}, + "root-old-a-reissued": {"self", "root-old-a", "cross-root-old-a-sig-b", "cross-root-old-b-sig-a", "both-root-old-b", "both-root-old-b", "cross-root-old-a-sig-root-new-a", "any-root-new", "any-root-new", "any-root-new", "any-root-new"}, + "root-old-b": {"self", "root-old-b-reissued", "cross-root-old-b-sig-a", "cross-root-old-a-sig-b", "both-root-old-a", "both-root-old-a", "cross-root-old-a-sig-root-new-a", "any-root-new", "any-root-new", "any-root-new", "any-root-new"}, + "root-old-b-reissued": {"self", "root-old-b", "cross-root-old-b-sig-a", "cross-root-old-a-sig-b", "both-root-old-a", "both-root-old-a", "cross-root-old-a-sig-root-new-a", "any-root-new", "any-root-new", "any-root-new", "any-root-new"}, + "cross-root-old-b-sig-a": {"self", "all-root-old", "all-root-old", "all-root-old", "all-root-old", "all-root-old", "cross-root-old-a-sig-root-new-a", "any-root-new", "any-root-new", "any-root-new", "any-root-new"}, + "cross-root-old-a-sig-b": {"self", "all-root-old", "all-root-old", "all-root-old", "all-root-old", "all-root-old", "cross-root-old-a-sig-root-new-a", "any-root-new", "any-root-new", "any-root-new", "any-root-new"}, + + // Link + "cross-root-old-a-sig-root-new-a": {"self", "root-new-a-or-cross", "any-root-new", "any-root-new", "any-root-new"}, + }, + Aliases: map[string]string{ + "root-new-a-or-cross": "root-new-a,cross-root-new-a-sig-b", + "root-new-b-or-cross": "root-new-b,cross-root-new-b-sig-a", + "both-root-new": "root-new-a,root-new-b", + "any-root-new": "root-new-a,cross-root-new-a-sig-b,root-new-b,cross-root-new-b-sig-a", + "both-root-old-a": "root-old-a,root-old-a-reissued", + "both-root-old-b": "root-old-b,root-old-b-reissued", + "all-root-old": "root-old-a,root-old-a-reissued,root-old-b,root-old-b-reissued,cross-root-old-b-sig-a,cross-root-old-a-sig-b", + }, + }, + }, + }, + } + + for testIndex, testCase := range testCases { + mount := fmt.Sprintf("pki-test-%v", testIndex) + mountPKIEndpoint(t, client, mount) + knownKeys := make(map[string]string) + knownCerts := make(map[string]string) + for stepIndex, testStep := range testCase.Steps { + t.Logf("Running %v / %v", testIndex, stepIndex) + testStep.Run(t, client, mount, knownKeys, knownCerts) + } + + } +} From 4caf0a2efc543fdcf61f53ee77604706af832a51 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Fri, 15 Apr 2022 13:24:22 -0400 Subject: [PATCH 38/76] Allow manual construction of issuer chain Signed-off-by: Alexander Scheel --- builtin/logical/pki/chain_util.go | 62 ++++++++++++++++- builtin/logical/pki/path_fetch_issuers.go | 84 ++++++++++++++++++++--- builtin/logical/pki/storage.go | 13 ++-- 3 files changed, 140 insertions(+), 19 deletions(-) diff --git a/builtin/logical/pki/chain_util.go b/builtin/logical/pki/chain_util.go index c870339c09c55..953a4e1376522 100644 --- a/builtin/logical/pki/chain_util.go +++ b/builtin/logical/pki/chain_util.go @@ -240,6 +240,39 @@ func rebuildIssuersChains(ctx context.Context, s logical.Storage, referenceCert processedIssuers := make(map[issuerId]bool, len(issuers)) toVisit := make([]issuerId, 0, len(issuers)) + // Handle any explicitly constructed certificate chains. Here, we don't + // validate much what the user provides; if they provide since-deleted + // refs, skip them; if they duplicate entries, add them multiple times. + // The other chain building logic will be able to deduplicate them when + // used as parents to other certificates. + for _, candidate := range issuers { + entry := issuerIdEntryMap[candidate] + if len(entry.ManualChain) == 0 { + continue + } + + entry.CAChain = nil + for _, parentId := range entry.ManualChain { + parentEntry := issuerIdEntryMap[parentId] + if parentEntry == nil { + continue + } + + entry.CAChain = append(entry.CAChain, parentEntry.Certificate) + } + + // Mark this node as processed and add its children. + processedIssuers[candidate] = true + children, ok := issuerIdChildrenMap[candidate] + if !ok { + continue + } + + for _, child := range children { + toVisit = append(toVisit, child) + } + } + // Setup the toVisit queue. for _, candidate := range issuers { parentCerts, ok := issuerIdParentsMap[candidate] @@ -431,13 +464,18 @@ func processAnyCliqueOrCycle( // entries afterwards). // To begin, cache all cliques that we know about. - allCliques, issuerIdCliqueMap, allCliqueNodes, err := findAllCliques(issuerIdCertMap, subjectIssuerIdsMap, issuers) + allCliques, issuerIdCliqueMap, allCliqueNodes, err := findAllCliques(processedIssuers, issuerIdCertMap, subjectIssuerIdsMap, issuers) if err != nil { // Found a clique that is too large; exit with an error. return nil, err } for _, issuer := range issuers { + // Skip anything that's already been processed. + if processed, ok := processedIssuers[issuer]; ok && processed { + continue + } + // This first branch is finding cliques. However, finding a clique is // not sufficient as discussed above -- we also need to find any // incident cycle as this cycle is a parent and child to the clique, @@ -448,7 +486,6 @@ func processAnyCliqueOrCycle( // Finally -- it isn't enough to consider this chain in isolation // either. We need to consider _all_ parents and ensure they've been // processed before processing this closure. - var cliques [][]issuerId var cycles [][]issuerId closure := make(map[issuerId]bool) @@ -542,6 +579,11 @@ func processAnyCliqueOrCycle( // everything as processed, growing the toVisit queue in the process. // For every node we've found... for node := range closure { + // Skip anything that's already been processed. + if processed, ok := processedIssuers[node]; ok && processed { + continue + } + // Before we begin, mark this node as processed (so we can continue // later) and add children to toVisit. processedIssuers[node] = true @@ -707,6 +749,7 @@ func processAnyCliqueOrCycle( } func findAllCliques( + processedIssuers map[issuerId]bool, issuerIdCertMap map[issuerId]*x509.Certificate, subjectIssuerIdsMap map[string][]issuerId, issuers []issuerId, @@ -717,6 +760,11 @@ func findAllCliques( for _, node := range issuers { // Check if the node has already been visited... + if processed, ok := processedIssuers[node]; ok && processed { + // ...if so it might have had a manually constructed chain; skip + // it for clique detection. + continue + } if _, ok := issuerIdCliqueMap[node]; ok { // ...if so it must be on another clique; skip the clique finding // so we don't get duplicated cliques. @@ -724,7 +772,7 @@ func findAllCliques( } // See if this is a node on a clique and find that clique. - cliqueNodes, err := isOnReissuedClique(issuerIdCertMap, subjectIssuerIdsMap, node) + cliqueNodes, err := isOnReissuedClique(processedIssuers, issuerIdCertMap, subjectIssuerIdsMap, node) if err != nil { // Clique is too large. return nil, nil, nil, err @@ -749,6 +797,7 @@ func findAllCliques( } func isOnReissuedClique( + processedIssuers map[issuerId]bool, issuerIdCertMap map[issuerId]*x509.Certificate, subjectIssuerIdsMap map[string][]issuerId, node issuerId, @@ -816,6 +865,13 @@ func isOnReissuedClique( // included in candidates), the condition should vacuously hold. var clique []issuerId for _, candidate := range candidates { + // Skip already processed nodes, even if they could be clique + // candidates. We'll treat them as any other (already processed) + // external parent in that scenario. + if processed, ok := processedIssuers[candidate]; ok && processed { + continue + } + candidateCert := issuerIdCertMap[candidate] hasRightKey := bytes.Equal(candidateCert.RawSubjectPublicKeyInfo, spki) hasMatchingIssuer := string(candidateCert.RawIssuer) == issuer diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 57b9aab7d6852..300ce083c66a5 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -66,6 +66,12 @@ func pathGetIssuer(b *backend) *framework.Path { func buildPathGetIssuer(b *backend, pattern string) *framework.Path { fields := map[string]*framework.FieldSchema{} fields = addIssuerRefNameFields(fields) + fields["manual_chain"] = &framework.FieldSchema{ + Type: framework.TypeCommaStringSlice, + Description: `Chain of issuer references to use to build this +issuer's computed CAChain field, when non-empty.`, + } + return &framework.Path{ // Returns a JSON entry. Pattern: pattern, @@ -106,13 +112,19 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data return nil, err } + var respManualChain []string + for _, entity := range issuer.ManualChain { + respManualChain = append(respManualChain, string(entity)) + } + return &logical.Response{ Data: map[string]interface{}{ - "issuer_id": issuer.ID, - "issuer_name": issuer.Name, - "key_id": issuer.KeyID, - "certificate": issuer.Certificate, - "ca_chain": issuer.CAChain, + "issuer_id": issuer.ID, + "issuer_name": issuer.Name, + "key_id": issuer.KeyID, + "certificate": issuer.Certificate, + "manual_chain": respManualChain, + "ca_chain": issuer.CAChain, }, }, nil } @@ -128,6 +140,8 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da return logical.ErrorResponse(err.Error()), nil } + newPath := data.Get("manual_chain").([]string) + ref, err := resolveIssuerReference(ctx, req.Storage, issuerName) if err != nil { return nil, err @@ -141,22 +155,72 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da return nil, err } + modified := false + if newName != issuer.Name { issuer.Name = newName + modified = true + } + + var updateChain bool + var constructedChain []issuerId + for index, newPathRef := range newPath { + // Allow self for the first entry. + if index == 0 && newPathRef == "self" { + newPathRef = string(ref) + } + + resolvedId, err := resolveIssuerReference(ctx, req.Storage, newPathRef) + if err != nil { + return nil, err + } + + if index == 0 && resolvedId != ref { + return logical.ErrorResponse(fmt.Sprintf("expected first cert in chain to be a self-reference, but was: %v/%v", newPathRef, resolvedId)), nil + } + + constructedChain = append(constructedChain, resolvedId) + if len(issuer.ManualChain) < len(constructedChain) || constructedChain[index] != issuer.ManualChain[index] { + updateChain = true + } + } + if len(issuer.ManualChain) != len(constructedChain) { + updateChain = true + } + + if updateChain { + issuer.ManualChain = constructedChain + + // Building the chain will write the issuer to disk; no need to do it + // twice. + modified = false + err := rebuildIssuersChains(ctx, req.Storage, issuer) + if err != nil { + return nil, err + } + } + + if modified { err := writeIssuer(ctx, req.Storage, issuer) if err != nil { return nil, err } } + var respManualChain []string + for _, entity := range issuer.ManualChain { + respManualChain = append(respManualChain, string(entity)) + } + return &logical.Response{ Data: map[string]interface{}{ - "issuer_id": issuer.ID, - "issuer_name": issuer.Name, - "key_id": issuer.KeyID, - "certificate": issuer.Certificate, - "ca_chain": issuer.CAChain, + "issuer_id": issuer.ID, + "issuer_name": issuer.Name, + "key_id": issuer.KeyID, + "certificate": issuer.Certificate, + "manual_chain": respManualChain, + "ca_chain": issuer.CAChain, }, }, nil } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index ea2b0a96cff84..fbcd95d0fa635 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -49,12 +49,13 @@ type key struct { } type issuer struct { - ID issuerId `json:"id" structs:"id" mapstructure:"id"` - Name string `json:"name" structs:"name" mapstructure:"name"` - KeyID keyId `json:"key_id" structs:"key_id" mapstructure:"key_id"` - Certificate string `json:"certificate" structs:"certificate" mapstructure:"certificate"` - CAChain []string `json:"ca_chain" structs:"ca_chain" mapstructure:"ca_chain"` - SerialNumber string `json:"serial_number" structs:"serial_number" mapstructure:"serial_number"` + ID issuerId `json:"id" structs:"id" mapstructure:"id"` + Name string `json:"name" structs:"name" mapstructure:"name"` + KeyID keyId `json:"key_id" structs:"key_id" mapstructure:"key_id"` + Certificate string `json:"certificate" structs:"certificate" mapstructure:"certificate"` + CAChain []string `json:"ca_chain" structs:"ca_chain" mapstructure:"ca_chain"` + ManualChain []issuerId `json:"manual_chain" structs:"manual_chain" mapstructure:"manual_chain"` + SerialNumber string `json:"serial_number" structs:"serial_number" mapstructure:"serial_number"` } type keyConfig struct { From 5ffcf4bc5151935e30c07c3b01b1b51e39b87da2 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 18 Apr 2022 14:22:09 -0400 Subject: [PATCH 39/76] Fix handling of duplicate names With the new issuer field (manual_chain), we can no longer err when a name already exists: we might be updating the existing issuer (with the same name), but changing its manual_chain field. Detect this error and correctly handle it. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend_test.go | 4 ++-- builtin/logical/pki/path_fetch_issuers.go | 18 +++++++++++------- builtin/logical/pki/util.go | 12 +++++++----- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index 9dd2850676e39..da9f032fbac8f 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -4724,7 +4724,7 @@ func TestRootWithExistingKey(t *testing.T) { "issuer_name": "my-issuer1", }) require.Error(t, err) - require.Contains(t, err.Error(), "issuer name already used") + require.Contains(t, err.Error(), "issuer name already in use") // Create the second CA resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ @@ -4747,7 +4747,7 @@ func TestRootWithExistingKey(t *testing.T) { "key_name": "root-key2", }) require.Error(t, err) - require.Contains(t, err.Error(), "key name already used") + require.Contains(t, err.Error(), "key name already in use") // Create a third CA re-using key from CA 1 resp, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/existing", map[string]interface{}{ diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 300ce083c66a5..317eeddc68265 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -135,13 +135,6 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da return logical.ErrorResponse("missing issuer reference"), nil } - newName, err := getIssuerName(ctx, req.Storage, data) - if err != nil { - return logical.ErrorResponse(err.Error()), nil - } - - newPath := data.Get("manual_chain").([]string) - ref, err := resolveIssuerReference(ctx, req.Storage, issuerName) if err != nil { return nil, err @@ -155,6 +148,17 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da return nil, err } + newName, err := getIssuerName(ctx, req.Storage, data) + if err != nil && err != errIssuerNameInUse { + // If the error is name already in use, and the new name is the + // old name for this issuer, we're not actually updating the + // issuer name (or causing a conflict) -- so don't err out. Other + // errs should still be surfaced, however. + return logical.ErrorResponse(err.Error()), nil + } + + newPath := data.Get("manual_chain").([]string) + modified := false if newName != issuer.Name { diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 62fc1d0e3c641..2d0add8b0c5fc 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -20,6 +20,8 @@ const ( ) var nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex(issuerRefParam) + "$") +var errIssuerNameInUse = errutil.UserError{Err: "issuer name already in use"} +var errKeyNameInUse = errutil.UserError{Err: "key name already in use"} func normalizeSerial(serial string) string { return strings.Replace(strings.ToLower(serial), ":", "-", -1) @@ -134,19 +136,19 @@ func getIssuerName(ctx context.Context, s logical.Storage, data *framework.Field issuerName = strings.TrimSpace(issuerNameIface.(string)) if strings.ToLower(issuerName) == defaultRef { - return "", errutil.UserError{Err: "reserved keyword 'default' can not be used as issuer name"} + return issuerName, errutil.UserError{Err: "reserved keyword 'default' can not be used as issuer name"} } if !nameMatcher.MatchString(issuerName) { - return "", errutil.UserError{Err: "issuer name contained invalid characters"} + return issuerName, errutil.UserError{Err: "issuer name contained invalid characters"} } issuer_id, err := resolveIssuerReference(ctx, s, issuerName) if err == nil { - return "", errutil.UserError{Err: "issuer name already used."} + return issuerName, errIssuerNameInUse } if err != nil && issuer_id != IssuerRefNotFound { - return "", errutil.InternalError{Err: err.Error()} + return issuerName, errutil.InternalError{Err: err.Error()} } } return issuerName, nil @@ -167,7 +169,7 @@ func getKeyName(ctx context.Context, s logical.Storage, data *framework.FieldDat } key_id, err := resolveKeyReference(ctx, s, keyName) if err == nil { - return "", errutil.UserError{Err: "key name already used."} + return "", errKeyNameInUse } if err != nil && key_id != KeyRefNotFound { From 4a3de6a3e3320eb740da1007885f17e4bb81cdbc Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 18 Apr 2022 14:23:50 -0400 Subject: [PATCH 40/76] Add tests for manual chain building We break the clique, instead building these chains manually, ensuring that the remaining chains do not change and only the modified certs change. We then reset them (back to implicit chain building) and ensure we get the same results as earlier. Signed-off-by: Alexander Scheel --- builtin/logical/pki/chain_test.go | 86 +++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/builtin/logical/pki/chain_test.go b/builtin/logical/pki/chain_test.go index f44b0116f9adf..e0ff00d219021 100644 --- a/builtin/logical/pki/chain_test.go +++ b/builtin/logical/pki/chain_test.go @@ -324,6 +324,24 @@ func (c CBValidateChain) Run(t *testing.T, client *api.Client, mount string, kno } } +// Update an issuer +type CBUpdateIssuer struct { + Name string + CAChain []string +} + +func (c CBUpdateIssuer) Run(t *testing.T, client *api.Client, mount string, knownKeys map[string]string, knownCerts map[string]string) { + url := mount + "/issuer/" + c.Name + data := make(map[string]interface{}) + data["issuer_name"] = c.Name + data["manual_chain"] = c.CAChain + + _, err := client.Logical().Write(url, data) + if err != nil { + t.Fatalf("failed to update issuer (%v): %v / body: %v", c.Name, err, data) + } +} + type CBTestStep interface { Run(t *testing.T, client *api.Client, mount string, knownKeys map[string]string, knownCerts map[string]string) } @@ -533,6 +551,74 @@ func Test_CAChainBuilding(t *testing.T) { "full-cycle": "root-old-a,root-old-b,root-old-c,cross-old-new,cross-new-old,root-new-a,root-new-b", }, }, + // Update each old root to only include itself. + CBUpdateIssuer{ + Name: "root-old-a", + CAChain: []string{"root-old-a"}, + }, + CBUpdateIssuer{ + Name: "root-old-b", + CAChain: []string{"root-old-b"}, + }, + CBUpdateIssuer{ + Name: "root-old-c", + CAChain: []string{"root-old-c"}, + }, + // Step 19 + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self"}, + "root-old-b": {"self"}, + "root-old-c": {"self"}, + "cross-old-new": {"self", "cross-new-old", "both-cliques", "both-cliques", "both-cliques", "both-cliques", "both-cliques"}, + "cross-new-old": {"self", "cross-old-new", "both-cliques", "both-cliques", "both-cliques", "both-cliques", "both-cliques"}, + "root-new-a": {"self", "root-new-b", "both-cross-old-new", "both-cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + "root-new-b": {"self", "root-new-a", "both-cross-old-new", "both-cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + "inter-a-root-new": {"self", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle"}, + }, + Aliases: map[string]string{ + "root-old-ac": "root-old-a,root-old-c", + "root-old-ab": "root-old-a,root-old-b", + "root-old-bc": "root-old-b,root-old-c", + "root-old-abc": "root-old-a,root-old-b,root-old-c", + "root-new-ab": "root-new-a,root-new-b", + "both-cross-old-new": "cross-old-new,cross-new-old", + "both-cliques": "root-old-a,root-old-b,root-old-c,root-new-a,root-new-b", + "full-cycle": "root-old-a,root-old-b,root-old-c,cross-old-new,cross-new-old,root-new-a,root-new-b", + }, + }, + // Reset the old roots; should get the original chains back. + CBUpdateIssuer{ + Name: "root-old-a", + }, + CBUpdateIssuer{ + Name: "root-old-b", + }, + CBUpdateIssuer{ + Name: "root-old-c", + }, + CBValidateChain{ + Chains: map[string][]string{ + "root-old-a": {"self", "root-old-bc", "root-old-bc", "both-cross-old-new", "both-cross-old-new", "root-new-ab", "root-new-ab"}, + "root-old-b": {"self", "root-old-ac", "root-old-ac", "both-cross-old-new", "both-cross-old-new", "root-new-ab", "root-new-ab"}, + "root-old-c": {"self", "root-old-ab", "root-old-ab", "both-cross-old-new", "both-cross-old-new", "root-new-ab", "root-new-ab"}, + "cross-old-new": {"self", "cross-new-old", "both-cliques", "both-cliques", "both-cliques", "both-cliques", "both-cliques"}, + "cross-new-old": {"self", "cross-old-new", "both-cliques", "both-cliques", "both-cliques", "both-cliques", "both-cliques"}, + "root-new-a": {"self", "root-new-b", "both-cross-old-new", "both-cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + "root-new-b": {"self", "root-new-a", "both-cross-old-new", "both-cross-old-new", "root-old-abc", "root-old-abc", "root-old-abc"}, + "inter-a-root-new": {"self", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle", "full-cycle"}, + }, + Aliases: map[string]string{ + "root-old-ac": "root-old-a,root-old-c", + "root-old-ab": "root-old-a,root-old-b", + "root-old-bc": "root-old-b,root-old-c", + "root-old-abc": "root-old-a,root-old-b,root-old-c", + "root-new-ab": "root-new-a,root-new-b", + "both-cross-old-new": "cross-old-new,cross-new-old", + "both-cliques": "root-old-a,root-old-b,root-old-c,root-new-a,root-new-b", + "full-cycle": "root-old-a,root-old-b,root-old-c,cross-old-new,cross-new-old,root-new-a,root-new-b", + }, + }, }, }, { From 361e8b81295bac49326c0acb81ed662bd1ed9d6c Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 19 Apr 2022 16:50:20 -0400 Subject: [PATCH 41/76] Add stricter verification of issuers PEM format This ensures each issuer is only a single certificate entry (as validated by count and parsing) without any trailing data. We further ensure that each certificate PEM has leading and trailing spaces removed with only a single trailing new line remaining. Signed-off-by: Alexander Scheel --- builtin/logical/pki/path_manage_issuers.go | 5 ++++ builtin/logical/pki/storage.go | 25 ++++++++++++++++++- .../logical/pki/storage_migrations_test.go | 2 +- builtin/logical/pki/storage_test.go | 9 ++++--- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index f0b8c3f5e86c4..d251a014c9c35 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -130,6 +130,11 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d var issuers []string var keys []string + // By decoding and re-encoding PEM blobs, we can pass strict PEM blobs + // to the import functionality (importKeys, importIssuers). This allows + // them to validate no duplicate issuers exist (and place greater + // restrictions during parsing) but allows this code to accept OpenSSL + // parsed chains (with full textual output between PEM entries). for len(bytes.TrimSpace(pemBytes)) > 0 { pemBlock, pemBytes = pem.Decode(pemBytes) if pemBlock == nil { diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index fbcd95d0fa635..e5e831e0fa873 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -250,10 +250,13 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName } func (i issuer) GetCertificate() (*x509.Certificate, error) { - block, _ := pem.Decode([]byte(i.Certificate)) + block, extra := pem.Decode([]byte(i.Certificate)) if block == nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to parse certificate from issuer: invalid PEM: %v", i.ID)} } + if len(strings.TrimSpace(string(extra))) > 0 { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to parse certificate for issuer (%v): trailing PEM data: %v", i.ID, string(extra))} + } return x509.ParseCertificate(block.Bytes) } @@ -373,7 +376,21 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu // already existed during import (in which case, *issuer points to the // existing issuer reference and identifier); the last return field is // whether or not an error occurred. + + // Before we begin, we need to ensure the PEM formatted certificate looks + // good. Restricting to "just" `CERTIFICATE` entries is a little + // restrictive, as it could be a `X509 CERTIFICATE` entry or a custom + // value wrapping an actual DER cert. So validating the contents of the + // PEM header is out of the question (and validating the contents of the + // PEM block is left to our GetCertificate call below). + // + // However, we should trim all leading and trailing spaces and add a + // single new line. This allows callers to blindly concatenate PEM + // blobs from the API and get roughly what they'd expect. // + // Discussed further in #11960 and RFC 7468. + certValue = strings.TrimSpace(certValue) + "\n" + // Before we can import a known issuer, we first need to know if the issuer // exists in storage already. This means iterating through all known // issuers and comparing their private value against this value. @@ -412,6 +429,12 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu result.Name = issuerName result.Certificate = certValue + // We shouldn't add CSRs or multiple certificates in this + countCertificates := strings.Count(result.Certificate, "-BEGIN ") + if countCertificates != 1 { + return nil, false, fmt.Errorf("bad issuer: potentially multiple PEM blobs in one certificate storage entry:\n%v", result.Certificate) + } + // Extracting the certificate is necessary for two reasons: first, it lets // us fetch the serial number; second, for the public key comparison with // known keys. diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index 02936180eeeeb..3e1e531c16f48 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -76,7 +76,7 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { require.Equal(t, issuerId, issuer.ID) require.Equal(t, bundle.SerialNumber, issuer.SerialNumber) - require.Equal(t, bundle.Certificate, issuer.Certificate) + require.Equal(t, strings.TrimSpace(bundle.Certificate), strings.TrimSpace(issuer.Certificate)) require.Equal(t, keyId, issuer.KeyID) // FIXME: Add tests for CAChain... diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index a47a324c3c417..eef5ec5022070 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -3,6 +3,7 @@ package pki import ( "context" "crypto/rand" + "strings" "testing" "github.com/hashicorp/vault/sdk/framework" @@ -119,7 +120,7 @@ func Test_KeysIssuerImport(t *testing.T) { issuer1_ref1, existing, err := importIssuer(ctx, s, issuer1.Certificate, "issuer1") require.NoError(t, err) require.False(t, existing) - require.Equal(t, issuer1.Certificate, issuer1_ref1.Certificate) + require.Equal(t, strings.TrimSpace(issuer1.Certificate), strings.TrimSpace(issuer1_ref1.Certificate)) require.Equal(t, key1_ref1.ID, issuer1_ref1.KeyID) require.Equal(t, "issuer1", issuer1_ref1.Name) @@ -128,7 +129,7 @@ func Test_KeysIssuerImport(t *testing.T) { issuer1_ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) - require.Equal(t, issuer1.Certificate, issuer1_ref1.Certificate) + require.Equal(t, strings.TrimSpace(issuer1.Certificate), strings.TrimSpace(issuer1_ref1.Certificate)) require.Equal(t, issuer1_ref1.ID, issuer1_ref2.ID) require.Equal(t, key1_ref1.ID, issuer1_ref2.KeyID) require.Equal(t, issuer1_ref1.Name, issuer1_ref2.Name) @@ -143,7 +144,7 @@ func Test_KeysIssuerImport(t *testing.T) { issuer2_ref, existing, err := importIssuer(ctx, s, issuer2.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) - require.Equal(t, issuer2.Certificate, issuer2_ref.Certificate) + require.Equal(t, strings.TrimSpace(issuer2.Certificate), strings.TrimSpace(issuer2_ref.Certificate)) require.Equal(t, issuer2.ID, issuer2_ref.ID) require.Equal(t, "", issuer2_ref.Name) require.Equal(t, issuer2.KeyID, issuer2_ref.KeyID) @@ -173,7 +174,7 @@ func genIssuerAndKey(t *testing.T, b *backend) (issuer, key) { pkiIssuer := issuer{ ID: issuerId, KeyID: keyId, - Certificate: certBundle.Certificate, + Certificate: strings.TrimSpace(certBundle.Certificate) + "\n", CAChain: certBundle.CAChain, SerialNumber: certBundle.SerialNumber, } From 3768fdee8d458d8b47c8023fb683e75425cd8071 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 12 Apr 2022 11:45:45 -0400 Subject: [PATCH 42/76] Fix full chain building Don't set the legacy IssuingCA field on the certificate bundle, as we prefer the CAChain field over it. Additionally, building the full chain could result in duplicate certificates when the CAChain included the leaf certificate itself. When building the full chain, ensure we don't include the bundle's certificate twice. Signed-off-by: Alexander Scheel --- builtin/logical/pki/storage.go | 1 - sdk/helper/certutil/types.go | 12 ++++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index e5e831e0fa873..703d7522352b9 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -604,7 +604,6 @@ func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuer var bundle certutil.CertBundle bundle.Certificate = issuer.Certificate - bundle.IssuingCA = issuer.CAChain[0] bundle.CAChain = issuer.CAChain bundle.SerialNumber = issuer.SerialNumber diff --git a/sdk/helper/certutil/types.go b/sdk/helper/certutil/types.go index aab082dc08dc7..c648d797fbe01 100644 --- a/sdk/helper/certutil/types.go +++ b/sdk/helper/certutil/types.go @@ -704,10 +704,14 @@ func (b *CAInfoBundle) GetCAChain() []*CertBlock { func (b *CAInfoBundle) GetFullChain() []*CertBlock { var chain []*CertBlock - chain = append(chain, &CertBlock{ - Certificate: b.Certificate, - Bytes: b.CertificateBytes, - }) + // Some bundles already include the root included in the chain, + // so don't include it twice. + if len(b.CAChain) == 0 || !bytes.Equal(b.CAChain[0].Bytes, b.CertificateBytes) { + chain = append(chain, &CertBlock{ + Certificate: b.Certificate, + Bytes: b.CertificateBytes, + }) + } if len(b.CAChain) > 0 { chain = append(chain, b.CAChain...) From 3ff4dfd43a081ab06a50511562557f42ab370a4a Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 12 Apr 2022 11:56:39 -0400 Subject: [PATCH 43/76] Add stricter tests for full chain construction We wish to ensure that each desired certificate in the chain is only present once. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index da9f032fbac8f..9498835f6b655 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -4066,8 +4066,8 @@ func runFullCAChainTest(t *testing.T, keyType string) { } fullChain := resp.Data["ca_chain"].(string) - if !strings.Contains(fullChain, rootCert) { - t.Fatal("expected full chain to contain root certificate") + if strings.Count(fullChain, rootCert) != 1 { + t.Fatalf("expected full chain to contain root certificate; got %v occurrences", strings.Count(fullChain, rootCert)) } // Now generate an intermediate at /pki-intermediate, signed by the root. @@ -4134,11 +4134,11 @@ func runFullCAChainTest(t *testing.T, keyType string) { require.Equal(t, 0, len(crl.TBSCertList.RevokedCertificates)) fullChain = resp.Data["ca_chain"].(string) - if !strings.Contains(fullChain, intermediateCert) { - t.Fatal("expected full chain to contain intermediate certificate") + if strings.Count(fullChain, intermediateCert) != 1 { + t.Fatalf("expected full chain to contain intermediate certificate; got %v occurrences", strings.Count(fullChain, intermediateCert)) } - if !strings.Contains(fullChain, rootCert) { - t.Fatal("expected full chain to contain root certificate") + if strings.Count(fullChain, rootCert) != 1 { + t.Fatalf("expected full chain to contain root certificate; got %v occurrences", strings.Count(fullChain, rootCert)) } // Finally, import this signing cert chain into a new mount to ensure @@ -4171,11 +4171,11 @@ func runFullCAChainTest(t *testing.T, keyType string) { } fullChain = resp.Data["ca_chain"].(string) - if !strings.Contains(fullChain, intermediateCert) { - t.Fatal("expected full chain to contain intermediate certificate") + if strings.Count(fullChain, intermediateCert) != 1 { + t.Fatalf("expected full chain to contain intermediate certificate; got %v occurrences", strings.Count(fullChain, intermediateCert)) } - if !strings.Contains(fullChain, rootCert) { - t.Fatal("expected full chain to contain root certificate") + if strings.Count(fullChain, rootCert) != 1 { + t.Fatalf("expected full chain to contain root certificate; got %v occurrences", strings.Count(fullChain, rootCert)) } // Now issue a short-lived certificate from our pki-external. From 46a2e48a71ca7a33461f72252891639312bc1c1c Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Wed, 20 Apr 2022 12:58:29 -0400 Subject: [PATCH 44/76] Rename PKI types to avoid constant variable name collisions keyId -> keyID issuerId -> issuerID key -> keyEntry issuer -> issuerEntry keyConfig -> keyConfigEntry issuerConfig -> issuerConfigEntry --- builtin/logical/pki/ca_util.go | 2 - builtin/logical/pki/chain_util.go | 164 +++++++++--------- builtin/logical/pki/config_util.go | 8 +- builtin/logical/pki/path_fetch_issuers.go | 2 +- builtin/logical/pki/storage.go | 162 ++++++++--------- .../logical/pki/storage_migrations_test.go | 4 +- builtin/logical/pki/storage_test.go | 18 +- builtin/logical/pki/util.go | 12 +- 8 files changed, 186 insertions(+), 186 deletions(-) diff --git a/builtin/logical/pki/ca_util.go b/builtin/logical/pki/ca_util.go index b89cd087bc069..3265a2aae7090 100644 --- a/builtin/logical/pki/ca_util.go +++ b/builtin/logical/pki/ca_util.go @@ -104,7 +104,6 @@ func generateCSRBundle(ctx context.Context, b *backend, input *inputBundle, data return certutil.CreateCSRWithKeyGenerator(data, addBasicConstraints, randomSource, existingGeneratePrivateKey(ctx, input.req.Storage, keyRef)) } - return certutil.CreateCSRWithRandomSource(data, addBasicConstraints, randomSource) } @@ -239,4 +238,3 @@ func existingGeneratePrivateKey(ctx context.Context, s logical.Storage, keyRef s return nil } } - diff --git a/builtin/logical/pki/chain_util.go b/builtin/logical/pki/chain_util.go index 953a4e1376522..1d9caaec6f422 100644 --- a/builtin/logical/pki/chain_util.go +++ b/builtin/logical/pki/chain_util.go @@ -10,7 +10,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -func prettyIssuer(issuerIdEntryMap map[issuerId]*issuer, issuer issuerId) string { +func prettyIssuer(issuerIdEntryMap map[issuerID]*issuerEntry, issuer issuerID) string { if entry, ok := issuerIdEntryMap[issuer]; ok && len(entry.Name) > 0 { return "[id:" + string(issuer) + "/name:" + entry.Name + "]" } @@ -18,7 +18,7 @@ func prettyIssuer(issuerIdEntryMap map[issuerId]*issuer, issuer issuerId) string return "[" + string(issuer) + "]" } -func rebuildIssuersChains(ctx context.Context, s logical.Storage, referenceCert *issuer /* optional */) error { +func rebuildIssuersChains(ctx context.Context, s logical.Storage, referenceCert *issuerEntry /* optional */) error { // This function rebuilds the CAChain field of all known issuers. This // function should usually be invoked when a new issuer is added to the // pool of issuers. @@ -92,22 +92,22 @@ func rebuildIssuersChains(ctx context.Context, s logical.Storage, referenceCert // fourth maps that certificate back to the other issuers with that // subject (note the keyword _other_: we'll exclude self-loops here) -- // either via a parent or child relationship. - issuerIdEntryMap := make(map[issuerId]*issuer, len(issuers)) - issuerIdCertMap := make(map[issuerId]*x509.Certificate, len(issuers)) - issuerIdParentsMap := make(map[issuerId][]issuerId, len(issuers)) - issuerIdChildrenMap := make(map[issuerId][]issuerId, len(issuers)) + issuerIdEntryMap := make(map[issuerID]*issuerEntry, len(issuers)) + issuerIdCertMap := make(map[issuerID]*x509.Certificate, len(issuers)) + issuerIdParentsMap := make(map[issuerID][]issuerID, len(issuers)) + issuerIdChildrenMap := make(map[issuerID][]issuerID, len(issuers)) // For every known issuer, we map that subject back to the id of issuers - // containing that subject. This lets us build our issuerId -> parents + // containing that subject. This lets us build our issuerID -> parents // mapping efficiently. Worst case we'll have a single linear chain where // every entry has a distinct subject. - subjectIssuerIdsMap := make(map[string][]issuerId, len(issuers)) + subjectIssuerIdsMap := make(map[string][]issuerID, len(issuers)) // First, read every issuer entry from storage. We'll propagate entries // to three of the maps here: all but issuerIdParentsMap and // issuerIdChildrenMap, which we'll do in a second pass. for _, identifier := range issuers { - var stored *issuer + var stored *issuerEntry // When the reference issuer is provided and matches this identifier, // prefer the updated reference copy instead. @@ -237,8 +237,8 @@ func rebuildIssuersChains(ctx context.Context, s logical.Storage, referenceCert // manually building their chain prior to starting the topographical sort. // // This thus runs in O(|V| + |E|) -> O(n^2) in the number of issuers. - processedIssuers := make(map[issuerId]bool, len(issuers)) - toVisit := make([]issuerId, 0, len(issuers)) + processedIssuers := make(map[issuerID]bool, len(issuers)) + toVisit := make([]issuerID, 0, len(issuers)) // Handle any explicitly constructed certificate chains. Here, we don't // validate much what the user provides; if they provide since-deleted @@ -301,7 +301,7 @@ func rebuildIssuersChains(ctx context.Context, s logical.Storage, referenceCert // ensure we don't accidentally infinite-loop (if we introduce a bug). maxVisitCount := len(issuers)*len(issuers)*len(issuers) + 100 for len(toVisit) > 0 && maxVisitCount >= 0 { - var issuer issuerId + var issuer issuerID issuer, toVisit = toVisit[0], toVisit[1:] // If (and only if) we're presently starved for next nodes to visit, @@ -406,7 +406,7 @@ func rebuildIssuersChains(ctx context.Context, s logical.Storage, referenceCert return nil } -func addToChainIfNotExisting(includedParentCerts map[string]bool, entry *issuer, certToAdd string) { +func addToChainIfNotExisting(includedParentCerts map[string]bool, entry *issuerEntry, certToAdd string) { included, ok := includedParentCerts[certToAdd] if ok && included { return @@ -417,15 +417,15 @@ func addToChainIfNotExisting(includedParentCerts map[string]bool, entry *issuer, } func processAnyCliqueOrCycle( - issuers []issuerId, - processedIssuers map[issuerId]bool, - toVisit []issuerId, - issuerIdEntryMap map[issuerId]*issuer, - issuerIdCertMap map[issuerId]*x509.Certificate, - issuerIdParentsMap map[issuerId][]issuerId, - issuerIdChildrenMap map[issuerId][]issuerId, - subjectIssuerIdsMap map[string][]issuerId, -) ([]issuerId /* toVisit */, error) { + issuers []issuerID, + processedIssuers map[issuerID]bool, + toVisit []issuerID, + issuerIdEntryMap map[issuerID]*issuerEntry, + issuerIdCertMap map[issuerID]*x509.Certificate, + issuerIdParentsMap map[issuerID][]issuerID, + issuerIdChildrenMap map[issuerID][]issuerID, + subjectIssuerIdsMap map[string][]issuerID, +) ([]issuerID /* toVisit */, error) { // Topological sort really only works on directed acyclic graphs (DAGs). // But a pool of arbitrary (issuer) certificates are actually neither! // This pool could contain both cliques and cycles. Because this could @@ -486,15 +486,15 @@ func processAnyCliqueOrCycle( // Finally -- it isn't enough to consider this chain in isolation // either. We need to consider _all_ parents and ensure they've been // processed before processing this closure. - var cliques [][]issuerId - var cycles [][]issuerId - closure := make(map[issuerId]bool) + var cliques [][]issuerID + var cycles [][]issuerID + closure := make(map[issuerID]bool) - var cliquesToProcess []issuerId + var cliquesToProcess []issuerID cliquesToProcess = append(cliquesToProcess, issuer) for len(cliquesToProcess) > 0 { - var node issuerId + var node issuerID node, cliquesToProcess = cliquesToProcess[0], cliquesToProcess[1:] // Skip potential clique nodes which have already been processed @@ -666,7 +666,7 @@ func processAnyCliqueOrCycle( // Unable to find node; return an error. This shouldn't happen // generally. pretty := prettyIssuer(issuerIdEntryMap, issuer) - return nil, fmt.Errorf("Unable to find node (%v) in closure (%v) but not in cycles (%v) or cliques (%v)", pretty, closure, cycles, cliques) + return nil, fmt.Errorf("unable to find node (%v) in closure (%v) but not in cycles (%v) or cliques (%v)", pretty, closure, cycles, cliques) } } } @@ -691,7 +691,7 @@ func processAnyCliqueOrCycle( return nil, err } - closure := make(map[issuerId]bool) + closure := make(map[issuerID]bool) for _, cycle := range cycles { for _, node := range cycle { closure[node] = true @@ -749,14 +749,14 @@ func processAnyCliqueOrCycle( } func findAllCliques( - processedIssuers map[issuerId]bool, - issuerIdCertMap map[issuerId]*x509.Certificate, - subjectIssuerIdsMap map[string][]issuerId, - issuers []issuerId, -) ([][]issuerId, map[issuerId]int, []issuerId, error) { - var allCliques [][]issuerId - issuerIdCliqueMap := make(map[issuerId]int) - var allCliqueNodes []issuerId + processedIssuers map[issuerID]bool, + issuerIdCertMap map[issuerID]*x509.Certificate, + subjectIssuerIdsMap map[string][]issuerID, + issuers []issuerID, +) ([][]issuerID, map[issuerID]int, []issuerID, error) { + var allCliques [][]issuerID + issuerIdCliqueMap := make(map[issuerID]int) + var allCliqueNodes []issuerID for _, node := range issuers { // Check if the node has already been visited... @@ -797,11 +797,11 @@ func findAllCliques( } func isOnReissuedClique( - processedIssuers map[issuerId]bool, - issuerIdCertMap map[issuerId]*x509.Certificate, - subjectIssuerIdsMap map[string][]issuerId, - node issuerId, -) ([]issuerId, error) { + processedIssuers map[issuerID]bool, + issuerIdCertMap map[issuerID]*x509.Certificate, + subjectIssuerIdsMap map[string][]issuerID, + node issuerID, +) ([]issuerID, error) { // Finding max cliques in arbitrary graphs is a nearly pathological // problem, usually left to the realm of SAT solvers and NP-Complete // theoretical. @@ -829,7 +829,7 @@ func isOnReissuedClique( // under this reissued clique detection code). // // What does this mean for our algorithm? A simple greedy search is - // sufficient. If we index our certificates by subject -> issuerId + // sufficient. If we index our certificates by subject -> issuerID // (and cache its value across calls, which we've already done for // building the parent/child relationship), we can find all other issuers // with the same public key and subject as the existing node fairly @@ -863,7 +863,7 @@ func isOnReissuedClique( // condition (the subject half), so validate they match the other half // (the issuer half) and the second condition. For node (which is // included in candidates), the condition should vacuously hold. - var clique []issuerId + var clique []issuerID for _, candidate := range candidates { // Skip already processed nodes, even if they could be clique // candidates. We'll treat them as any other (already processed) @@ -895,7 +895,7 @@ func isOnReissuedClique( return clique, nil } -func containsIssuer(collection []issuerId, target issuerId) bool { +func containsIssuer(collection []issuerID, target issuerID) bool { if len(collection) == 0 { return false } @@ -909,7 +909,7 @@ func containsIssuer(collection []issuerId, target issuerId) bool { return false } -func appendCycleIfNotExisting(knownCycles [][]issuerId, candidate []issuerId) [][]issuerId { +func appendCycleIfNotExisting(knownCycles [][]issuerID, candidate []issuerID) [][]issuerID { // There's two ways to do cycle detection: canonicalize the cycles, // rewriting them to have the least (or max) element first or just // brute force the detection. @@ -945,7 +945,7 @@ func appendCycleIfNotExisting(knownCycles [][]issuerId, candidate []issuerId) [] return knownCycles } -func canonicalizeCycle(cycle []issuerId) []issuerId { +func canonicalizeCycle(cycle []issuerID) []issuerID { // Find the minimum value and put it at the head, keeping the relative // ordering the same. minIndex := 0 @@ -964,10 +964,10 @@ func canonicalizeCycle(cycle []issuerId) []issuerId { } func findCyclesNearClique( - processedIssuers map[issuerId]bool, - issuerIdChildrenMap map[issuerId][]issuerId, - cliqueNodes []issuerId, -) ([][]issuerId, error) { + processedIssuers map[issuerID]bool, + issuerIdChildrenMap map[issuerID][]issuerID, + cliqueNodes []issuerID, +) ([][]issuerID, error) { // When we have a reissued clique, we need to find all cycles next to it. // Presumably, because they all have non-empty parents, they should not // have been visited yet. We further know that (because we're exploring @@ -983,7 +983,7 @@ func findCyclesNearClique( // Copy the clique nodes as excluded nodes; we'll avoid exploring cycles // which have parents that have been already explored. excludeNodes := cliqueNodes[:] - var knownCycles [][]issuerId + var knownCycles [][]issuerID // We know the node has at least one child, since the clique is non-empty. for _, child := range issuerIdChildrenMap[cliqueNode] { @@ -1013,11 +1013,11 @@ func findCyclesNearClique( } func findAllCyclesWithNode( - processedIssuers map[issuerId]bool, - issuerIdChildrenMap map[issuerId][]issuerId, - source issuerId, - exclude []issuerId, -) ([][]issuerId, error) { + processedIssuers map[issuerID]bool, + issuerIdChildrenMap map[issuerID][]issuerID, + source issuerID, + exclude []issuerID, +) ([][]issuerID, error) { // We wish to find all cycles involving this particular node and report // the corresponding paths. This is a full-graph traversal (excluding // certain paths) as we're not just checking if a cycle occurred, but @@ -1027,28 +1027,28 @@ func findAllCyclesWithNode( maxCycleSize := 8 // Whether we've visited any given node. - cycleVisited := make(map[issuerId]bool) - visitCounts := make(map[issuerId]int) - parentCounts := make(map[issuerId]map[issuerId]bool) + cycleVisited := make(map[issuerID]bool) + visitCounts := make(map[issuerID]int) + parentCounts := make(map[issuerID]map[issuerID]bool) // Paths to the specified node. Some of these might be cycles. - pathsTo := make(map[issuerId][][]issuerId) + pathsTo := make(map[issuerID][][]issuerID) // Nodes to visit. - var visitQueue []issuerId + var visitQueue []issuerID // Add the source node to start. In order to set up the paths to a // given node, we seed pathsTo with the single path involving just // this node visitQueue = append(visitQueue, source) - pathsTo[source] = [][]issuerId{{source}} + pathsTo[source] = [][]issuerID{{source}} // Begin building paths. // // Loop invariant: // pathTo[x] contains valid paths to reach this node, from source. for len(visitQueue) > 0 { - var current issuerId + var current issuerID current, visitQueue = visitQueue[0], visitQueue[1:] // If we've already processed this node, we have a cycle. Skip this @@ -1093,7 +1093,7 @@ func findAllCyclesWithNode( // Track this parent->child relationship to know when to exit. setOfParents, ok := parentCounts[child] if !ok { - setOfParents = make(map[issuerId]bool) + setOfParents = make(map[issuerID]bool) parentCounts[child] = setOfParents } _, existingParent := setOfParents[current] @@ -1110,7 +1110,7 @@ func findAllCyclesWithNode( // externally with an existing path). addedPath := false if _, ok := pathsTo[child]; !ok { - pathsTo[child] = make([][]issuerId, 0) + pathsTo[child] = make([][]issuerID, 0) } for _, path := range pathsTo[current] { if child != source { @@ -1131,7 +1131,7 @@ func findAllCyclesWithNode( } // Make sure to deep copy the path. - newPath := make([]issuerId, 0, len(path)+1) + newPath := make([]issuerID, 0, len(path)+1) newPath = append(newPath, path...) newPath = append(newPath, child) @@ -1176,7 +1176,7 @@ func findAllCyclesWithNode( // Ok, we've now exited from our loop. Any cycles would've been detected // and their paths recorded in pathsTo. Now we can iterate over these // (starting a source), clean them up and validate them. - var cycles [][]issuerId + var cycles [][]issuerID for _, cycle := range pathsTo[source] { // Skip the trivial cycle. if len(cycle) == 1 && cycle[0] == source { @@ -1209,8 +1209,8 @@ func findAllCyclesWithNode( return cycles, nil } -func reversedCycle(cycle []issuerId) []issuerId { - var result []issuerId +func reversedCycle(cycle []issuerID) []issuerID { + var result []issuerID for index := len(cycle) - 1; index >= 0; index-- { result = append(result, cycle[index]) } @@ -1219,11 +1219,11 @@ func reversedCycle(cycle []issuerId) []issuerId { } func computeParentsFromClosure( - processedIssuers map[issuerId]bool, - issuerIdParentsMap map[issuerId][]issuerId, - closure map[issuerId]bool, -) (map[issuerId]bool, bool) { - parents := make(map[issuerId]bool) + processedIssuers map[issuerID]bool, + issuerIdParentsMap map[issuerID][]issuerID, + closure map[issuerID]bool, +) (map[issuerID]bool, bool) { + parents := make(map[issuerID]bool) for node := range closure { nodeParents, ok := issuerIdParentsMap[node] if !ok { @@ -1248,11 +1248,11 @@ func computeParentsFromClosure( } func addNodeCertsToEntry( - issuerIdEntryMap map[issuerId]*issuer, - issuerIdChildrenMap map[issuerId][]issuerId, + issuerIdEntryMap map[issuerID]*issuerEntry, + issuerIdChildrenMap map[issuerID][]issuerID, includedParentCerts map[string]bool, - entry *issuer, - issuersCollection ...[]issuerId, + entry *issuerEntry, + issuersCollection ...[]issuerID, ) { for _, collection := range issuersCollection { // Find a starting point into this collection such that it verifies @@ -1291,10 +1291,10 @@ func addNodeCertsToEntry( } func addParentChainsToEntry( - issuerIdEntryMap map[issuerId]*issuer, + issuerIdEntryMap map[issuerID]*issuerEntry, includedParentCerts map[string]bool, - entry *issuer, - parents map[issuerId]bool, + entry *issuerEntry, + parents map[issuerID]bool, ) { for parent := range parents { nodeEntry := issuerIdEntryMap[parent] diff --git a/builtin/logical/pki/config_util.go b/builtin/logical/pki/config_util.go index 0dddc620ab27e..6a926c2a3736f 100644 --- a/builtin/logical/pki/config_util.go +++ b/builtin/logical/pki/config_util.go @@ -25,14 +25,14 @@ func isDefaultIssuerSet(ctx context.Context, s logical.Storage) (bool, error) { return strings.TrimSpace(config.DefaultIssuerId.String()) != "", nil } -func updateDefaultKeyId(ctx context.Context, s logical.Storage, id keyId) error { +func updateDefaultKeyId(ctx context.Context, s logical.Storage, id keyID) error { config, err := getKeysConfig(ctx, s) if err != nil { return err } if config.DefaultKeyId != id { - return setKeysConfig(ctx, s, &keyConfig{ + return setKeysConfig(ctx, s, &keyConfigEntry{ DefaultKeyId: id, }) } @@ -40,14 +40,14 @@ func updateDefaultKeyId(ctx context.Context, s logical.Storage, id keyId) error return nil } -func updateDefaultIssuerId(ctx context.Context, s logical.Storage, id issuerId) error { +func updateDefaultIssuerId(ctx context.Context, s logical.Storage, id issuerID) error { config, err := getIssuersConfig(ctx, s) if err != nil { return err } if config.DefaultIssuerId != id { - return setIssuersConfig(ctx, s, &issuerConfig{ + return setIssuersConfig(ctx, s, &issuerConfigEntry{ DefaultIssuerId: id, }) } diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 317eeddc68265..e6c2045fa3ae9 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -167,7 +167,7 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da } var updateChain bool - var constructedChain []issuerId + var constructedChain []issuerID for index, newPathRef := range newPath { // Allow self for the first entry. if index == 0 && newPathRef == "self" { diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 703d7522352b9..0c1390d519b71 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -24,90 +24,90 @@ const ( legacyCertBundlePath = "config/ca_bundle" ) -type keyId string +type keyID string -func (p keyId) String() string { +func (p keyID) String() string { return string(p) } -type issuerId string +type issuerID string -func (p issuerId) String() string { +func (p issuerID) String() string { return string(p) } const ( - IssuerRefNotFound = issuerId("not-found") - KeyRefNotFound = keyId("not-found") + IssuerRefNotFound = issuerID("not-found") + KeyRefNotFound = keyID("not-found") ) -type key struct { - ID keyId `json:"id" structs:"id" mapstructure:"id"` +type keyEntry struct { + ID keyID `json:"id" structs:"id" mapstructure:"id"` Name string `json:"name" structs:"name" mapstructure:"name"` PrivateKeyType certutil.PrivateKeyType `json:"private_key_type" structs:"private_key_type" mapstructure:"private_key_type"` PrivateKey string `json:"private_key" structs:"private_key" mapstructure:"private_key"` } -type issuer struct { - ID issuerId `json:"id" structs:"id" mapstructure:"id"` +type issuerEntry struct { + ID issuerID `json:"id" structs:"id" mapstructure:"id"` Name string `json:"name" structs:"name" mapstructure:"name"` - KeyID keyId `json:"key_id" structs:"key_id" mapstructure:"key_id"` + KeyID keyID `json:"key_id" structs:"key_id" mapstructure:"key_id"` Certificate string `json:"certificate" structs:"certificate" mapstructure:"certificate"` CAChain []string `json:"ca_chain" structs:"ca_chain" mapstructure:"ca_chain"` - ManualChain []issuerId `json:"manual_chain" structs:"manual_chain" mapstructure:"manual_chain"` + ManualChain []issuerID `json:"manual_chain" structs:"manual_chain" mapstructure:"manual_chain"` SerialNumber string `json:"serial_number" structs:"serial_number" mapstructure:"serial_number"` } -type keyConfig struct { - DefaultKeyId keyId `json:"default" structs:"default" mapstructure:"default"` +type keyConfigEntry struct { + DefaultKeyId keyID `json:"default" structs:"default" mapstructure:"default"` } -type issuerConfig struct { - DefaultIssuerId issuerId `json:"default" structs:"default" mapstructure:"default"` +type issuerConfigEntry struct { + DefaultIssuerId issuerID `json:"default" structs:"default" mapstructure:"default"` } -func (k key) GetSigner() (crypto.Signer, error) { +func (k keyEntry) GetSigner() (crypto.Signer, error) { signer, _, err := certutil.ParsePEMKey(k.PrivateKey) return signer, err } -func listKeys(ctx context.Context, s logical.Storage) ([]keyId, error) { +func listKeys(ctx context.Context, s logical.Storage) ([]keyID, error) { strList, err := s.List(ctx, keyPrefix) if err != nil { return nil, err } - keyIds := make([]keyId, 0, len(strList)) + keyIds := make([]keyID, 0, len(strList)) for _, entry := range strList { - keyIds = append(keyIds, keyId(entry)) + keyIds = append(keyIds, keyID(entry)) } return keyIds, nil } -func fetchKeyById(ctx context.Context, s logical.Storage, keyId keyId) (*key, error) { +func fetchKeyById(ctx context.Context, s logical.Storage, keyId keyID) (*keyEntry, error) { if len(keyId) == 0 { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki key: empty key identifier")} } - keyEntry, err := s.Get(ctx, keyPrefix+keyId.String()) + entry, err := s.Get(ctx, keyPrefix+keyId.String()) if err != nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki key: %v", err)} } - if keyEntry == nil { + if entry == nil { // FIXME: Dedicated/specific error for this? return nil, errutil.UserError{Err: fmt.Sprintf("pki key id %s does not exist", keyId.String())} } - var key key - if err := keyEntry.DecodeJSON(&key); err != nil { + var key keyEntry + if err := entry.DecodeJSON(&key); err != nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode pki key with id %s: %v", keyId.String(), err)} } return &key, nil } -func writeKey(ctx context.Context, s logical.Storage, key key) error { +func writeKey(ctx context.Context, s logical.Storage, key keyEntry) error { keyId := key.ID json, err := logical.StorageEntryJSON(keyPrefix+keyId.String(), key) @@ -118,7 +118,7 @@ func writeKey(ctx context.Context, s logical.Storage, key key) error { return s.Put(ctx, json) } -func deleteKey(ctx context.Context, s logical.Storage, id keyId) (bool, error) { +func deleteKey(ctx context.Context, s logical.Storage, id keyID) (bool, error) { wasDefault := false config, err := getKeysConfig(ctx, s) @@ -128,7 +128,7 @@ func deleteKey(ctx context.Context, s logical.Storage, id keyId) (bool, error) { if config.DefaultKeyId == id { wasDefault = true - config.DefaultKeyId = keyId("") + config.DefaultKeyId = keyID("") if err := setKeysConfig(ctx, s, config); err != nil { return wasDefault, err } @@ -137,7 +137,7 @@ func deleteKey(ctx context.Context, s logical.Storage, id keyId) (bool, error) { return wasDefault, s.Delete(ctx, keyPrefix+id.String()) } -func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName string) (*key, bool, error) { +func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName string) (*keyEntry, bool, error) { // importKey imports the specified PEM-format key (from keyValue) into // the new PKI storage format. The first return field is a reference to // the new key; the second is whether or not the key already existed @@ -177,7 +177,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName } // Haven't found a key, so we've gotta create it and write it into storage. - var result key + var result keyEntry result.ID = genKeyId() result.Name = keyName result.PrivateKey = keyValue @@ -249,7 +249,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName return &result, false, nil } -func (i issuer) GetCertificate() (*x509.Certificate, error) { +func (i issuerEntry) GetCertificate() (*x509.Certificate, error) { block, extra := pem.Decode([]byte(i.Certificate)) if block == nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to parse certificate from issuer: invalid PEM: %v", i.ID)} @@ -261,26 +261,26 @@ func (i issuer) GetCertificate() (*x509.Certificate, error) { return x509.ParseCertificate(block.Bytes) } -func listIssuers(ctx context.Context, s logical.Storage) ([]issuerId, error) { +func listIssuers(ctx context.Context, s logical.Storage) ([]issuerID, error) { strList, err := s.List(ctx, issuerPrefix) if err != nil { return nil, err } - issuerIds := make([]issuerId, 0, len(strList)) + issuerIds := make([]issuerID, 0, len(strList)) for _, entry := range strList { - issuerIds = append(issuerIds, issuerId(entry)) + issuerIds = append(issuerIds, issuerID(entry)) } return issuerIds, nil } -func resolveKeyReference(ctx context.Context, s logical.Storage, reference string) (keyId, error) { +func resolveKeyReference(ctx context.Context, s logical.Storage, reference string) (keyID, error) { if reference == defaultRef { // Handle fetching the default key. config, err := getKeysConfig(ctx, s) if err != nil { - return keyId("config-error"), err + return keyID("config-error"), err } if len(config.DefaultKeyId) == 0 { return KeyRefNotFound, fmt.Errorf("no default key currently configured") @@ -291,21 +291,21 @@ func resolveKeyReference(ctx context.Context, s logical.Storage, reference strin keys, err := listKeys(ctx, s) if err != nil { - return keyId("list-error"), err + return keyID("list-error"), err } // Cheaper to list keys and check if an id is a match... - for _, key_id := range keys { - if key_id == keyId(reference) { - return key_id, nil + for _, keyId := range keys { + if keyId == keyID(reference) { + return keyId, nil } } // ... than to pull all keys from storage. - for _, key_id := range keys { - key, err := fetchKeyById(ctx, s, key_id) + for _, keyId := range keys { + key, err := fetchKeyById(ctx, s, keyId) if err != nil { - return keyId("key-read"), err + return keyID("key-read"), err } if key.Name == reference { @@ -317,29 +317,29 @@ func resolveKeyReference(ctx context.Context, s logical.Storage, reference strin return KeyRefNotFound, errutil.UserError{Err: fmt.Sprintf("unable to find PKI key for reference: %v", reference)} } -func fetchIssuerById(ctx context.Context, s logical.Storage, issuerId issuerId) (*issuer, error) { +func fetchIssuerById(ctx context.Context, s logical.Storage, issuerId issuerID) (*issuerEntry, error) { if len(issuerId) == 0 { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki issuer: empty issuer identifier")} } - issuerEntry, err := s.Get(ctx, issuerPrefix+issuerId.String()) + entry, err := s.Get(ctx, issuerPrefix+issuerId.String()) if err != nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki issuer: %v", err)} } - if issuerEntry == nil { + if entry == nil { // FIXME: Dedicated/specific error for this? return nil, errutil.UserError{Err: fmt.Sprintf("pki issuer id %s does not exist", issuerId.String())} } - var issuer issuer - if err := issuerEntry.DecodeJSON(&issuer); err != nil { + var issuer issuerEntry + if err := entry.DecodeJSON(&issuer); err != nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode pki issuer with id %s: %v", issuerId.String(), err)} } return &issuer, nil } -func writeIssuer(ctx context.Context, s logical.Storage, issuer *issuer) error { +func writeIssuer(ctx context.Context, s logical.Storage, issuer *issuerEntry) error { issuerId := issuer.ID json, err := logical.StorageEntryJSON(issuerPrefix+issuerId.String(), issuer) @@ -350,7 +350,7 @@ func writeIssuer(ctx context.Context, s logical.Storage, issuer *issuer) error { return s.Put(ctx, json) } -func deleteIssuer(ctx context.Context, s logical.Storage, id issuerId) (bool, error) { +func deleteIssuer(ctx context.Context, s logical.Storage, id issuerID) (bool, error) { wasDefault := false config, err := getIssuersConfig(ctx, s) @@ -360,7 +360,7 @@ func deleteIssuer(ctx context.Context, s logical.Storage, id issuerId) (bool, er if config.DefaultIssuerId == id { wasDefault = true - config.DefaultIssuerId = issuerId("") + config.DefaultIssuerId = issuerID("") if err := setIssuersConfig(ctx, s, config); err != nil { return wasDefault, err } @@ -369,7 +369,7 @@ func deleteIssuer(ctx context.Context, s logical.Storage, id issuerId) (bool, er return wasDefault, s.Delete(ctx, issuerPrefix+id.String()) } -func importIssuer(ctx context.Context, s logical.Storage, certValue string, issuerName string) (*issuer, bool, error) { +func importIssuer(ctx context.Context, s logical.Storage, certValue string, issuerName string) (*issuerEntry, bool, error) { // importIssuers imports the specified PEM-format certificate (from // certValue) into the new PKI storage format. The first return field is a // reference to the new issuer; the second is whether or not the issuer @@ -424,7 +424,7 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu // Haven't found an issuer, so we've gotta create it and write it into // storage. - var result issuer + var result issuerEntry result.ID = genIssuerId() result.Name = issuerName result.Certificate = certValue @@ -502,7 +502,7 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu return &result, false, nil } -func setKeysConfig(ctx context.Context, s logical.Storage, config *keyConfig) error { +func setKeysConfig(ctx context.Context, s logical.Storage, config *keyConfigEntry) error { json, err := logical.StorageEntryJSON(storageKeyConfig, config) if err != nil { return err @@ -511,15 +511,15 @@ func setKeysConfig(ctx context.Context, s logical.Storage, config *keyConfig) er return s.Put(ctx, json) } -func getKeysConfig(ctx context.Context, s logical.Storage) (*keyConfig, error) { - keyConfigEntry, err := s.Get(ctx, storageKeyConfig) +func getKeysConfig(ctx context.Context, s logical.Storage) (*keyConfigEntry, error) { + entry, err := s.Get(ctx, storageKeyConfig) if err != nil { return nil, err } - keyConfig := &keyConfig{} - if keyConfigEntry != nil { - if err := keyConfigEntry.DecodeJSON(keyConfig); err != nil { + keyConfig := &keyConfigEntry{} + if entry != nil { + if err := entry.DecodeJSON(keyConfig); err != nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode key configuration: %v", err)} } } @@ -527,7 +527,7 @@ func getKeysConfig(ctx context.Context, s logical.Storage) (*keyConfig, error) { return keyConfig, nil } -func setIssuersConfig(ctx context.Context, s logical.Storage, config *issuerConfig) error { +func setIssuersConfig(ctx context.Context, s logical.Storage, config *issuerConfigEntry) error { json, err := logical.StorageEntryJSON(storageIssuerConfig, config) if err != nil { return err @@ -536,15 +536,15 @@ func setIssuersConfig(ctx context.Context, s logical.Storage, config *issuerConf return s.Put(ctx, json) } -func getIssuersConfig(ctx context.Context, s logical.Storage) (*issuerConfig, error) { - issuerConfigEntry, err := s.Get(ctx, storageIssuerConfig) +func getIssuersConfig(ctx context.Context, s logical.Storage) (*issuerConfigEntry, error) { + entry, err := s.Get(ctx, storageIssuerConfig) if err != nil { return nil, err } - issuerConfig := &issuerConfig{} - if issuerConfigEntry != nil { - if err := issuerConfigEntry.DecodeJSON(issuerConfig); err != nil { + issuerConfig := &issuerConfigEntry{} + if entry != nil { + if err := entry.DecodeJSON(issuerConfig); err != nil { return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode issuer configuration: %v", err)} } } @@ -552,12 +552,12 @@ func getIssuersConfig(ctx context.Context, s logical.Storage) (*issuerConfig, er return issuerConfig, nil } -func resolveIssuerReference(ctx context.Context, s logical.Storage, reference string) (issuerId, error) { +func resolveIssuerReference(ctx context.Context, s logical.Storage, reference string) (issuerID, error) { if reference == defaultRef { // Handle fetching the default issuer. config, err := getIssuersConfig(ctx, s) if err != nil { - return issuerId("config-error"), err + return issuerID("config-error"), err } if len(config.DefaultIssuerId) == 0 { return IssuerRefNotFound, fmt.Errorf("no default issuer currently configured") @@ -568,21 +568,21 @@ func resolveIssuerReference(ctx context.Context, s logical.Storage, reference st issuers, err := listIssuers(ctx, s) if err != nil { - return issuerId("list-error"), err + return issuerID("list-error"), err } // Cheaper to list issuers and check if an id is a match... - for _, issuer_id := range issuers { - if issuer_id == issuerId(reference) { - return issuer_id, nil + for _, issuerId := range issuers { + if issuerId == issuerID(reference) { + return issuerId, nil } } // ... than to pull all issuers from storage. - for _, issuer_id := range issuers { - issuer, err := fetchIssuerById(ctx, s, issuer_id) + for _, issuerId := range issuers { + issuer, err := fetchIssuerById(ctx, s, issuerId) if err != nil { - return issuerId("issuer-read"), err + return issuerID("issuer-read"), err } if issuer.Name == reference { @@ -596,7 +596,7 @@ func resolveIssuerReference(ctx context.Context, s logical.Storage, reference st // Builds a certutil.CertBundle from the specified issuer identifier, // optionally loading the key or not. -func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuerId, loadKey bool) (*certutil.CertBundle, error) { +func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuerID, loadKey bool) (*certutil.CertBundle, error) { issuer, err := fetchIssuerById(ctx, s, id) if err != nil { return nil, err @@ -608,7 +608,7 @@ func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuer bundle.SerialNumber = issuer.SerialNumber // Fetch the key if it exists. Sometimes we don't need the key immediately. - if loadKey && issuer.KeyID != keyId("") { + if loadKey && issuer.KeyID != keyID("") { key, err := fetchKeyById(ctx, s, issuer.KeyID) if err != nil { return nil, err @@ -621,7 +621,7 @@ func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuer return &bundle, nil } -func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.CertBundle, issuerName string, keyName string) (*issuer, *key, error) { +func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.CertBundle, issuerName string, keyName string) (*issuerEntry, *keyEntry, error) { myKey, _, err := importKey(ctx, s, caBundle.PrivateKey, keyName) if err != nil { return nil, nil, err @@ -641,12 +641,12 @@ func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.Ce return myIssuer, myKey, nil } -func genIssuerId() issuerId { - return issuerId(genUuid()) +func genIssuerId() issuerID { + return issuerID(genUuid()) } -func genKeyId() keyId { - return keyId(genUuid()) +func genKeyId() keyID { + return keyID(genUuid()) } func genUuid() string { diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index 3e1e531c16f48..e655ce3b70fa5 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -92,11 +92,11 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { // Make sure we setup the default values keysConfig, err := getKeysConfig(ctx, s) require.NoError(t, err) - require.Equal(t, &keyConfig{DefaultKeyId: keyId}, keysConfig) + require.Equal(t, &keyConfigEntry{DefaultKeyId: keyId}, keysConfig) issuersConfig, err := getIssuersConfig(ctx, s) require.NoError(t, err) - require.Equal(t, &issuerConfig{DefaultIssuerId: issuerId}, issuersConfig) + require.Equal(t, &issuerConfigEntry{DefaultIssuerId: issuerId}, issuersConfig) // Make sure if we attempt to re-run the migration nothing happens... err = migrateStorage(ctx, request, b.Logger()) diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index eef5ec5022070..4d516ebd22f9f 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -20,17 +20,17 @@ func Test_ConfigsRoundTrip(t *testing.T) { // Verify we handle nothing stored properly keyConfigEmpty, err := getKeysConfig(ctx, s) require.NoError(t, err) - require.Equal(t, &keyConfig{}, keyConfigEmpty) + require.Equal(t, &keyConfigEntry{}, keyConfigEmpty) issuerConfigEmpty, err := getIssuersConfig(ctx, s) require.NoError(t, err) - require.Equal(t, &issuerConfig{}, issuerConfigEmpty) + require.Equal(t, &issuerConfigEntry{}, issuerConfigEmpty) // Now attempt to store and reload properly - origKeyConfig := &keyConfig{ + origKeyConfig := &keyConfigEntry{ DefaultKeyId: genKeyId(), } - origIssuerConfig := &issuerConfig{ + origIssuerConfig := &issuerConfigEntry{ DefaultIssuerId: genIssuerId(), } @@ -84,12 +84,12 @@ func Test_IssuerRoundTrip(t *testing.T) { keys, err := listKeys(ctx, s) require.NoError(t, err) - require.ElementsMatch(t, []keyId{key1.ID, key2.ID}, keys) + require.ElementsMatch(t, []keyID{key1.ID, key2.ID}, keys) issuers, err := listIssuers(ctx, s) require.NoError(t, err) - require.ElementsMatch(t, []issuerId{issuer1.ID, issuer2.ID}, issuers) + require.ElementsMatch(t, []issuerID{issuer1.ID, issuer2.ID}, issuers) } func Test_KeysIssuerImport(t *testing.T) { @@ -158,12 +158,12 @@ func Test_KeysIssuerImport(t *testing.T) { require.Equal(t, "", key2_ref.Name) } -func genIssuerAndKey(t *testing.T, b *backend) (issuer, key) { +func genIssuerAndKey(t *testing.T, b *backend) (issuerEntry, keyEntry) { certBundle := genCertBundle(t, b) keyId := genKeyId() - pkiKey := key{ + pkiKey := keyEntry{ ID: keyId, PrivateKeyType: certBundle.PrivateKeyType, PrivateKey: certBundle.PrivateKey, @@ -171,7 +171,7 @@ func genIssuerAndKey(t *testing.T, b *backend) (issuer, key) { issuerId := genIssuerId() - pkiIssuer := issuer{ + pkiIssuer := issuerEntry{ ID: issuerId, KeyID: keyId, Certificate: strings.TrimSpace(certBundle.Certificate) + "\n", diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 2d0add8b0c5fc..0a922af799ac1 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -19,9 +19,11 @@ const ( defaultRef = "default" ) -var nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex(issuerRefParam) + "$") -var errIssuerNameInUse = errutil.UserError{Err: "issuer name already in use"} -var errKeyNameInUse = errutil.UserError{Err: "key name already in use"} +var ( + nameMatcher = regexp.MustCompile("^" + framework.GenericNameRegex(issuerRefParam) + "$") + errIssuerNameInUse = errutil.UserError{Err: "issuer name already in use"} + errKeyNameInUse = errutil.UserError{Err: "key name already in use"} +) func normalizeSerial(serial string) string { return strings.Replace(strings.ToLower(serial), ":", "-", -1) @@ -142,12 +144,12 @@ func getIssuerName(ctx context.Context, s logical.Storage, data *framework.Field if !nameMatcher.MatchString(issuerName) { return issuerName, errutil.UserError{Err: "issuer name contained invalid characters"} } - issuer_id, err := resolveIssuerReference(ctx, s, issuerName) + issuerId, err := resolveIssuerReference(ctx, s, issuerName) if err == nil { return issuerName, errIssuerNameInUse } - if err != nil && issuer_id != IssuerRefNotFound { + if err != nil && issuerId != IssuerRefNotFound { return issuerName, errutil.InternalError{Err: err.Error()} } } From c4d384fd725364aca5dda31d6fa35fe384453e3f Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 21 Apr 2022 12:18:37 -0400 Subject: [PATCH 45/76] Update CRL handling for multiple issuers When building CRLs, we've gotta make sure certs issued by that issuer land up on that issuer's CRL and not some other CRL. If no CRL is found (matching a cert), we'll place it on the default CRL. However, in the event of equivalent issuers (those with the same subject AND the same key material) -- perhaps due to reissuance -- we'll only create a single (unified) CRL for them. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 1 + builtin/logical/pki/crl_util.go | 280 ++++++++++++++++++--- builtin/logical/pki/path_config_crl.go | 2 +- builtin/logical/pki/path_manage_issuers.go | 2 +- builtin/logical/pki/path_revoke.go | 2 +- builtin/logical/pki/path_root.go | 2 +- builtin/logical/pki/path_tidy.go | 2 +- builtin/logical/pki/storage.go | 52 +++- 8 files changed, 300 insertions(+), 43 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index da60099b358a7..7142d927a2769 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -76,6 +76,7 @@ func Backend(conf *logical.BackendConfig) *backend { LocalStorage: []string{ "revoked/", "crl", + "crls/", "certs/", }, diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index 2b656f58fc63d..aa74f7ffea3f4 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -1,6 +1,7 @@ package pki import ( + "bytes" "context" "crypto/rand" "crypto/x509" @@ -15,10 +16,13 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) +const revokedPath = "revoked/" + type revocationInfo struct { CertificateBytes []byte `json:"certificate_bytes"` RevocationTime int64 `json:"revocation_time"` RevocationTimeUTC time.Time `json:"revocation_time_utc"` + CertificateIssuer issuerID `json:"issuer_id"` } // Revokes a cert, and tries to be smart about error recovery @@ -53,7 +57,7 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st alreadyRevoked := false var revInfo revocationInfo - revEntry, err := fetchCertBySerial(ctx, req, "revoked/", serial) + revEntry, err := fetchCertBySerial(ctx, req, revokedPath, serial) if err != nil { switch err.(type) { case errutil.UserError: @@ -117,7 +121,7 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st revInfo.RevocationTime = currTime.Unix() revInfo.RevocationTimeUTC = currTime.UTC() - revEntry, err = logical.StorageEntryJSON("revoked/"+normalizeSerial(serial), revInfo) + revEntry, err = logical.StorageEntryJSON(revokedPath+normalizeSerial(serial), revInfo) if err != nil { return nil, fmt.Errorf("error creating revocation entry") } @@ -128,7 +132,7 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st } } - crlErr := buildCRL(ctx, b, req, false) + crlErr := buildCRLs(ctx, b, req, false) if crlErr != nil { switch crlErr.(type) { case errutil.UserError: @@ -149,64 +153,184 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st return resp, nil } -// Builds a CRL by going through the list of revoked certificates and building -// a new CRL with the stored revocation times and serial numbers. -func buildCRL(ctx context.Context, b *backend, req *logical.Request, forceNew bool) error { - crlInfo, err := b.CRL(ctx, req.Storage) +func buildCRLs(ctx context.Context, b *backend, req *logical.Request, forceNew bool) error { + // In order to build all CRLs, we need knowledge of all issuers. Any two + // issuers with the same keys _and_ subject should have the same CRL since + // they're functionally equivalent. + // + // When building CRLs, there's two types of CRLs: an "internal" CRL for + // just certificates issued by this issuer, and a "default" CRL, which + // not only contains certificates by this issuer, but also ones issued + // by "unknown" or past issuers. This means we need knowledge of not + // only all issuers (to tell whether or not to include these orphaned + // certs) but whether the present issuer is the configured default. + // + // If a configured default is lacking, we won't provision these + // certificates on any CRL. + // + // In order to know which CRL a given cert belongs on, we have to read + // it into memory, identify the corresponding issuer, and update its + // map with the revoked cert instance. If no such issuer is found, we'll + // place it in the default issuer's CRL. + // + // By not updating storage, we allow issuers to come and go (either by + // direct deletion or by having their keys delete, preventing CRLs from + // being signed) -- and when they return, we'll correctly place certs + // on their CRLs. + issuers, err := listIssuers(ctx, req.Storage) if err != nil { - return errutil.InternalError{Err: fmt.Sprintf("error fetching CRL config information: %s", err)} + return fmt.Errorf("error building CRL: while listing issuers: %v", err) } - crlLifetime := b.crlLifetime - var revokedCerts []pkix.RevokedCertificate - var revInfo revocationInfo - var revokedSerials []string + config, err := getIssuersConfig(ctx, req.Storage) + if err != nil { + return fmt.Errorf("error building CRLs: while getting the default config: %v", err) + } - if crlInfo != nil { - if crlInfo.Expiry != "" { - crlDur, err := time.ParseDuration(crlInfo.Expiry) - if err != nil { - return errutil.InternalError{Err: fmt.Sprintf("error parsing CRL duration of %s", crlInfo.Expiry)} - } - crlLifetime = crlDur + // We map issuerID->entry for fast lookup and also issuerID->Cert for + // signature verification and correlation of revoked certs. + issuerIDEntryMap := make(map[issuerID]*issuerEntry, len(issuers)) + issuerIDCertMap := make(map[issuerID]*x509.Certificate, len(issuers)) + + // We use a double map (keyID->subject->issuerID) to store whether or not this + // key+subject paring has been seen before. We can then iterate over each + // key/subject and choose any representative issuer for that combination. + keySubjectIssuersMap := make(map[keyID]map[string][]issuerID) + for _, issuer := range issuers { + thisEntry, err := fetchIssuerById(ctx, req.Storage, issuer) + if err != nil { + return fmt.Errorf("error building CRLs: unable to fetch specified issuer (%v): %v", issuer, err) } - if crlInfo.Disable { - if !forceNew { - return nil + if len(thisEntry.KeyID) == 0 { + continue + } + + issuerIDEntryMap[issuer] = thisEntry + + thisCert, err := thisEntry.GetCertificate() + if err != nil { + return fmt.Errorf("error building CRLs: unable to parse issuer (%v)'s certificate: %v", issuer, err) + } + issuerIDCertMap[issuer] = thisCert + + subject := string(thisCert.RawIssuer) + if _, ok := keySubjectIssuersMap[thisEntry.KeyID]; !ok { + keySubjectIssuersMap[thisEntry.KeyID] = make(map[string][]issuerID) + } + + keySubjectIssuersMap[thisEntry.KeyID][subject] = append(keySubjectIssuersMap[thisEntry.KeyID][subject], issuer) + } + + // Fetch the cluster-local CRL mapping so we know where to write the + // CRLs. + crlConfig, err := getLocalCRLConfig(ctx, req.Storage) + if err != nil { + return fmt.Errorf("error building CRLs: unable to fetch cluster-local CRL configuration: %v", err) + } + + // Next, we load and parse all revoked certificates. We need to assign + // these certificates to an issuer. Some certificates will not be + // assignable (if they were issued by a since-deleted issuer), so we need + // a separate pool for those. + unassignedCerts, revokedCertsMap, err := getRevokedCertEntries(ctx, req, issuerIDCertMap) + if err != nil { + return fmt.Errorf("error building CRLs: unable to get revoked certificate entries: %v", err) + } + + // Now we can call buildCRL once, on an arbitrary/representative issuer + // from each of these (keyID, subject) sets. + for _, subjectIssuersMap := range keySubjectIssuersMap { + for _, issuersSet := range subjectIssuersMap { + if len(issuersSet) == 0 { + continue + } + + var revokedCerts []pkix.RevokedCertificate + representative := issuersSet[0] + var crlIdentifier crlID + var crlIdIssuer issuerID + for _, issuerId := range issuersSet { + if issuerId == config.DefaultIssuerId { + if len(unassignedCerts) > 0 { + revokedCerts = append(revokedCerts, unassignedCerts...) + } + + representative = issuerId + } + + if thisRevoked, ok := revokedCertsMap[issuerId]; ok && len(thisRevoked) > 0 { + revokedCerts = append(revokedCerts, thisRevoked...) + } + + if thisCRLId, ok := crlConfig.IssuerIDCRLMap[issuerId]; ok && len(thisCRLId) > 0 { + if len(crlIdentifier) > 0 && crlIdentifier != thisCRLId { + return fmt.Errorf("error building CRLs: two issuers with same keys/subjects (%v vs %v) have different internal CRL IDs: %v vs %v", issuerId, crlIdIssuer, thisCRLId, crlIdentifier) + } + + crlIdentifier = thisCRLId + crlIdIssuer = issuerId + } + } + + if len(crlIdentifier) == 0 { + // Create a new random UUID for this CRL if none exists. + crlIdentifier = genCRLId() + } + + // Update all issuers in this group to set the CRL Issuer + for _, issuerId := range issuersSet { + crlConfig.IssuerIDCRLMap[issuerId] = crlIdentifier + } + + if err := buildCRL(ctx, b, req, forceNew, representative, revokedCerts, crlIdentifier); err != nil { + return fmt.Errorf("error building CRLs: unable to build CRL for issuer (%v): %v", representative, err) } - goto WRITE } } - revokedSerials, err = req.Storage.List(ctx, "revoked/") + // Finally, persist our potentially updated local CRL config + if err := setLocalCRLConfig(ctx, req.Storage, crlConfig); err != nil { + return fmt.Errorf("error building CRLs: unable to persist updated cluster-local CRL config: %v", err) + } + + // All good :-) + return nil +} + +func getRevokedCertEntries(ctx context.Context, req *logical.Request, issuerIDCertMap map[issuerID]*x509.Certificate) ([]pkix.RevokedCertificate, map[issuerID][]pkix.RevokedCertificate, error) { + var unassignedCerts []pkix.RevokedCertificate + revokedCertsMap := make(map[issuerID][]pkix.RevokedCertificate) + + revokedSerials, err := req.Storage.List(ctx, revokedPath) if err != nil { - return errutil.InternalError{Err: fmt.Sprintf("error fetching list of revoked certs: %s", err)} + return nil, nil, errutil.InternalError{Err: fmt.Sprintf("error fetching list of revoked certs: %s", err)} } for _, serial := range revokedSerials { - revokedEntry, err := req.Storage.Get(ctx, "revoked/"+serial) + var revInfo revocationInfo + revokedEntry, err := req.Storage.Get(ctx, revokedPath+serial) if err != nil { - return errutil.InternalError{Err: fmt.Sprintf("unable to fetch revoked cert with serial %s: %s", serial, err)} + return nil, nil, errutil.InternalError{Err: fmt.Sprintf("unable to fetch revoked cert with serial %s: %s", serial, err)} } if revokedEntry == nil { - return errutil.InternalError{Err: fmt.Sprintf("revoked certificate entry for serial %s is nil", serial)} + return nil, nil, errutil.InternalError{Err: fmt.Sprintf("revoked certificate entry for serial %s is nil", serial)} } if revokedEntry.Value == nil || len(revokedEntry.Value) == 0 { // TODO: In this case, remove it and continue? How likely is this to // happen? Alternately, could skip it entirely, or could implement a // delete function so that there is a way to remove these - return errutil.InternalError{Err: fmt.Sprintf("found revoked serial but actual certificate is empty")} + return nil, nil, errutil.InternalError{Err: fmt.Sprintf("found revoked serial but actual certificate is empty")} } err = revokedEntry.DecodeJSON(&revInfo) if err != nil { - return errutil.InternalError{Err: fmt.Sprintf("error decoding revocation entry for serial %s: %s", serial, err)} + return nil, nil, errutil.InternalError{Err: fmt.Sprintf("error decoding revocation entry for serial %s: %s", serial, err)} } revokedCert, err := x509.ParseCertificate(revInfo.CertificateBytes) if err != nil { - return errutil.InternalError{Err: fmt.Sprintf("unable to parse stored revoked certificate with serial %s: %s", serial, err)} + return nil, nil, errutil.InternalError{Err: fmt.Sprintf("unable to parse stored revoked certificate with serial %s: %s", serial, err)} } // NOTE: We have to change this to UTC time because the CRL standard @@ -219,11 +343,99 @@ func buildCRL(ctx context.Context, b *backend, req *logical.Request, forceNew bo } else { newRevCert.RevocationTime = time.Unix(revInfo.RevocationTime, 0).UTC() } - revokedCerts = append(revokedCerts, newRevCert) + + // If we have a CertificateIssuer field on the revocation entry, + // prefer it to manually checking each issuer signature, assuming it + // appears valid. Its highly unlikely for two different issuers + // to have the same id (after the first was deleted). + if len(revInfo.CertificateIssuer) > 0 { + issuerId := revInfo.CertificateIssuer + if _, issuerExists := issuerIDCertMap[issuerId]; issuerExists { + revokedCertsMap[issuerId] = append(revokedCertsMap[issuerId], newRevCert) + continue + } + + // Otherwise, fall through and update the entry. + } + + // Now we need to assign the revoked certificate to an issuer. + foundParent := false + for issuerId, issuerCert := range issuerIDCertMap { + if bytes.Equal(revokedCert.RawIssuer, issuerCert.RawSubject) { + if err := revokedCert.CheckSignatureFrom(issuerCert); err == nil { + // Valid mapping. Add it to the specified entry. + revokedCertsMap[issuerId] = append(revokedCertsMap[issuerId], newRevCert) + revInfo.CertificateIssuer = issuerId + foundParent = true + break + } + } + } + + if !foundParent { + // If the parent isn't found, add it to the unassigned bucket. + unassignedCerts = append(unassignedCerts, newRevCert) + } else { + // When the CertificateIssuer field wasn't found on the existing + // entry (or was invalid), and we've found a new value for it, + // we should update the entry to make future CRL builds faster. + revokedEntry, err = logical.StorageEntryJSON(revokedPath+serial, revInfo) + if err != nil { + return nil, nil, fmt.Errorf("error creating revocation entry for existing cert: %v", serial) + } + + err = req.Storage.Put(ctx, revokedEntry) + if err != nil { + return nil, nil, fmt.Errorf("error updating revoked certificate at existing location: %v", serial) + } + } + } + + return unassignedCerts, revokedCertsMap, nil +} + +// Builds a CRL by going through the list of revoked certificates and building +// a new CRL with the stored revocation times and serial numbers. +func buildCRL(ctx context.Context, b *backend, req *logical.Request, forceNew bool, thisIssuerId issuerID, revoked []pkix.RevokedCertificate, identifier crlID) error { + crlInfo, err := b.CRL(ctx, req.Storage) + if err != nil { + return errutil.InternalError{Err: fmt.Sprintf("error fetching CRL config information: %s", err)} + } + + crlLifetime := b.crlLifetime + var revokedCerts []pkix.RevokedCertificate + + if crlInfo != nil { + if crlInfo.Expiry != "" { + crlDur, err := time.ParseDuration(crlInfo.Expiry) + if err != nil { + return errutil.InternalError{Err: fmt.Sprintf("error parsing CRL duration of %s", crlInfo.Expiry)} + } + crlLifetime = crlDur + } + + if crlInfo.Disable { + if !forceNew { + return nil + } + goto WRITE + } } + revokedCerts = revoked + WRITE: - signingBundle, caErr := fetchCAInfo(ctx, b, req, defaultRef) + bundle, caErr := fetchCertBundleByIssuerId(ctx, req.Storage, thisIssuerId, true /* need the signing key */) + if caErr != nil { + switch caErr.(type) { + case errutil.UserError: + return errutil.UserError{Err: fmt.Sprintf("could not fetch the CA certificate: %s", caErr)} + default: + return errutil.InternalError{Err: fmt.Sprintf("error fetching CA certificate: %s", caErr)} + } + } + + signingBundle, caErr := parseCABundle(ctx, b, req, bundle) if caErr != nil { switch caErr.(type) { case errutil.UserError: @@ -239,7 +451,7 @@ WRITE: } err = req.Storage.Put(ctx, &logical.StorageEntry{ - Key: "crl", + Key: "crls/" + identifier.String(), Value: crlBytes, }) if err != nil { diff --git a/builtin/logical/pki/path_config_crl.go b/builtin/logical/pki/path_config_crl.go index f9e5ceb423fc7..e10c8d686b6b1 100644 --- a/builtin/logical/pki/path_config_crl.go +++ b/builtin/logical/pki/path_config_crl.go @@ -111,7 +111,7 @@ func (b *backend) pathCRLWrite(ctx context.Context, req *logical.Request, d *fra if oldDisable != config.Disable { // It wasn't disabled but now it is, rotate - crlErr := buildCRL(ctx, b, req, true) + crlErr := buildCRLs(ctx, b, req, true) if crlErr != nil { switch crlErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index d251a014c9c35..fc43452f280be 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -184,7 +184,7 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d } if len(createdIssuers) > 0 { - err := buildCRL(ctx, b, req, true) + err := buildCRLs(ctx, b, req, true) if err != nil { return nil, err } diff --git a/builtin/logical/pki/path_revoke.go b/builtin/logical/pki/path_revoke.go index f5032bb528075..ee19a2e604306 100644 --- a/builtin/logical/pki/path_revoke.go +++ b/builtin/logical/pki/path_revoke.go @@ -68,7 +68,7 @@ func (b *backend) pathRotateCRLRead(ctx context.Context, req *logical.Request, d b.revokeStorageLock.RLock() defer b.revokeStorageLock.RUnlock() - crlErr := buildCRL(ctx, b, req, false) + crlErr := buildCRLs(ctx, b, req, false) if crlErr != nil { switch crlErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index ae1a6369820f7..2b248940b3f30 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -191,7 +191,7 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, } // Build a fresh CRL - err = buildCRL(ctx, b, req, true) + err = buildCRLs(ctx, b, req, true) if err != nil { return nil, err } diff --git a/builtin/logical/pki/path_tidy.go b/builtin/logical/pki/path_tidy.go index 915d5b3503884..458ec30313789 100644 --- a/builtin/logical/pki/path_tidy.go +++ b/builtin/logical/pki/path_tidy.go @@ -225,7 +225,7 @@ func (b *backend) pathTidyWrite(ctx context.Context, req *logical.Request, d *fr } if rebuildCRL { - if err := buildCRL(ctx, b, req, false); err != nil { + if err := buildCRLs(ctx, b, req, false); err != nil { return err } } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 0c1390d519b71..d7f5b9bc45749 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -15,10 +15,11 @@ import ( ) const ( - storageKeyConfig = "config/keys" - storageIssuerConfig = "config/issuers" - keyPrefix = "config/key/" - issuerPrefix = "config/issuer/" + storageKeyConfig = "config/keys" + storageIssuerConfig = "config/issuers" + keyPrefix = "config/key/" + issuerPrefix = "config/issuer/" + storageLocalCRLConfig = "crls/config" legacyMigrationBundleLogKey = "config/legacyMigrationBundleLog" legacyCertBundlePath = "config/ca_bundle" @@ -36,6 +37,12 @@ func (p issuerID) String() string { return string(p) } +type crlID string + +func (p crlID) String() string { + return string(p) +} + const ( IssuerRefNotFound = issuerID("not-found") KeyRefNotFound = keyID("not-found") @@ -58,6 +65,10 @@ type issuerEntry struct { SerialNumber string `json:"serial_number" structs:"serial_number" mapstructure:"serial_number"` } +type localCRLConfigEntry struct { + IssuerIDCRLMap map[issuerID]crlID `json:"issuer_id_crl_map" structs:"issuer_id_crl_map" mapstructure:"issuer_id_crl_map"` +} + type keyConfigEntry struct { DefaultKeyId keyID `json:"default" structs:"default" mapstructure:"default"` } @@ -502,6 +513,35 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu return &result, false, nil } +func setLocalCRLConfig(ctx context.Context, s logical.Storage, mapping *localCRLConfigEntry) error { + json, err := logical.StorageEntryJSON(storageLocalCRLConfig, mapping) + if err != nil { + return err + } + + return s.Put(ctx, json) +} + +func getLocalCRLConfig(ctx context.Context, s logical.Storage) (*localCRLConfigEntry, error) { + entry, err := s.Get(ctx, storageLocalCRLConfig) + if err != nil { + return nil, err + } + + mapping := &localCRLConfigEntry{} + if entry != nil { + if err := entry.DecodeJSON(mapping); err != nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("unable to decode cluster-local CRL configuration: %v", err)} + } + } + + if len(mapping.IssuerIDCRLMap) == 0 { + mapping.IssuerIDCRLMap = make(map[issuerID]crlID) + } + + return mapping, nil +} + func setKeysConfig(ctx context.Context, s logical.Storage, config *keyConfigEntry) error { json, err := logical.StorageEntryJSON(storageKeyConfig, config) if err != nil { @@ -649,6 +689,10 @@ func genKeyId() keyID { return keyID(genUuid()) } +func genCRLId() crlID { + return crlID(genUuid()) +} + func genUuid() string { aUuid, err := uuid.GenerateUUID() if err != nil { From a63f803aacbc4d4c54e7a9386ed4cd00b26bc4e1 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 21 Apr 2022 12:51:16 -0400 Subject: [PATCH 46/76] Allow fetching updated CRL locations This updates fetchCertBySerial to support querying the default issuer's CRL. Signed-off-by: Alexander Scheel --- builtin/logical/pki/cert_util.go | 5 ++++- builtin/logical/pki/storage.go | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index c49136c8de194..8b4ba6b3bbf2e 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -154,7 +154,10 @@ func fetchCertBySerial(ctx context.Context, req *logical.Request, prefix, serial legacyPath = "revoked/" + colonSerial path = "revoked/" + hyphenSerial case serial == "crl": - path = "crl" + path, err = resolveIssuerCRLPath(ctx, req.Storage, defaultRef) + if err != nil { + return nil, err + } default: legacyPath = "certs/" + colonSerial path = "certs/" + hyphenSerial diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index d7f5b9bc45749..fba158efe792c 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -634,6 +634,24 @@ func resolveIssuerReference(ctx context.Context, s logical.Storage, reference st return IssuerRefNotFound, errutil.UserError{Err: fmt.Sprintf("unable to find PKI issuer for reference: %v", reference)} } +func resolveIssuerCRLPath(ctx context.Context, s logical.Storage, reference string) (string, error) { + issuer, err := resolveIssuerReference(ctx, s, reference) + if err != nil { + return "crl", err + } + + crlConfig, err := getLocalCRLConfig(ctx, s) + if err != nil { + return "crl", err + } + + if crlId, ok := crlConfig.IssuerIDCRLMap[issuer]; ok && len(crlId) > 0 { + return fmt.Sprintf("crls/%v", crlId), nil + } + + return "crl", fmt.Errorf("unable to find CRL for issuer: id:%v/ref:%v", issuer, reference) +} + // Builds a certutil.CertBundle from the specified issuer identifier, // optionally loading the key or not. func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuerID, loadKey bool) (*certutil.CertBundle, error) { From 05c44f8b082e1738f6b04ce784519ae19edc1408 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 21 Apr 2022 13:00:31 -0400 Subject: [PATCH 47/76] Remove legacy CRL storage location test case Signed-off-by: Alexander Scheel --- builtin/logical/pki/cert_util_test.go | 30 --------------------------- 1 file changed, 30 deletions(-) diff --git a/builtin/logical/pki/cert_util_test.go b/builtin/logical/pki/cert_util_test.go index b3507613302e8..d90963c079cfb 100644 --- a/builtin/logical/pki/cert_util_test.go +++ b/builtin/logical/pki/cert_util_test.go @@ -86,36 +86,6 @@ func TestPki_FetchCertBySerial(t *testing.T) { t.Fatalf("error on %s for hyphen-based storage path: err: %v, entry: %v", name, err, certEntry) } } - - noConvCases := map[string]struct { - Req *logical.Request - Prefix string - Serial string - }{ - "crl": { - &logical.Request{ - Storage: storage, - }, - "", - "crl", - }, - } - - // Test for ca and crl case - for name, tc := range noConvCases { - err := storage.Put(context.Background(), &logical.StorageEntry{ - Key: tc.Serial, - Value: []byte("some data"), - }) - if err != nil { - t.Fatalf("error writing to storage on %s: %s", name, err) - } - - certEntry, err := fetchCertBySerial(context.Background(), tc.Req, tc.Prefix, tc.Serial) - if err != nil || certEntry == nil { - t.Fatalf("error on %s: err: %v, entry: %v", name, err, certEntry) - } - } } // Demonstrate that multiple OUs in the name are handled in an From 1182a7107244a41ea1b011a2b17f3cfe174f4344 Mon Sep 17 00:00:00 2001 From: Kit Haines Date: Thu, 21 Apr 2022 13:20:43 -0400 Subject: [PATCH 48/76] Update to CRLv2 Format to copy RawIssuer When using the older Certificate.CreateCRL(...) call, Go's x509 library copies the parsed pkix.Name version of the CRL Issuer's Subject field. For certain constructed CAs, this fails since pkix.Name is not suitable for round-tripping. This also builds a CRLv1 (per RFC 5280) CRL. In updating to the newer x509.CreateRevocationList(...) call, we can construct the CRL in the CRLv2 format and correctly copy the issuer's name. However, this requires holding an additional field per-CRL, the CRLNumber field, which is required in Go's implementation of CRLv2 (though OPTIONAL in the spec). We store this on the new LocalCRLConfigEntry object, per-CRL. Co-authored-by: Alexander Scheel Signed-off-by: Alexander Scheel --- builtin/logical/pki/crl_util.go | 21 ++++++++++++++++++--- builtin/logical/pki/storage.go | 5 +++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index aa74f7ffea3f4..bde21c76838b2 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -8,6 +8,7 @@ import ( "crypto/x509/pkix" "errors" "fmt" + "math/big" "strings" "time" @@ -276,6 +277,7 @@ func buildCRLs(ctx context.Context, b *backend, req *logical.Request, forceNew b if len(crlIdentifier) == 0 { // Create a new random UUID for this CRL if none exists. crlIdentifier = genCRLId() + crlConfig.CRLNumberMap[crlIdentifier] = 1 } // Update all issuers in this group to set the CRL Issuer @@ -283,7 +285,13 @@ func buildCRLs(ctx context.Context, b *backend, req *logical.Request, forceNew b crlConfig.IssuerIDCRLMap[issuerId] = crlIdentifier } - if err := buildCRL(ctx, b, req, forceNew, representative, revokedCerts, crlIdentifier); err != nil { + // We always update the CRL Number since we never want to + // duplicate numbers and missing numbers is fine. + crlNumber := crlConfig.CRLNumberMap[crlIdentifier] + crlConfig.CRLNumberMap[crlIdentifier] += 1 + + // Lastly, build the CRL. + if err := buildCRL(ctx, b, req, forceNew, representative, revokedCerts, crlIdentifier, crlNumber); err != nil { return fmt.Errorf("error building CRLs: unable to build CRL for issuer (%v): %v", representative, err) } } @@ -396,7 +404,7 @@ func getRevokedCertEntries(ctx context.Context, req *logical.Request, issuerIDCe // Builds a CRL by going through the list of revoked certificates and building // a new CRL with the stored revocation times and serial numbers. -func buildCRL(ctx context.Context, b *backend, req *logical.Request, forceNew bool, thisIssuerId issuerID, revoked []pkix.RevokedCertificate, identifier crlID) error { +func buildCRL(ctx context.Context, b *backend, req *logical.Request, forceNew bool, thisIssuerId issuerID, revoked []pkix.RevokedCertificate, identifier crlID, crlNumber int64) error { crlInfo, err := b.CRL(ctx, req.Storage) if err != nil { return errutil.InternalError{Err: fmt.Sprintf("error fetching CRL config information: %s", err)} @@ -445,7 +453,14 @@ WRITE: } } - crlBytes, err := signingBundle.Certificate.CreateCRL(rand.Reader, signingBundle.PrivateKey, revokedCerts, time.Now(), time.Now().Add(crlLifetime)) + revocationListTemplate := &x509.RevocationList{ + RevokedCertificates: revokedCerts, + Number: big.NewInt(crlNumber), + ThisUpdate: time.Now(), + NextUpdate: time.Now().Add(crlLifetime), + } + + crlBytes, err := x509.CreateRevocationList(rand.Reader, revocationListTemplate, signingBundle.Certificate, signingBundle.PrivateKey) if err != nil { return errutil.InternalError{Err: fmt.Sprintf("error creating new CRL: %s", err)} } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index fba158efe792c..94a0f1239d843 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -67,6 +67,7 @@ type issuerEntry struct { type localCRLConfigEntry struct { IssuerIDCRLMap map[issuerID]crlID `json:"issuer_id_crl_map" structs:"issuer_id_crl_map" mapstructure:"issuer_id_crl_map"` + CRLNumberMap map[crlID]int64 `json:"crl_number_map" structs:"crl_number_map" mapstructure:"crl_number_map"` } type keyConfigEntry struct { @@ -539,6 +540,10 @@ func getLocalCRLConfig(ctx context.Context, s logical.Storage) (*localCRLConfigE mapping.IssuerIDCRLMap = make(map[issuerID]crlID) } + if len(mapping.CRLNumberMap) == 0 { + mapping.CRLNumberMap = make(map[crlID]int64) + } + return mapping, nil } From 48ec19e763b963a22a7975fd91c5d1187ae288ad Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 21 Apr 2022 13:58:05 -0400 Subject: [PATCH 49/76] Add comment regarding CRL non-assignment in GOTO In previous versions of Vault, it was possible to sign an empty CRL (when the CRL was disabled and a force-rebuild was requested). Add a comment about this case. Signed-off-by: Alexander Scheel --- builtin/logical/pki/crl_util.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index bde21c76838b2..e2755b6801536 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -426,6 +426,13 @@ func buildCRL(ctx context.Context, b *backend, req *logical.Request, forceNew bo if !forceNew { return nil } + + // NOTE: in this case, the passed argument (revoked) is not added + // to the revokedCerts list. This is because we want to sign an + // **empty** CRL (as the CRL was disabled but we've specified the + // forceNew option). In previous versions of Vault (1.10 series and + // earlier), we'd have queried the certs below, whereas we now have + // an assignment from a pre-queried list. goto WRITE } } From 9189e7f6254371a91f39df2964c2ebfbedbf45b4 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Fri, 22 Apr 2022 08:55:56 -0400 Subject: [PATCH 50/76] Allow fetching the specified issuer's CRL We add a new API endpoint to fetch the specified issuer's CRL directly (rather than the default issuer's CRL at /crl and /certs/crl). We also add a new test to validate the CRL in a multi-root scenario and ensure it is signed with the correct keys. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 1 + builtin/logical/pki/backend_test.go | 56 ++++++++++++++ builtin/logical/pki/path_fetch_issuers.go | 94 +++++++++++++++++++++++ 3 files changed, 151 insertions(+) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 7142d927a2769..33d986f7670ae 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -113,6 +113,7 @@ func Backend(conf *logical.BackendConfig) *backend { // Issuer APIs pathListIssuers(&b), pathGetIssuer(&b), + pathGetIssuerCRL(&b), pathImportIssuer(&b), pathIssuerSignIntermediate(&b), pathIssuerSignSelfIssued(&b), diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index 9498835f6b655..645f706ab45f0 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -3978,6 +3978,46 @@ func TestBackend_RevokePlusTidy_Intermediate(t *testing.T) { func getParsedCrl(t *testing.T, client *api.Client, mountPoint string) *pkix.CertificateList { path := fmt.Sprintf("/v1/%s/crl", mountPoint) + return getParsedCrlAtPath(t, client, path) +} + +func getParsedCrlForIssuer(t *testing.T, client *api.Client, mountPoint string, issuer string) *pkix.CertificateList { + path := fmt.Sprintf("/v1/%v/issuer/%v/crl", mountPoint, issuer) + crl := getParsedCrlAtPath(t, client, path) + + // Now fetch the issuer as well and verify the certificate + path = fmt.Sprintf("/v1/%v/issuer/%v/der", mountPoint, issuer) + req := client.NewRequest("GET", path) + resp, err := client.RawRequest(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + certBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("err: %s", err) + } + if len(certBytes) == 0 { + t.Fatalf("expected certificate in response body") + } + + cert, err := x509.ParseCertificate(certBytes) + if err != nil { + t.Fatal(err) + } + if cert == nil { + t.Fatalf("expected parsed certificate") + } + + if err := cert.CheckCRLSignature(crl); err != nil { + t.Fatalf("expected valid signature on CRL for issuer %v: %v", issuer, crl) + } + + return crl +} + +func getParsedCrlAtPath(t *testing.T, client *api.Client, path string) *pkix.CertificateList { req := client.NewRequest("GET", path) resp, err := client.RawRequest(req) if err != nil { @@ -4718,6 +4758,10 @@ func TestRootWithExistingKey(t *testing.T) { require.NotEmpty(t, myIssuerId1) require.NotEmpty(t, myKeyId1) + // Fetch the parsed CRL; it should be empty as we've not revoked anything + parsedCrl := getParsedCrlForIssuer(t, client, "pki-root", "my-issuer1") + require.Equal(t, len(parsedCrl.TBSCertList.RevokedCertificates), 0, "should have no revoked certificates") + // Fail if the specified issuer name is re-used. _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ "common_name": "root myvault.com", @@ -4740,6 +4784,10 @@ func TestRootWithExistingKey(t *testing.T) { require.NotEmpty(t, myIssuerId2) require.NotEmpty(t, myKeyId2) + // Fetch the parsed CRL; it should be empty as we've not revoked anything + parsedCrl = getParsedCrlForIssuer(t, client, "pki-root", "my-issuer2") + require.Equal(t, len(parsedCrl.TBSCertList.RevokedCertificates), 0, "should have no revoked certificates") + // Fail if the specified key name is re-used. _, err = client.Logical().WriteWithContext(ctx, "pki-root/issuers/generate/root/internal", map[string]interface{}{ "common_name": "root myvault.com", @@ -4762,6 +4810,14 @@ func TestRootWithExistingKey(t *testing.T) { require.NotEmpty(t, myIssuerId3) require.NotEmpty(t, myKeyId3) + // Fetch the parsed CRL; it should be empty as we've not revoking anything. + parsedCrl = getParsedCrlForIssuer(t, client, "pki-root", "my-issuer3") + require.Equal(t, len(parsedCrl.TBSCertList.RevokedCertificates), 0, "should have no revoked certificates") + // Signatures should be the same since this is just a reissued cert. We + // use signature as a proxy for "these two CRLs are equal". + firstCrl := getParsedCrlForIssuer(t, client, "pki-root", "my-issuer1") + require.Equal(t, parsedCrl.SignatureValue, firstCrl.SignatureValue) + require.NotEqual(t, myIssuerId1, myIssuerId2) require.NotEqual(t, myIssuerId1, myIssuerId3) require.NotEqual(t, myKeyId1, myKeyId2) diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index e6c2045fa3ae9..cff5fb98ae653 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -321,3 +321,97 @@ Writing to /issuer/:ref allows updating of the name field associated with the certificate. ` ) + +func pathGetIssuerCRL(b *backend) *framework.Path { + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "/crl(/pem)?" + return buildPathGetIssuerCRL(b, pattern) +} + +func buildPathGetIssuerCRL(b *backend, pattern string) *framework.Path { + fields := map[string]*framework.FieldSchema{} + fields = addIssuerRefNameFields(fields) + + return &framework.Path{ + // Returns raw values. + Pattern: pattern, + Fields: fields, + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.ReadOperation: b.pathGetIssuerCRL, + }, + + HelpSynopsis: pathGetIssuerCRLHelpSyn, + HelpDescription: pathGetIssuerCRLHelpDesc, + } +} + +func (b *backend) pathGetIssuerCRL(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + issuerName := getIssuerRef(data) + if len(issuerName) == 0 { + return logical.ErrorResponse("missing issuer reference"), nil + } + + crlPath, err := resolveIssuerCRLPath(ctx, req.Storage, issuerName) + if err != nil { + return nil, err + } + + crlEntry, err := req.Storage.Get(ctx, crlPath) + if err != nil { + return nil, err + } + + var certificate []byte + if crlEntry != nil && len(crlEntry.Value) > 0 { + certificate = []byte(crlEntry.Value) + } + + contentType := "application/pkix-crl" + + if strings.HasSuffix(req.Path, "/pem") { + contentType = "application/x-pem-file" + + // Rather return an empty response rather than an empty + // PEM blob. + if len(certificate) > 0 { + pemBlock := pem.Block{ + Type: "X509 CRL", + Bytes: certificate, + } + + certificate = pem.EncodeToMemory(&pemBlock) + } + } + + statusCode := 200 + if len(certificate) == 0 { + statusCode = 204 + } + + return &logical.Response{ + Data: map[string]interface{}{ + logical.HTTPContentType: contentType, + logical.HTTPRawBody: certificate, + logical.HTTPStatusCode: statusCode, + }, + }, nil +} + +const ( + pathGetIssuerCRLHelpSyn = `Fetch an issuer's Certificate Revocation Log (CRL).` + pathGetIssuerCRLHelpDesc = ` +This allows fetching the specified issuer's CRL. Note that this is different +than the legacy path (/crl and /certs/crl) in that this is per-issuer and not +just the default issuer's CRL. + +Two issuers will have the same CRL if they have the same key material and if +they have the same Subject value. + +:ref can be either the literal value "default", in which case /config/issuers +will be consulted for the present default issuer, an identifier of an issuer, +or its assigned name value. + +Use /issuer/:ref/crl/pem to return just the certificate in PEM form; the raw +/issuer/:ref/crl is in DER form. +` +) From 9f10a6772c0f60a3702ab2befb080571ca398b52 Mon Sep 17 00:00:00 2001 From: Steven Clark Date: Fri, 22 Apr 2022 11:06:28 -0400 Subject: [PATCH 51/76] Add new PKI key prefix to seal wrapped storage (#15126) --- builtin/logical/pki/backend.go | 3 ++- builtin/logical/pki/backend_test.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 33d986f7670ae..fd74dddbfb82e 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -86,7 +86,8 @@ func Backend(conf *logical.BackendConfig) *backend { }, SealWrapStorage: []string{ - "config/ca_bundle", + legacyCertBundlePath, + keyPrefix, }, }, diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index 645f706ab45f0..a780101d1dd05 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -4908,6 +4908,17 @@ func TestIntermediateWithExistingKey(t *testing.T) { require.Equal(t, myKeyId1, myKeyId3, "our new ca did not seem to reuse the key as we expected.") } +func TestSealWrappedStorageConfigured(t *testing.T) { + b, _ := createBackendWithStorage(t) + wrappedEntries := b.Backend.PathsSpecial.SealWrapStorage + + // Make sure our legacy bundle is within the list + // NOTE: do not convert these test values to constants, we should always have these paths within seal wrap config + require.Contains(t, wrappedEntries, "config/ca_bundle", "Legacy bundle missing from seal wrap") + // The trailing / is important as it treats the entire folder requiring seal wrapping, not just config/key + require.Contains(t, wrappedEntries, "config/key/", "key prefix with trailing / missing from seal wrap.") +} + var ( initTest sync.Once rsaCAKey string From 2b740ece2c489915d96f452f6a889ee81d5fa160 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Wed, 13 Apr 2022 16:34:20 -0400 Subject: [PATCH 52/76] Refactor common backend initialization within backend_test - Leverage an existing helper method within the PKI backend tests to setup a PKI backend with storage. --- builtin/logical/pki/backend_test.go | 88 +++-------------------------- 1 file changed, 9 insertions(+), 79 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index a780101d1dd05..b199989560e27 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -281,18 +281,7 @@ func TestBackend_InvalidParameter(t *testing.T) { func TestBackend_CSRValues(t *testing.T) { initTest.Do(setCerts) - defaultLeaseTTLVal := time.Hour * 24 - maxLeaseTTLVal := time.Hour * 24 * 32 - b, err := Factory(context.Background(), &logical.BackendConfig{ - Logger: nil, - System: &logical.StaticSystemView{ - DefaultLeaseTTLVal: defaultLeaseTTLVal, - MaxLeaseTTLVal: maxLeaseTTLVal, - }, - }) - if err != nil { - t.Fatalf("Unable to create backend: %s", err) - } + b, _ := createBackendWithStorage(t) testCase := logicaltest.TestCase{ LogicalBackend: b, @@ -308,18 +297,7 @@ func TestBackend_CSRValues(t *testing.T) { func TestBackend_URLsCRUD(t *testing.T) { initTest.Do(setCerts) - defaultLeaseTTLVal := time.Hour * 24 - maxLeaseTTLVal := time.Hour * 24 * 32 - b, err := Factory(context.Background(), &logical.BackendConfig{ - Logger: nil, - System: &logical.StaticSystemView{ - DefaultLeaseTTLVal: defaultLeaseTTLVal, - MaxLeaseTTLVal: maxLeaseTTLVal, - }, - }) - if err != nil { - t.Fatalf("Unable to create backend: %s", err) - } + b, _ := createBackendWithStorage(t) testCase := logicaltest.TestCase{ LogicalBackend: b, @@ -354,18 +332,8 @@ func TestBackend_Roles(t *testing.T) { t.Run(tc.name, func(t *testing.T) { initTest.Do(setCerts) - defaultLeaseTTLVal := time.Hour * 24 - maxLeaseTTLVal := time.Hour * 24 * 32 - b, err := Factory(context.Background(), &logical.BackendConfig{ - Logger: nil, - System: &logical.StaticSystemView{ - DefaultLeaseTTLVal: defaultLeaseTTLVal, - MaxLeaseTTLVal: maxLeaseTTLVal, - }, - }) - if err != nil { - t.Fatalf("Unable to create backend: %s", err) - } + b, _ := createBackendWithStorage(t) + testCase := logicaltest.TestCase{ LogicalBackend: b, Steps: []logicaltest.TestStep{ @@ -1749,13 +1717,7 @@ func generateRoleSteps(t *testing.T, useCSRs bool) []logicaltest.TestStep { } func TestBackend_PathFetchValidRaw(t *testing.T) { - config := logical.TestBackendConfig() - storage := &logical.InmemStorage{} - config.StorageView = storage - - b := Backend(config) - err := b.Setup(context.Background(), config) - require.NoError(t, err) + b, storage := createBackendWithStorage(t) resp, err := b.HandleRequest(context.Background(), &logical.Request{ Operation: logical.UpdateOperation, @@ -1884,15 +1846,7 @@ func TestBackend_PathFetchValidRaw(t *testing.T) { func TestBackend_PathFetchCertList(t *testing.T) { // create the backend - config := logical.TestBackendConfig() - storage := &logical.InmemStorage{} - config.StorageView = storage - - b := Backend(config) - err := b.Setup(context.Background(), config) - if err != nil { - t.Fatal(err) - } + b, storage := createBackendWithStorage(t) // generate root rootData := map[string]interface{}{ @@ -2034,15 +1988,7 @@ func TestBackend_SignVerbatim(t *testing.T) { func runTestSignVerbatim(t *testing.T, keyType string) { // create the backend - config := logical.TestBackendConfig() - storage := &logical.InmemStorage{} - config.StorageView = storage - - b := Backend(config) - err := b.Setup(context.Background(), config) - if err != nil { - t.Fatal(err) - } + b, storage := createBackendWithStorage(t) // generate root rootData := map[string]interface{}{ @@ -2479,15 +2425,7 @@ func TestBackend_SignIntermediate_AllowedPastCA(t *testing.T) { func TestBackend_SignSelfIssued(t *testing.T) { // create the backend - config := logical.TestBackendConfig() - storage := &logical.InmemStorage{} - config.StorageView = storage - - b := Backend(config) - err := b.Setup(context.Background(), config) - if err != nil { - t.Fatal(err) - } + b, storage := createBackendWithStorage(t) // generate root rootData := map[string]interface{}{ @@ -2626,15 +2564,7 @@ func TestBackend_SignSelfIssued(t *testing.T) { // require_matching_certificate_algorithms flag. func TestBackend_SignSelfIssued_DifferentTypes(t *testing.T) { // create the backend - config := logical.TestBackendConfig() - storage := &logical.InmemStorage{} - config.StorageView = storage - - b := Backend(config) - err := b.Setup(context.Background(), config) - if err != nil { - t.Fatal(err) - } + b, storage := createBackendWithStorage(t) // generate root rootData := map[string]interface{}{ From 814588271da9af76c9c507ba7f1d3eed8642704e Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Wed, 13 Apr 2022 16:40:44 -0400 Subject: [PATCH 53/76] Add ability to read legacy cert bundle if the migration has not occurred on secondaries. - Track the migration state forbidding an issuer/key writing api call if we have not migrated - For operations that just need to read the CA bundle, use the same tracking variable to switch between reading the legacy bundle or use the new key/issuer storage. - Add an invalidation function that will listen for updates to our log path to refresh the state on secondary clusters. --- builtin/logical/pki/backend.go | 49 ++++++++++++-- builtin/logical/pki/cert_util.go | 31 ++++++--- builtin/logical/pki/path_intermediate.go | 4 ++ builtin/logical/pki/path_manage_issuers.go | 4 ++ builtin/logical/pki/path_roles_test.go | 2 + builtin/logical/pki/path_root.go | 4 ++ builtin/logical/pki/storage_migrations.go | 79 +++++++++++++++------- builtin/logical/pki/util.go | 2 +- 8 files changed, 136 insertions(+), 39 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index fd74dddbfb82e..c12854775cf83 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/hashicorp/vault/sdk/helper/consts" @@ -139,6 +140,7 @@ func Backend(conf *logical.BackendConfig) *backend { BackendType: logical.TypeLogical, InitializeFunc: b.initialize, + Invalidate: b.invalidate, } b.crlLifetime = time.Hour * 72 @@ -146,6 +148,8 @@ func Backend(conf *logical.BackendConfig) *backend { b.tidyStatus = &tidyStatus{state: tidyStatusInactive} b.storage = conf.StorageView + b.pkiStorageVersion.Store(0) + return &b } @@ -159,6 +163,8 @@ type backend struct { tidyStatusLock sync.RWMutex tidyStatus *tidyStatus + + pkiStorageVersion atomic.Value } type ( @@ -255,19 +261,52 @@ func (b *backend) metricsWrap(callType string, roleMode int, ofunc roleOperation // initialize is used to perform a possible PKI storage migration if needed func (b *backend) initialize(ctx context.Context, req *logical.InitializationRequest) error { - logger := b.Logger().Named("initialize") - // Early exit if not a primary cluster or performance secondary with a local mount. if b.System().ReplicationState().HasState(consts.ReplicationDRSecondary|consts.ReplicationPerformanceStandby) || (!b.System().LocalMount() && b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary)) { - logger.Debug("skipping pki migration as we are not on primary or secondary with a local mount") + b.Logger().Debug("skipping pki migration as we are not on primary or secondary with a local mount") return nil } - if err := migrateStorage(ctx, req, logger); err != nil { - logger.Error("Error during migration of PKI mount: " + err.Error()) + if err := migrateStorage(ctx, req, b.Logger()); err != nil { + b.Logger().Error("Error during migration of PKI mount: " + err.Error()) return err } + b.updatePkiStorageVersion(ctx) + return nil } + +func (b *backend) useLegacyBundleCaStorage() bool { + version := b.pkiStorageVersion.Load() + return version == nil || version == 0 +} + +func (b *backend) updatePkiStorageVersion(ctx context.Context) { + info, err := getMigrationInfo(ctx, b.storage) + if err != nil { + b.Logger().Error(fmt.Sprintf("Failed loading PKI migration status, staying in legacy mode: %v", err)) + return + } + + if info.isRequired { + b.Logger().Info("PKI migration status is required, reading cert bundle from legacy ca location") + b.pkiStorageVersion.Store(0) + } + + if !info.isRequired { + b.Logger().Debug("PKI migration completed, reading cert bundle from key/issuer storage") + b.pkiStorageVersion.Store(1) + } +} + +func (b *backend) invalidate(ctx context.Context, key string) { + switch { + case strings.HasPrefix(key, legacyMigrationBundleLogKey): + // This is for a secondary cluster to pick up that the migration has completed + // and reset its compatibility mode. + b.updatePkiStorageVersion(ctx) + } + // FIXME: We need to hook into CRL generation here for issuer/bundle updates. +} diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index 8b4ba6b3bbf2e..3275754d57685 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -89,18 +89,10 @@ func getFormat(data *framework.FieldData) string { return format } -// Fetches the CA info. Unlike other certificates, the CA info is stored -// in the backend as a CertBundle, because we are storing its private key +// Fetches the CA info. func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRef string) (*certutil.CAInfoBundle, error) { - id, err := resolveIssuerReference(ctx, req.Storage, issuerRef) + bundle, err := fetchCertBundle(ctx, b, req.Storage, issuerRef) if err != nil { - // Usually a bad label from the user or misconfigured default. - return nil, errutil.UserError{Err: err.Error()} - } - - bundle, err := fetchCertBundleByIssuerId(ctx, req.Storage, id, true) - if err != nil { - // Once we have an issuer id, usually a bug on our side if it isn't there. return nil, errutil.InternalError{Err: err.Error()} } @@ -134,6 +126,25 @@ func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRe return caInfo, nil } +// fetchCertBundle is our flex point to load either the legacy ca bundle if migration has yet to be +// performed or load the bundle from the new key/issuer storage. Any function that needs a bundle +// should load it using this method to maintain compatibility on secondary nodes for which their +// primary's have not upgraded yet. +func fetchCertBundle(ctx context.Context, b *backend, s logical.Storage, issuerRef string) (*certutil.CertBundle, error) { + if b.useLegacyBundleCaStorage() { + // We have not completed the migration so attempt to load the bundle from the legacy location + return getLegacyCertBundle(ctx, s) + } + + id, err := resolveIssuerReference(ctx, s, issuerRef) + if err != nil { + // Usually a bad label from the user or misconfigured default. + return nil, err + } + + return fetchCertBundleByIssuerId(ctx, s, id, true) +} + // Allows fetching certificates from the backend; it handles the slightly // separate pathing for CRL, and revoked certificates. // diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index 2ab87f985a100..29337a9b3bac9 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -47,6 +47,10 @@ appended to the bundle.`, func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { var err error + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not create intermediary until migration has completed"), nil + } + exported, format, role, errorResp := b.getGenerationParams(ctx, data, req.MountPoint) if errorResp != nil { return errorResp, nil diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index fc43452f280be..df4d1c2008eea 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -96,6 +96,10 @@ secret-key (optional) and certificates.`, func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { keysAllowed := strings.HasSuffix(req.Path, "bundle") || req.Path == "config/ca" + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not import issuers until migration has completed"), nil + } + var pemBundle string var certificate string rawPemBundle, bundleOk := data.GetOk("pem_bundle") diff --git a/builtin/logical/pki/path_roles_test.go b/builtin/logical/pki/path_roles_test.go index faacd23a02207..0ab52ae7ae92a 100644 --- a/builtin/logical/pki/path_roles_test.go +++ b/builtin/logical/pki/path_roles_test.go @@ -20,6 +20,8 @@ func createBackendWithStorage(t *testing.T) (*backend, logical.Storage) { if err != nil { t.Fatal(err) } + // Assume for our tests we have performed the migration already. + b.pkiStorageVersion.Store(1) return b, config.StorageView } diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 2b248940b3f30..843c4bb413fd2 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -87,6 +87,10 @@ func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, da func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { var err error + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not create root CA until migration has completed"), nil + } + exported, format, role, errorResp := b.getGenerationParams(ctx, data, req.MountPoint) if errorResp != nil { return errorResp, nil diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index 6082af90e5b68..2c160a324ba1d 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -16,55 +16,88 @@ import ( // and we need to perform it again... const latestMigrationVersion = 1 -type legacyBundleMigration struct { +type legacyBundleMigrationLog struct { Hash string `json:"hash" structs:"hash" mapstructure:"hash"` Created time.Time `json:"created" structs:"created" mapstructure:"created"` MigrationVersion int `json:"migrationVersion" structs:"migrationVersion" mapstructure:"migrationVersion"` } -func migrateStorage(ctx context.Context, req *logical.InitializationRequest, logger log.Logger) error { - s := req.Storage - legacyBundle, err := getLegacyCertBundle(ctx, s) +type migrationInfo struct { + isRequired bool + legacyBundle *certutil.CertBundle + legacyBundleHash string + migrationLog *legacyBundleMigrationLog +} + +func getMigrationInfo(ctx context.Context, s logical.Storage) (migrationInfo, error) { + migrationInfo := migrationInfo{ + isRequired: false, + legacyBundle: nil, + legacyBundleHash: "", + } + var err error + + migrationInfo.legacyBundle, err = getLegacyCertBundle(ctx, s) if err != nil { - return err + return migrationInfo, err } - if legacyBundle == nil { - // No legacy certs to migrate, we are done... - logger.Debug("No legacy certs found, no migration required.") - return nil + if migrationInfo.legacyBundle == nil { + return migrationInfo, nil } - migrationEntry, err := getLegacyBundleMigrationLog(ctx, s) + migrationInfo.migrationLog, err = getLegacyBundleMigrationLog(ctx, s) if err != nil { - return err + return migrationInfo, err } - hash, err := computeHashOfLegacyBundle(legacyBundle) + + migrationInfo.legacyBundleHash, err = computeHashOfLegacyBundle(migrationInfo.legacyBundle) if err != nil { - return err + return migrationInfo, err } - if migrationEntry != nil { + if migrationInfo.migrationLog != nil { // At this point we have already migrated something previously. - if migrationEntry.Hash == hash && - migrationEntry.MigrationVersion == latestMigrationVersion { + if migrationInfo.migrationLog.Hash == migrationInfo.legacyBundleHash && + migrationInfo.migrationLog.MigrationVersion == latestMigrationVersion { + return migrationInfo, nil + } + } + + migrationInfo.isRequired = true + return migrationInfo, nil +} + +func migrateStorage(ctx context.Context, req *logical.InitializationRequest, logger log.Logger) error { + s := req.Storage + migrationInfo, err := getMigrationInfo(ctx, req.Storage) + if err != nil { + return err + } + + if !migrationInfo.isRequired { + if migrationInfo.legacyBundle == nil { + // No legacy certs to migrate, we are done... + logger.Debug("No legacy certs found, no migration required.") + } + if migrationInfo.migrationLog != nil { // The hashes are the same, no need to try and re-import... logger.Debug("existing migration hash found and matched legacy bundle, skipping migration.") - return nil } + return nil } logger.Info("performing PKI migration to new keys/issuers layout") - anIssuer, aKey, err := writeCaBundle(ctx, s, legacyBundle, "current", "current") + anIssuer, aKey, err := writeCaBundle(ctx, s, migrationInfo.legacyBundle, "current", "current") if err != nil { return err } logger.Debug("Migration generated the following ids and set them as defaults", "issuer id", anIssuer.ID, "key id", aKey.ID) - err = setLegacyBundleMigrationLog(ctx, s, &legacyBundleMigration{ - Hash: hash, + err = setLegacyBundleMigrationLog(ctx, s, &legacyBundleMigrationLog{ + Hash: migrationInfo.legacyBundleHash, Created: time.Now(), MigrationVersion: latestMigrationVersion, }) @@ -90,7 +123,7 @@ func computeHashOfLegacyBundle(bundle *certutil.CertBundle) (string, error) { return hex.EncodeToString(hasher.Sum(nil)), nil } -func getLegacyBundleMigrationLog(ctx context.Context, s logical.Storage) (*legacyBundleMigration, error) { +func getLegacyBundleMigrationLog(ctx context.Context, s logical.Storage) (*legacyBundleMigrationLog, error) { entry, err := s.Get(ctx, legacyMigrationBundleLogKey) if err != nil { return nil, err @@ -100,7 +133,7 @@ func getLegacyBundleMigrationLog(ctx context.Context, s logical.Storage) (*legac return nil, nil } - lbm := &legacyBundleMigration{} + lbm := &legacyBundleMigrationLog{} err = entry.DecodeJSON(lbm) if err != nil { // If we can't decode our bundle, lets scrap it and assume a blank value, @@ -110,7 +143,7 @@ func getLegacyBundleMigrationLog(ctx context.Context, s logical.Storage) (*legac return lbm, nil } -func setLegacyBundleMigrationLog(ctx context.Context, s logical.Storage, lbm *legacyBundleMigration) error { +func setLegacyBundleMigrationLog(ctx context.Context, s logical.Storage, lbm *legacyBundleMigrationLog) error { json, err := logical.StorageEntryJSON(legacyMigrationBundleLogKey, lbm) if err != nil { return err diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 0a922af799ac1..0bd0acc05bcf6 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -191,7 +191,7 @@ func getKeyRef(data *framework.FieldData) string { func extractRef(data *framework.FieldData, paramName string) string { value := strings.TrimSpace(data.Get(paramName).(string)) - if strings.ToLower(value) == defaultRef { + if strings.EqualFold(value, defaultRef) { return defaultRef } return value From e9e4a064b5045037aa59b0ebe416abf1056de330 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Fri, 22 Apr 2022 16:06:57 -0400 Subject: [PATCH 54/76] Always write migration entry to trigger secondary clusters to wake up - Some PR feedback and handle a case in which the primary cluster does not have a CA bundle within storage but somehow a secondary does. --- builtin/logical/pki/backend.go | 12 ++-- builtin/logical/pki/cert_util.go | 18 ++++- builtin/logical/pki/path_intermediate.go | 2 +- builtin/logical/pki/storage_migrations.go | 70 +++++++++---------- .../logical/pki/storage_migrations_test.go | 41 +++++++++-- 5 files changed, 90 insertions(+), 53 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index c12854775cf83..81df482afeb83 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -260,15 +260,15 @@ func (b *backend) metricsWrap(callType string, roleMode int, ofunc roleOperation } // initialize is used to perform a possible PKI storage migration if needed -func (b *backend) initialize(ctx context.Context, req *logical.InitializationRequest) error { +func (b *backend) initialize(ctx context.Context, _ *logical.InitializationRequest) error { // Early exit if not a primary cluster or performance secondary with a local mount. if b.System().ReplicationState().HasState(consts.ReplicationDRSecondary|consts.ReplicationPerformanceStandby) || (!b.System().LocalMount() && b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary)) { - b.Logger().Debug("skipping pki migration as we are not on primary or secondary with a local mount") + b.Logger().Debug("skipping PKI migration as we are not on primary or secondary with a local mount") return nil } - if err := migrateStorage(ctx, req, b.Logger()); err != nil { + if err := migrateStorage(ctx, b.storage, b.Logger()); err != nil { b.Logger().Error("Error during migration of PKI mount: " + err.Error()) return err } @@ -291,11 +291,9 @@ func (b *backend) updatePkiStorageVersion(ctx context.Context) { } if info.isRequired { - b.Logger().Info("PKI migration status is required, reading cert bundle from legacy ca location") + b.Logger().Info("PKI migration is required, reading cert bundle from legacy ca location") b.pkiStorageVersion.Store(0) - } - - if !info.isRequired { + } else { b.Logger().Debug("PKI migration completed, reading cert bundle from key/issuer storage") b.pkiStorageVersion.Store(1) } diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index 3275754d57685..0eea30ec50835 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -89,11 +89,22 @@ func getFormat(data *framework.FieldData) string { return format } -// Fetches the CA info. +// fetchCAInfo will fetch the CA info, will return an error if no ca info exists. func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRef string) (*certutil.CAInfoBundle, error) { bundle, err := fetchCertBundle(ctx, b, req.Storage, issuerRef) if err != nil { - return nil, errutil.InternalError{Err: err.Error()} + switch err.(type) { + case errutil.UserError: + return nil, err + case errutil.InternalError: + return nil, err + default: + return nil, errutil.InternalError{Err: fmt.Sprintf("error fetching CA info: %v", err)} + } + } + + if bundle == nil { + return nil, errutil.UserError{Err: "no CA information is present"} } parsedBundle, err := parseCABundle(ctx, b, req, bundle) @@ -130,6 +141,7 @@ func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRe // performed or load the bundle from the new key/issuer storage. Any function that needs a bundle // should load it using this method to maintain compatibility on secondary nodes for which their // primary's have not upgraded yet. +// NOTE: This function can return a nil, nil response. func fetchCertBundle(ctx context.Context, b *backend, s logical.Storage, issuerRef string) (*certutil.CertBundle, error) { if b.useLegacyBundleCaStorage() { // We have not completed the migration so attempt to load the bundle from the legacy location @@ -139,7 +151,7 @@ func fetchCertBundle(ctx context.Context, b *backend, s logical.Storage, issuerR id, err := resolveIssuerReference(ctx, s, issuerRef) if err != nil { // Usually a bad label from the user or misconfigured default. - return nil, err + return nil, errutil.UserError{Err: err.Error()} } return fetchCertBundleByIssuerId(ctx, s, id, true) diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index 29337a9b3bac9..ed519b17adb83 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -48,7 +48,7 @@ func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Req var err error if b.useLegacyBundleCaStorage() { - return logical.ErrorResponse("Can not create intermediary until migration has completed"), nil + return logical.ErrorResponse("Can not create intermediate until migration has completed"), nil } exported, format, role, errorResp := b.getGenerationParams(ctx, data, req.MountPoint) diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index 2c160a324ba1d..4856d21d88fa5 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -34,18 +34,15 @@ func getMigrationInfo(ctx context.Context, s logical.Storage) (migrationInfo, er isRequired: false, legacyBundle: nil, legacyBundleHash: "", + migrationLog: nil, } - var err error + var err error migrationInfo.legacyBundle, err = getLegacyCertBundle(ctx, s) if err != nil { return migrationInfo, err } - if migrationInfo.legacyBundle == nil { - return migrationInfo, nil - } - migrationInfo.migrationLog, err = getLegacyBundleMigrationLog(ctx, s) if err != nil { return migrationInfo, err @@ -56,46 +53,43 @@ func getMigrationInfo(ctx context.Context, s logical.Storage) (migrationInfo, er return migrationInfo, err } - if migrationInfo.migrationLog != nil { - // At this point we have already migrated something previously. - if migrationInfo.migrationLog.Hash == migrationInfo.legacyBundleHash && - migrationInfo.migrationLog.MigrationVersion == latestMigrationVersion { - return migrationInfo, nil - } + // Even if there isn't anything to migrate, we always want to write out the log entry + // as that will trigger the secondary clusters to toggle/wake up + if (migrationInfo.migrationLog == nil) || + (migrationInfo.migrationLog.Hash != migrationInfo.legacyBundleHash) || + (migrationInfo.migrationLog.MigrationVersion != latestMigrationVersion) { + migrationInfo.isRequired = true } - migrationInfo.isRequired = true return migrationInfo, nil } -func migrateStorage(ctx context.Context, req *logical.InitializationRequest, logger log.Logger) error { - s := req.Storage - migrationInfo, err := getMigrationInfo(ctx, req.Storage) +func migrateStorage(ctx context.Context, s logical.Storage, logger log.Logger) error { + migrationInfo, err := getMigrationInfo(ctx, s) if err != nil { return err } if !migrationInfo.isRequired { - if migrationInfo.legacyBundle == nil { - // No legacy certs to migrate, we are done... - logger.Debug("No legacy certs found, no migration required.") - } - if migrationInfo.migrationLog != nil { - // The hashes are the same, no need to try and re-import... - logger.Debug("existing migration hash found and matched legacy bundle, skipping migration.") - } + // No migration was deemed to be required. + logger.Debug("existing migration found and was considered valid, skipping migration.") return nil } logger.Info("performing PKI migration to new keys/issuers layout") - - anIssuer, aKey, err := writeCaBundle(ctx, s, migrationInfo.legacyBundle, "current", "current") - if err != nil { - return err + if migrationInfo.legacyBundle != nil { + anIssuer, aKey, err := writeCaBundle(ctx, s, migrationInfo.legacyBundle, "current", "current") + if err != nil { + return err + } + logger.Debug("Migration generated the following ids and set them as defaults", + "issuer id", anIssuer.ID, "key id", aKey.ID) + } else { + logger.Debug("No legacy CA certs found, no migration required.") } - logger.Debug("Migration generated the following ids and set them as defaults", - "issuer id", anIssuer.ID, "key id", aKey.ID) + // We always want to write out this log entry as the secondary clusters leverage this path to wake up + // if they were upgraded prior to the primary cluster's migration occurred. err = setLegacyBundleMigrationLog(ctx, s, &legacyBundleMigrationLog{ Hash: migrationInfo.legacyBundleHash, Created: time.Now(), @@ -104,21 +98,25 @@ func migrateStorage(ctx context.Context, req *logical.InitializationRequest, log if err != nil { return err } + logger.Info("successfully completed migration to new keys/issuers layout") return nil } func computeHashOfLegacyBundle(bundle *certutil.CertBundle) (string, error) { - // We only hash the main certificate and the certs within the CAChain, - // assuming that any sort of change that occurred would have influenced one of those two fields. hasher := sha256.New() - if _, err := hasher.Write([]byte(bundle.Certificate)); err != nil { - return "", err - } - for _, cert := range bundle.CAChain { - if _, err := hasher.Write([]byte(cert)); err != nil { + // Generate an empty hash if the bundle does not exist. + if bundle != nil { + // We only hash the main certificate and the certs within the CAChain, + // assuming that any sort of change that occurred would have influenced one of those two fields. + if _, err := hasher.Write([]byte(bundle.Certificate)); err != nil { return "", err } + for _, cert := range bundle.CAChain { + if _, err := hasher.Write([]byte(cert)); err != nil { + return "", err + } + } } return hex.EncodeToString(hasher.Sum(nil)), nil } diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index e655ce3b70fa5..bed87fc796640 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -11,11 +11,16 @@ import ( ) func Test_migrateStorageEmptyStorage(t *testing.T) { + startTime := time.Now() ctx := context.Background() b, s := createBackendWithStorage(t) - request := &logical.InitializationRequest{Storage: s} - err := migrateStorage(ctx, request, b.Logger()) + // Reset the version the helper above set to 1. + b.pkiStorageVersion.Store(0) + require.True(t, b.useLegacyBundleCaStorage(), "pre migration we should have been told to use legacy storage.") + + request := &logical.InitializationRequest{Storage: s} + err := b.initialize(ctx, request) require.NoError(t, err) issuerIds, err := listIssuers(ctx, s) @@ -28,13 +33,35 @@ func Test_migrateStorageEmptyStorage(t *testing.T) { logEntry, err := getLegacyBundleMigrationLog(ctx, s) require.NoError(t, err) - require.Nil(t, logEntry) + require.NotNil(t, logEntry) + require.Equal(t, latestMigrationVersion, logEntry.MigrationVersion) + require.True(t, len(strings.TrimSpace(logEntry.Hash)) > 0, + "Hash value (%s) should not have been empty", logEntry.Hash) + require.True(t, startTime.Before(logEntry.Created), + "created log entry time (%v) was before our start time(%v)?", logEntry.Created, startTime) + + require.False(t, b.useLegacyBundleCaStorage(), "post migration we are still told to use legacy storage") + + // Make sure we can re-run the migration without issues + request = &logical.InitializationRequest{Storage: s} + err = b.initialize(ctx, request) + require.NoError(t, err) + logEntry2, err := getLegacyBundleMigrationLog(ctx, s) + require.NoError(t, err) + require.NotNil(t, logEntry2) + + // Make sure the hash and created times have not changed. + require.Equal(t, logEntry.Created, logEntry2.Created) + require.Equal(t, logEntry.Hash, logEntry2.Hash) } func Test_migrateStorageSimpleBundle(t *testing.T) { startTime := time.Now() ctx := context.Background() b, s := createBackendWithStorage(t) + // Reset the version the helper above set to 1. + b.pkiStorageVersion.Store(0) + require.True(t, b.useLegacyBundleCaStorage(), "pre migration we should have been told to use legacy storage.") bundle := genCertBundle(t, b) json, err := logical.StorageEntryJSON(legacyCertBundlePath, bundle) @@ -43,8 +70,8 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { require.NoError(t, err) request := &logical.InitializationRequest{Storage: s} - - err = migrateStorage(ctx, request, b.Logger()) + err = b.initialize(ctx, request) + require.NoError(t, err) require.NoError(t, err) issuerIds, err := listIssuers(ctx, s) @@ -99,7 +126,7 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { require.Equal(t, &issuerConfigEntry{DefaultIssuerId: issuerId}, issuersConfig) // Make sure if we attempt to re-run the migration nothing happens... - err = migrateStorage(ctx, request, b.Logger()) + err = migrateStorage(ctx, s, b.Logger()) require.NoError(t, err) logEntry2, err := getLegacyBundleMigrationLog(ctx, s) require.NoError(t, err) @@ -107,4 +134,6 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { require.Equal(t, logEntry.Created, logEntry2.Created) require.Equal(t, logEntry.Hash, logEntry2.Hash) + + require.False(t, b.useLegacyBundleCaStorage(), "post migration we are still told to use legacy storage") } From 5b692bb61d8b591fb2351075e57d59134f49d5a4 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 25 Apr 2022 10:51:23 -0400 Subject: [PATCH 55/76] Update CA Chain to report entire chain This merges the ca_chain JSON field (of the /certs/ca_chain path) with the regular certificate field, returning the root of trust always. This also affects the non-JSON (raw) endpoints as well. We return the default issuer's chain here, rather than all known issuers (as that may not form a strict chain). Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend_test.go | 6 ++++-- builtin/logical/pki/path_fetch.go | 12 +----------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index b199989560e27..70b4b36127a30 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -1735,7 +1735,7 @@ func TestBackend_PathFetchValidRaw(t *testing.T) { } rootCaAsPem := resp.Data["certificate"].(string) - // The ca_chain call at least for now does not return the root CA authority + // Chain should contain the root. resp, err = b.HandleRequest(context.Background(), &logical.Request{ Operation: logical.ReadOperation, Path: "ca_chain", @@ -1747,7 +1747,9 @@ func TestBackend_PathFetchValidRaw(t *testing.T) { if resp != nil && resp.IsError() { t.Fatalf("failed read ca_chain, %#v", resp) } - require.Equal(t, []byte{}, resp.Data[logical.HTTPRawBody], "ca_chain response should have been empty") + if strings.Count(string(resp.Data[logical.HTTPRawBody].([]byte)), rootCaAsPem) != 1 { + t.Fatalf("expected raw chain to contain the root cert") + } // The ca/pem should return us the actual CA... resp, err = b.HandleRequest(context.Background(), &logical.Request{ diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index c7a5d8d0e2e21..70b8190dbf4b6 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -205,17 +205,6 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data } if serial == "ca_chain" { - caChain := caInfo.GetCAChain() - var certStr string - for _, ca := range caChain { - block := pem.Block{ - Type: "CERTIFICATE", - Bytes: ca.Bytes, - } - certStr = strings.Join([]string{certStr, strings.TrimSpace(string(pem.EncodeToMemory(&block)))}, "\n") - } - certificate = []byte(strings.TrimSpace(certStr)) - rawChain := caInfo.GetFullChain() var chainStr string for _, ca := range rawChain { @@ -226,6 +215,7 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data chainStr = strings.Join([]string{chainStr, strings.TrimSpace(string(pem.EncodeToMemory(&block)))}, "\n") } fullChain = []byte(strings.TrimSpace(chainStr)) + certificate = fullChain } else if serial == "ca" { certificate = caInfo.Certificate.Raw From 21e0d48d4e5aca94b075fc7fbb91827e9d04d310 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 25 Apr 2022 14:55:57 -0400 Subject: [PATCH 56/76] Allow explicit issuer override on roles When a role is used to generate a certificate (such as with the sign/ and issue/ legacy paths or the legacy sign-verbatim/ paths), we prefer that issuer to the one on the request. This allows operators to set an issuer (other than default) for requests to be issued against, effectively making the change no different from the users' perspective as it is "just" a different role name. Signed-off-by: Alexander Scheel --- builtin/logical/pki/path_issue_sign.go | 32 +++++++++++++++++++++++--- builtin/logical/pki/path_roles.go | 25 ++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 9993cd5eeb675..8dade60aa7168 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/base64" "fmt" + "strings" "time" "github.com/hashicorp/vault/sdk/framework" @@ -206,6 +207,11 @@ func (b *backend) pathSignVerbatim(ctx context.Context, req *logical.Request, da *entry.GenerateLease = *role.GenerateLease } entry.NoStore = role.NoStore + entry.Issuer = role.Issuer + } + + if len(entry.Issuer) == 0 { + entry.Issuer = defaultRef } return b.pathIssueSignCert(ctx, req, data, entry, true, true) @@ -218,9 +224,29 @@ func (b *backend) pathIssueSignCert(ctx context.Context, req *logical.Request, d return nil, logical.ErrReadOnly } - issuerName := getIssuerRef(data) - if len(issuerName) == 0 { - return logical.ErrorResponse("missing issuer reference"), nil + // We prefer the issuer from the role in two cases: + // + // 1. On the legacy sign-verbatim paths, as we always provision an issuer + // in both the role and role-less cases, and + // 2. On the legacy sign/:role or issue/:role paths, as the issuer was + // set on the role directly (either via upgrade or not). Note that + // the updated issuer/:ref/{sign,issue}/:role path is not affected, + // and we instead pull the issuer out of the path instead (which + // allows users with access to those paths to manually choose their + // issuer in desired scenarios). + var issuerName string + if strings.HasPrefix(req.Path, "sign-verbatim/") || strings.HasPrefix(req.Path, "sign/") || strings.HasPrefix(req.Path, "issue/") { + issuerName = role.Issuer + if len(issuerName) == 0 { + issuerName = defaultRef + } + } else { + // Otherwise, we must have a newer API which requires an issuer + // reference. Fetch it in this case + issuerName = getIssuerRef(data) + if len(issuerName) == 0 { + return logical.ErrorResponse("missing issuer reference"), nil + } } format := getFormat(data) diff --git a/builtin/logical/pki/path_roles.go b/builtin/logical/pki/path_roles.go index 07fee55dc1a86..7da48344c9f77 100644 --- a/builtin/logical/pki/path_roles.go +++ b/builtin/logical/pki/path_roles.go @@ -405,6 +405,12 @@ for "generate_lease".`, Description: `Set the not after field of the certificate with specified date value. The value format should be given in UTC format YYYY-MM-ddTHH:MM:SSZ.`, }, + "issuer_ref": { + Type: framework.TypeString, + Description: `Reference to the issuer used to sign requests +serviced by this role.`, + Default: defaultRef, + }, }, Callbacks: map[logical.Operation]framework.OperationFunc{ @@ -527,6 +533,14 @@ func (b *backend) getRole(ctx context.Context, s logical.Storage, n string) (*ro modified = true } + // Set the issuer field to default if not set. We want to do this + // unconditionally as we should probably never have an empty issuer + // on a stored roles. + if len(result.Issuer) == 0 { + result.Issuer = defaultRef + modified = true + } + if modified && (b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary)) { jsonEntry, err := logical.StorageEntryJSON("role/"+n, &result) if err != nil { @@ -628,6 +642,7 @@ func (b *backend) pathRoleCreate(ctx context.Context, req *logical.Request, data BasicConstraintsValidForNonCA: data.Get("basic_constraints_valid_for_non_ca").(bool), NotBeforeDuration: time.Duration(data.Get("not_before_duration").(int)) * time.Second, NotAfter: data.Get("not_after").(string), + Issuer: data.Get("issuer_ref").(string), } allowedOtherSANs := data.Get("allowed_other_sans").([]string) @@ -690,6 +705,14 @@ func (b *backend) pathRoleCreate(ctx context.Context, req *logical.Request, data } *entry.AllowWildcardCertificates = allow_wildcard_certificates.(bool) + // Ensure issuers ref is set to to a non-empty value. Note that we never + // resolve the reference (to an issuerId) at role creation time; instead, + // resolve it at use time. This allows values such as `default` or other + // user-assigned names to "float" and change over time. + if len(entry.Issuer) == 0 { + entry.Issuer = defaultRef + } + // Store it jsonEntry, err := logical.StorageEntryJSON("role/"+name, entry) if err != nil { @@ -836,6 +859,7 @@ type roleEntry struct { BasicConstraintsValidForNonCA bool `json:"basic_constraints_valid_for_non_ca" mapstructure:"basic_constraints_valid_for_non_ca"` NotBeforeDuration time.Duration `json:"not_before_duration" mapstructure:"not_before_duration"` NotAfter string `json:"not_after" mapstructure:"not_after"` + Issuer string `json:"issuer" mapstructure:"issuer"` // Used internally for signing intermediates AllowExpirationPastCA bool } @@ -884,6 +908,7 @@ func (r *roleEntry) ToResponseData() map[string]interface{} { "basic_constraints_valid_for_non_ca": r.BasicConstraintsValidForNonCA, "not_before_duration": int64(r.NotBeforeDuration.Seconds()), "not_after": r.NotAfter, + "issuer_ref": r.Issuer, } if r.MaxPathLength != nil { responseData["max_path_length"] = r.MaxPathLength From 104b42e7940ecd3be7a0c33198e26095aa051bb6 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 25 Apr 2022 14:57:50 -0400 Subject: [PATCH 57/76] Add tests for role-based issuer selection Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend_test.go | 134 ++++++++++++++++++++++++++-- 1 file changed, 127 insertions(+), 7 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index 70b4b36127a30..f815891ec3952 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -1716,6 +1716,125 @@ func generateRoleSteps(t *testing.T, useCSRs bool) []logicaltest.TestStep { return ret } +func TestRolesAltIssuer(t *testing.T) { + coreConfig := &vault.CoreConfig{ + LogicalBackends: map[string]logical.Factory{ + "pki": Factory, + }, + } + cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + defer cluster.Cleanup() + + client := cluster.Cores[0].Client + var err error + err = client.Sys().Mount("pki", &api.MountInput{ + Type: "pki", + Config: api.MountConfigInput{ + DefaultLeaseTTL: "16h", + MaxLeaseTTL: "60h", + }, + }) + if err != nil { + t.Fatal(err) + } + + // Create two issuers. + resp, err := client.Logical().Write("pki/root/generate/internal", map[string]interface{}{ + "common_name": "root a - example.com", + "issuer_name": "root-a", + "key_type": "ec", + }) + require.NoError(t, err) + require.NotNil(t, resp) + rootAPem := resp.Data["certificate"].(string) + rootACert := parseCert(t, rootAPem) + + resp, err = client.Logical().Write("pki/root/generate/internal", map[string]interface{}{ + "common_name": "root b - example.com", + "issuer_name": "root-b", + "key_type": "ec", + }) + require.NoError(t, err) + require.NotNil(t, resp) + rootBPem := resp.Data["certificate"].(string) + rootBCert := parseCert(t, rootBPem) + + // Create three roles: one with no assignment, one with explicit root-a, + // one with explicit root-b. + _, err = client.Logical().Write("pki/roles/use-default", map[string]interface{}{ + "allow_any_name": true, + "enforce_hostnames": false, + "key_type": "ec", + }) + require.NoError(t, err) + + _, err = client.Logical().Write("pki/roles/use-root-a", map[string]interface{}{ + "allow_any_name": true, + "enforce_hostnames": false, + "key_type": "ec", + "issuer_ref": "root-a", + }) + require.NoError(t, err) + + _, err = client.Logical().Write("pki/roles/use-root-b", map[string]interface{}{ + "allow_any_name": true, + "enforce_hostnames": false, + "issuer_ref": "root-b", + }) + require.NoError(t, err) + + // Now issue certs against these roles. + resp, err = client.Logical().Write("pki/issue/use-default", map[string]interface{}{ + "common_name": "testing", + "ttl": "5s", + }) + require.NoError(t, err) + leafPem := resp.Data["certificate"].(string) + leafCert := parseCert(t, leafPem) + err = leafCert.CheckSignatureFrom(rootACert) + require.NoError(t, err, "should be signed by root-a but wasn't") + + resp, err = client.Logical().Write("pki/issue/use-root-a", map[string]interface{}{ + "common_name": "testing", + "ttl": "5s", + }) + require.NoError(t, err) + leafPem = resp.Data["certificate"].(string) + leafCert = parseCert(t, leafPem) + err = leafCert.CheckSignatureFrom(rootACert) + require.NoError(t, err, "should be signed by root-a but wasn't") + + resp, err = client.Logical().Write("pki/issue/use-root-b", map[string]interface{}{ + "common_name": "testing", + "ttl": "5s", + }) + require.NoError(t, err) + leafPem = resp.Data["certificate"].(string) + leafCert = parseCert(t, leafPem) + err = leafCert.CheckSignatureFrom(rootBCert) + require.NoError(t, err, "should be signed by root-b but wasn't") + + // Update the default issuer to be root B and make sure that the + // use-default role updates. + _, err = client.Logical().Write("pki/config/issuers", map[string]interface{}{ + "default": "root-b", + }) + require.NoError(t, err) + + resp, err = client.Logical().Write("pki/issue/use-default", map[string]interface{}{ + "common_name": "testing", + "ttl": "5s", + }) + require.NoError(t, err) + leafPem = resp.Data["certificate"].(string) + leafCert = parseCert(t, leafPem) + err = leafCert.CheckSignatureFrom(rootBCert) + require.NoError(t, err, "should be signed by root-b but wasn't") +} + func TestBackend_PathFetchValidRaw(t *testing.T) { b, storage := createBackendWithStorage(t) @@ -3543,6 +3662,7 @@ func TestReadWriteDeleteRoles(t *testing.T) { "province": []interface{}{}, "street_address": []interface{}{}, "code_signing_flag": false, + "issuer_ref": "default", } if diff := deep.Equal(expectedData, resp.Data); len(diff) > 0 { @@ -4873,7 +4993,7 @@ func mountPKIEndpoint(t *testing.T, client *api.Client, path string) { require.NoError(t, err, "failed mounting pki endpoint") } -func requireSignedBy(t *testing.T, cert x509.Certificate, key crypto.PublicKey) { +func requireSignedBy(t *testing.T, cert *x509.Certificate, key crypto.PublicKey) { switch key.(type) { case *rsa.PublicKey: requireRSASignedBy(t, cert, key.(*rsa.PublicKey)) @@ -4886,7 +5006,7 @@ func requireSignedBy(t *testing.T, cert x509.Certificate, key crypto.PublicKey) } } -func requireRSASignedBy(t *testing.T, cert x509.Certificate, key *rsa.PublicKey) { +func requireRSASignedBy(t *testing.T, cert *x509.Certificate, key *rsa.PublicKey) { require.Contains(t, []x509.SignatureAlgorithm{x509.SHA256WithRSA, x509.SHA512WithRSA}, cert.SignatureAlgorithm, "only sha256 signatures supported") @@ -4909,7 +5029,7 @@ func requireRSASignedBy(t *testing.T, cert x509.Certificate, key *rsa.PublicKey) require.NoError(t, err, "the certificate was not signed by the expected public rsa key.") } -func requireECDSASignedBy(t *testing.T, cert x509.Certificate, key *ecdsa.PublicKey) { +func requireECDSASignedBy(t *testing.T, cert *x509.Certificate, key *ecdsa.PublicKey) { require.Contains(t, []x509.SignatureAlgorithm{x509.ECDSAWithSHA256, x509.ECDSAWithSHA512}, cert.SignatureAlgorithm, "only ecdsa signatures supported") @@ -4928,21 +5048,21 @@ func requireECDSASignedBy(t *testing.T, cert x509.Certificate, key *ecdsa.Public require.True(t, verify, "the certificate was not signed by the expected public ecdsa key.") } -func requireED25519SignedBy(t *testing.T, cert x509.Certificate, key ed25519.PublicKey) { +func requireED25519SignedBy(t *testing.T, cert *x509.Certificate, key ed25519.PublicKey) { require.Equal(t, x509.PureEd25519, cert.SignatureAlgorithm) ed25519.Verify(key, cert.RawTBSCertificate, cert.Signature) } -func parseCert(t *testing.T, pemCert string) x509.Certificate { +func parseCert(t *testing.T, pemCert string) *x509.Certificate { block, _ := pem.Decode([]byte(pemCert)) require.NotNil(t, block, "failed to decode PEM block") cert, err := x509.ParseCertificate(block.Bytes) require.NoError(t, err) - return *cert + return cert } -func requireMatchingPublicKeys(t *testing.T, cert x509.Certificate, key crypto.PublicKey) { +func requireMatchingPublicKeys(t *testing.T, cert *x509.Certificate, key crypto.PublicKey) { certPubKey := cert.PublicKey require.True(t, reflect.DeepEqual(certPubKey, key), "public keys mismatched: got: %v, expected: %v", certPubKey, key) From 8b2f2b08cb353489092e0dbfcae4fbe5046fbfce Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 25 Apr 2022 09:43:27 -0400 Subject: [PATCH 58/76] Expand NotAfter limit enforcement behavior Vault previously strictly enforced NotAfter/ttl values on certificate requests, erring if the requested TTL extended past the NotAfter date of the issuer. In the event of issuing an intermediate, this behavior was ignored, instead permitting the issuance. Users generally do not think to check their issuer's NotAfter date when requesting a certificate; thus this behavior was generally surprising. Per RFC 5280 however, issuers need to maintain status information throughout the life cycle of the issued cert. If this leaf cert were to be issued for a longer duration than the parent issuer, the CA must still maintain revocation information past its expiration. Thus, we add an option to the issuer to change the desired behavior: - err, to err out, - permit, to permit the longer NotAfter date, or - truncate, to silently truncate the expiration to the issuer's NotAfter date. Since expiration of certificates in the system's trust store are not generally validated (when validating an arbitrary leaf, e.g., during TLS validation), permit should generally only be used in that case. However, browsers usually validate intermediate's validity periods, and thus truncate should likely be used (as with permit, the leaf's chain will not validate towards the end of the issuance period). Signed-off-by: Alexander Scheel --- builtin/logical/pki/cert_util.go | 35 ++++++++---- builtin/logical/pki/crl_util.go | 2 +- builtin/logical/pki/path_fetch_issuers.go | 57 +++++++++++++++---- builtin/logical/pki/path_roles.go | 2 - builtin/logical/pki/path_root.go | 6 +- builtin/logical/pki/storage.go | 24 ++++---- builtin/logical/pki/storage_migrations.go | 18 ++++-- .../logical/pki/storage_migrations_test.go | 2 +- sdk/helper/certutil/types.go | 11 +++- 9 files changed, 111 insertions(+), 46 deletions(-) diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index 0eea30ec50835..e0b8b816c7b83 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -91,7 +91,7 @@ func getFormat(data *framework.FieldData) string { // fetchCAInfo will fetch the CA info, will return an error if no ca info exists. func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRef string) (*certutil.CAInfoBundle, error) { - bundle, err := fetchCertBundle(ctx, b, req.Storage, issuerRef) + entry, bundle, err := fetchCertBundle(ctx, b, req.Storage, issuerRef) if err != nil { switch err.(type) { case errutil.UserError: @@ -119,7 +119,11 @@ func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRe return nil, errutil.UserError{Err: fmt.Sprintf("unable to fetch corresponding key for issuer %v; unable to use this issuer for signing", issuerRef)} } - caInfo := &certutil.CAInfoBundle{ParsedCertBundle: *parsedBundle, URLs: nil} + caInfo := &certutil.CAInfoBundle{ + ParsedCertBundle: *parsedBundle, + URLs: nil, + LeafNotAfterBehavior: entry.LeafNotAfterBehavior, + } entries, err := getURLs(ctx, req) if err != nil { @@ -142,7 +146,7 @@ func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRe // should load it using this method to maintain compatibility on secondary nodes for which their // primary's have not upgraded yet. // NOTE: This function can return a nil, nil response. -func fetchCertBundle(ctx context.Context, b *backend, s logical.Storage, issuerRef string) (*certutil.CertBundle, error) { +func fetchCertBundle(ctx context.Context, b *backend, s logical.Storage, issuerRef string) (*issuerEntry, *certutil.CertBundle, error) { if b.useLegacyBundleCaStorage() { // We have not completed the migration so attempt to load the bundle from the legacy location return getLegacyCertBundle(ctx, s) @@ -151,7 +155,7 @@ func fetchCertBundle(ctx context.Context, b *backend, s logical.Storage, issuerR id, err := resolveIssuerReference(ctx, s, issuerRef) if err != nil { // Usually a bad label from the user or misconfigured default. - return nil, errutil.UserError{Err: err.Error()} + return nil, nil, errutil.UserError{Err: err.Error()} } return fetchCertBundleByIssuerId(ctx, s, id, true) @@ -1279,13 +1283,22 @@ func generateCreationBundle(b *backend, data *inputBundle, caSign *certutil.CAIn } else { notAfter = time.Now().Add(ttl) } - // If it's not self-signed, verify that the issued certificate won't be - // valid past the lifetime of the CA certificate - if caSign != nil && - notAfter.After(caSign.Certificate.NotAfter) && !data.role.AllowExpirationPastCA { - - return nil, errutil.UserError{Err: fmt.Sprintf( - "cannot satisfy request, as TTL would result in notAfter %s that is beyond the expiration of the CA certificate at %s", notAfter.Format(time.RFC3339Nano), caSign.Certificate.NotAfter.Format(time.RFC3339Nano))} + if caSign != nil && notAfter.After(caSign.Certificate.NotAfter) { + // If it's not self-signed, verify that the issued certificate + // won't be valid past the lifetime of the CA certificate, and + // act accordingly. This is dependent based on the issuers's + // LeafNotAfterBehavior argument. + switch caSign.LeafNotAfterBehavior { + case certutil.PermitNotAfterBehavior: + // Explicitly do nothing. + case certutil.TruncateNotAfterBehavior: + notAfter = caSign.Certificate.NotAfter + case certutil.ErrNotAfterBehavior: + fallthrough + default: + return nil, errutil.UserError{Err: fmt.Sprintf( + "cannot satisfy request, as TTL would result in notAfter %s that is beyond the expiration of the CA certificate at %s", notAfter.Format(time.RFC3339Nano), caSign.Certificate.NotAfter.Format(time.RFC3339Nano))} + } } } diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index e2755b6801536..1fc40efaaf701 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -440,7 +440,7 @@ func buildCRL(ctx context.Context, b *backend, req *logical.Request, forceNew bo revokedCerts = revoked WRITE: - bundle, caErr := fetchCertBundleByIssuerId(ctx, req.Storage, thisIssuerId, true /* need the signing key */) + _, bundle, caErr := fetchCertBundleByIssuerId(ctx, req.Storage, thisIssuerId, true /* need the signing key */) if caErr != nil { switch caErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index cff5fb98ae653..68a6d08d69c9f 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/helper/certutil" "github.com/hashicorp/vault/sdk/logical" ) @@ -66,11 +67,24 @@ func pathGetIssuer(b *backend) *framework.Path { func buildPathGetIssuer(b *backend, pattern string) *framework.Path { fields := map[string]*framework.FieldSchema{} fields = addIssuerRefNameFields(fields) + + // Fields for updating issuer. fields["manual_chain"] = &framework.FieldSchema{ Type: framework.TypeCommaStringSlice, Description: `Chain of issuer references to use to build this issuer's computed CAChain field, when non-empty.`, } + fields["leaf_not_after_behavior"] = &framework.FieldSchema{ + Type: framework.TypeString, + Description: `Behavior of leaf's NotAfter fields: "err" to error +if the computed NotAfter date exceeds that of this issuer; "truncate" to +silently truncate to that of this issuer; or "permit" to allow this +issuance to succeed (with NotAfter exceeding that of an issuer). Note that +not all values will results in certificates that can be validated through +the entire validity period. It is suggested to use "truncate" for +intermediate CAs and "permit" only for root CAs.`, + Default: "err", + } return &framework.Path{ // Returns a JSON entry. @@ -119,12 +133,13 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data return &logical.Response{ Data: map[string]interface{}{ - "issuer_id": issuer.ID, - "issuer_name": issuer.Name, - "key_id": issuer.KeyID, - "certificate": issuer.Certificate, - "manual_chain": respManualChain, - "ca_chain": issuer.CAChain, + "issuer_id": issuer.ID, + "issuer_name": issuer.Name, + "key_id": issuer.KeyID, + "certificate": issuer.Certificate, + "manual_chain": respManualChain, + "ca_chain": issuer.CAChain, + "leaf_not_after_behavior": issuer.LeafNotAfterBehavior, }, }, nil } @@ -158,6 +173,18 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da } newPath := data.Get("manual_chain").([]string) + rawLeafBehavior := data.Get("leaf_not_after_behavior").(string) + var newLeafBehavior certutil.NotAfterBehavior + switch rawLeafBehavior { + case "err": + newLeafBehavior = certutil.ErrNotAfterBehavior + case "truncate": + newLeafBehavior = certutil.TruncateNotAfterBehavior + case "permit": + newLeafBehavior = certutil.PermitNotAfterBehavior + default: + return logical.ErrorResponse("Unknown value for field `leaf_not_after_behavior`. Possible values are `err`, `truncate`, and `permit`."), nil + } modified := false @@ -166,6 +193,11 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da modified = true } + if newLeafBehavior != issuer.LeafNotAfterBehavior { + issuer.LeafNotAfterBehavior = newLeafBehavior + modified = true + } + var updateChain bool var constructedChain []issuerID for index, newPathRef := range newPath { @@ -219,12 +251,13 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da return &logical.Response{ Data: map[string]interface{}{ - "issuer_id": issuer.ID, - "issuer_name": issuer.Name, - "key_id": issuer.KeyID, - "certificate": issuer.Certificate, - "manual_chain": respManualChain, - "ca_chain": issuer.CAChain, + "issuer_id": issuer.ID, + "issuer_name": issuer.Name, + "key_id": issuer.KeyID, + "certificate": issuer.Certificate, + "manual_chain": respManualChain, + "ca_chain": issuer.CAChain, + "leaf_not_after_behavior": issuer.LeafNotAfterBehavior, }, }, nil } diff --git a/builtin/logical/pki/path_roles.go b/builtin/logical/pki/path_roles.go index 7da48344c9f77..2f2a29adce4d0 100644 --- a/builtin/logical/pki/path_roles.go +++ b/builtin/logical/pki/path_roles.go @@ -860,8 +860,6 @@ type roleEntry struct { NotBeforeDuration time.Duration `json:"not_before_duration" mapstructure:"not_before_duration"` NotAfter string `json:"not_after" mapstructure:"not_after"` Issuer string `json:"issuer" mapstructure:"issuer"` - // Used internally for signing intermediates - AllowExpirationPastCA bool } func (r *roleEntry) ToResponseData() map[string]interface{} { diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 843c4bb413fd2..ba7c6f839a7a7 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -240,7 +240,6 @@ func (b *backend) pathIssuerSignIntermediate(ctx context.Context, req *logical.R AllowedOtherSANs: []string{"*"}, AllowedSerialNumbers: []string{"*"}, AllowedURISANs: []string{"*"}, - AllowExpirationPastCA: true, NotAfter: data.Get("not_after").(string), } *role.AllowWildcardCertificates = true @@ -262,6 +261,11 @@ func (b *backend) pathIssuerSignIntermediate(ctx context.Context, req *logical.R } } + // Since we are signing an intermediate, we explicitly want to override + // the leaf NotAfterBehavior to permit issuing intermediates longer than + // the life of this issuer. + signingBundle.LeafNotAfterBehavior = certutil.PermitNotAfterBehavior + useCSRValues := data.Get("use_csr_values").(bool) maxPathLengthIface, ok := data.GetOk("max_path_length") diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 94a0f1239d843..514bd95d35f90 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -56,13 +56,14 @@ type keyEntry struct { } type issuerEntry struct { - ID issuerID `json:"id" structs:"id" mapstructure:"id"` - Name string `json:"name" structs:"name" mapstructure:"name"` - KeyID keyID `json:"key_id" structs:"key_id" mapstructure:"key_id"` - Certificate string `json:"certificate" structs:"certificate" mapstructure:"certificate"` - CAChain []string `json:"ca_chain" structs:"ca_chain" mapstructure:"ca_chain"` - ManualChain []issuerID `json:"manual_chain" structs:"manual_chain" mapstructure:"manual_chain"` - SerialNumber string `json:"serial_number" structs:"serial_number" mapstructure:"serial_number"` + ID issuerID `json:"id" structs:"id" mapstructure:"id"` + Name string `json:"name" structs:"name" mapstructure:"name"` + KeyID keyID `json:"key_id" structs:"key_id" mapstructure:"key_id"` + Certificate string `json:"certificate" structs:"certificate" mapstructure:"certificate"` + CAChain []string `json:"ca_chain" structs:"ca_chain" mapstructure:"ca_chain"` + ManualChain []issuerID `json:"manual_chain" structs:"manual_chain" mapstructure:"manual_chain"` + SerialNumber string `json:"serial_number" structs:"serial_number" mapstructure:"serial_number"` + LeafNotAfterBehavior certutil.NotAfterBehavior `json:"not_after_behavior" structs:"not_after_behavior" mapstructure:"not_after_behavior"` } type localCRLConfigEntry struct { @@ -440,6 +441,7 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu result.ID = genIssuerId() result.Name = issuerName result.Certificate = certValue + result.LeafNotAfterBehavior = certutil.ErrNotAfterBehavior // We shouldn't add CSRs or multiple certificates in this countCertificates := strings.Count(result.Certificate, "-BEGIN ") @@ -659,10 +661,10 @@ func resolveIssuerCRLPath(ctx context.Context, s logical.Storage, reference stri // Builds a certutil.CertBundle from the specified issuer identifier, // optionally loading the key or not. -func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuerID, loadKey bool) (*certutil.CertBundle, error) { +func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuerID, loadKey bool) (*issuerEntry, *certutil.CertBundle, error) { issuer, err := fetchIssuerById(ctx, s, id) if err != nil { - return nil, err + return nil, nil, err } var bundle certutil.CertBundle @@ -674,14 +676,14 @@ func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuer if loadKey && issuer.KeyID != keyID("") { key, err := fetchKeyById(ctx, s, issuer.KeyID) if err != nil { - return nil, err + return nil, nil, err } bundle.PrivateKeyType = key.PrivateKeyType bundle.PrivateKey = key.PrivateKey } - return &bundle, nil + return issuer, &bundle, nil } func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.CertBundle, issuerName string, keyName string) (*issuerEntry, *keyEntry, error) { diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index 4856d21d88fa5..1796d34a39c42 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -38,7 +38,7 @@ func getMigrationInfo(ctx context.Context, s logical.Storage) (migrationInfo, er } var err error - migrationInfo.legacyBundle, err = getLegacyCertBundle(ctx, s) + _, migrationInfo.legacyBundle, err = getLegacyCertBundle(ctx, s) if err != nil { return migrationInfo, err } @@ -150,21 +150,27 @@ func setLegacyBundleMigrationLog(ctx context.Context, s logical.Storage, lbm *le return s.Put(ctx, json) } -func getLegacyCertBundle(ctx context.Context, s logical.Storage) (*certutil.CertBundle, error) { +func getLegacyCertBundle(ctx context.Context, s logical.Storage) (*issuerEntry, *certutil.CertBundle, error) { entry, err := s.Get(ctx, legacyCertBundlePath) if err != nil { - return nil, err + return nil, nil, err } if entry == nil { - return nil, nil + return nil, nil, nil } cb := &certutil.CertBundle{} err = entry.DecodeJSON(cb) if err != nil { - return nil, err + return nil, nil, err + } + + // Fake a storage entry with backwards compatibility in mind. We only need + // the fields in the CAInfoBundle; everything else doesn't matter. + issuer := &issuerEntry{ + LeafNotAfterBehavior: certutil.ErrNotAfterBehavior, } - return cb, nil + return issuer, cb, nil } diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index bed87fc796640..fb1f1dd159b2c 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -112,7 +112,7 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { require.Equal(t, bundle.PrivateKeyType, key.PrivateKeyType) // Make sure we kept the old bundle - certBundle, err := getLegacyCertBundle(ctx, s) + _, certBundle, err := getLegacyCertBundle(ctx, s) require.NoError(t, err) require.Equal(t, bundle, certBundle) diff --git a/sdk/helper/certutil/types.go b/sdk/helper/certutil/types.go index c648d797fbe01..5b70a6c31d9b1 100644 --- a/sdk/helper/certutil/types.go +++ b/sdk/helper/certutil/types.go @@ -675,9 +675,18 @@ type URLEntries struct { OCSPServers []string `json:"ocsp_servers" structs:"ocsp_servers" mapstructure:"ocsp_servers"` } +type NotAfterBehavior int + +const ( + ErrNotAfterBehavior NotAfterBehavior = iota + TruncateNotAfterBehavior + PermitNotAfterBehavior +) + type CAInfoBundle struct { ParsedCertBundle - URLs *URLEntries + URLs *URLEntries + LeafNotAfterBehavior NotAfterBehavior } func (b *CAInfoBundle) GetCAChain() []*CertBlock { From 8aa7caa657d0e2942b4acf69c05aff8ba7a051d1 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 25 Apr 2022 10:19:58 -0400 Subject: [PATCH 59/76] Add tests for expanded issuance behaviors Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend_test.go | 94 +++++++++++++++++++ .../logical/pki/storage_migrations_test.go | 2 + sdk/helper/certutil/certutil_test.go | 14 +++ 3 files changed, 110 insertions(+) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index f815891ec3952..bdaa6942a2b77 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -4960,6 +4960,100 @@ func TestIntermediateWithExistingKey(t *testing.T) { require.Equal(t, myKeyId1, myKeyId3, "our new ca did not seem to reuse the key as we expected.") } +func TestIssuanceTTLs(t *testing.T) { + coreConfig := &vault.CoreConfig{ + LogicalBackends: map[string]logical.Factory{ + "pki": Factory, + }, + } + cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + defer cluster.Cleanup() + + client := cluster.Cores[0].Client + var err error + err = client.Sys().Mount("pki", &api.MountInput{ + Type: "pki", + Config: api.MountConfigInput{ + DefaultLeaseTTL: "16h", + MaxLeaseTTL: "60h", + }, + }) + if err != nil { + t.Fatal(err) + } + + resp, err := client.Logical().Write("pki/root/generate/internal", map[string]interface{}{ + "common_name": "root example.com", + "issuer_name": "root", + "ttl": "15s", + "key_type": "ec", + }) + require.NoError(t, err) + require.NotNil(t, resp) + + _, err = client.Logical().Write("pki/roles/local-testing", map[string]interface{}{ + "allow_any_name": true, + "enforce_hostnames": false, + "key_type": "ec", + }) + require.NoError(t, err) + + _, err = client.Logical().Write("pki/issue/local-testing", map[string]interface{}{ + "common_name": "testing", + "ttl": "1s", + }) + require.NoError(t, err, "expected issuance to succeed due to shorter ttl than cert ttl") + + _, err = client.Logical().Write("pki/issue/local-testing", map[string]interface{}{ + "common_name": "testing", + }) + require.Error(t, err, "expected issuance to fail due to longer default ttl than cert ttl") + + resp, err = client.Logical().Write("pki/issuer/root", map[string]interface{}{ + "issuer_name": "root", + "leaf_not_after_behavior": "permit", + }) + require.NoError(t, err) + require.NotNil(t, resp) + + _, err = client.Logical().Write("pki/issue/local-testing", map[string]interface{}{ + "common_name": "testing", + }) + require.NoError(t, err, "expected issuance to succeed due to permitted longer TTL") + + resp, err = client.Logical().Write("pki/issuer/root", map[string]interface{}{ + "issuer_name": "root", + "leaf_not_after_behavior": "truncate", + }) + require.NoError(t, err) + require.NotNil(t, resp) + + _, err = client.Logical().Write("pki/issue/local-testing", map[string]interface{}{ + "common_name": "testing", + }) + require.NoError(t, err, "expected issuance to succeed due to truncated ttl") + + // Sleep until the parent cert expires. + time.Sleep(16 * time.Second) + + resp, err = client.Logical().Write("pki/issuer/root", map[string]interface{}{ + "issuer_name": "root", + "leaf_not_after_behavior": "err", + }) + require.NoError(t, err) + require.NotNil(t, resp) + + // Even 1s ttl should now fail. + _, err = client.Logical().Write("pki/issue/local-testing", map[string]interface{}{ + "common_name": "testing", + "ttl": "1s", + }) + require.Error(t, err, "expected issuance to fail due to longer default ttl than cert ttl") +} + func TestSealWrappedStorageConfigured(t *testing.T) { b, _ := createBackendWithStorage(t) wrappedEntries := b.Backend.PathsSpecial.SealWrapStorage diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index fb1f1dd159b2c..ccb826fd726be 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/hashicorp/vault/sdk/helper/certutil" "github.com/hashicorp/vault/sdk/logical" "github.com/stretchr/testify/require" ) @@ -96,6 +97,7 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { issuer, err := fetchIssuerById(ctx, s, issuerId) require.NoError(t, err) require.Equal(t, "current", issuer.Name) // RFC says we should import with Name=current + require.Equal(t, certutil.ErrNotAfterBehavior, issuer.LeafNotAfterBehavior) key, err := fetchKeyById(ctx, s, keyId) require.NoError(t, err) diff --git a/sdk/helper/certutil/certutil_test.go b/sdk/helper/certutil/certutil_test.go index 0b5b74ddb0df2..f92b7ce5f0191 100644 --- a/sdk/helper/certutil/certutil_test.go +++ b/sdk/helper/certutil/certutil_test.go @@ -896,6 +896,20 @@ func TestComparePublicKeysAndType(t *testing.T) { } } +func TestNotAfterValues(t *testing.T) { + if ErrNotAfterBehavior != 0 { + t.Fatalf("Expected ErrNotAfterBehavior=%v to have value 0", ErrNotAfterBehavior) + } + + if TruncateNotAfterBehavior != 1 { + t.Fatalf("Expected TruncateNotAfterBehavior=%v to have value 1", TruncateNotAfterBehavior) + } + + if PermitNotAfterBehavior != 2 { + t.Fatalf("Expected PermitNotAfterBehavior=%v to have value 2", PermitNotAfterBehavior) + } +} + func genRsaKey(t *testing.T) *rsa.PrivateKey { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { From 35a871638f7ab81131f0959ca251200a5d1ff804 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 26 Apr 2022 16:11:37 -0400 Subject: [PATCH 60/76] Add warning on keyless default issuer (#15178) Signed-off-by: Alexander Scheel --- builtin/logical/pki/path_config_ca.go | 23 +++++++++++++---- builtin/logical/pki/path_manage_issuers.go | 30 +++++++++++++++++----- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/builtin/logical/pki/path_config_ca.go b/builtin/logical/pki/path_config_ca.go index b70292d8e77ed..9cd4d1ca2544f 100644 --- a/builtin/logical/pki/path_config_ca.go +++ b/builtin/logical/pki/path_config_ca.go @@ -83,16 +83,29 @@ func (b *backend) pathCAIssuersWrite(ctx context.Context, req *logical.Request, return logical.ErrorResponse("Error resolving issuer reference: " + err.Error()), nil } + response := &logical.Response{ + Data: map[string]interface{}{ + "default": parsedIssuer, + }, + } + + entry, err := fetchIssuerById(ctx, req.Storage, parsedIssuer) + if err != nil { + return logical.ErrorResponse("Unable to fetch issuer: " + err.Error()), nil + } + + if len(entry.KeyID) == 0 { + msg := "This selected default issuer has no key associated with it. Some operations like issuing certificates and signing CRLs will be unavailable with the requested default issuer until a key is imported or the default issuer is changed." + response.AddWarning(msg) + b.Logger().Error(msg) + } + err = updateDefaultIssuerId(ctx, req.Storage, parsedIssuer) if err != nil { return logical.ErrorResponse("Error updating issuer configuration: " + err.Error()), nil } - return &logical.Response{ - Data: map[string]interface{}{ - "default": parsedIssuer, - }, - }, nil + return response, nil } const pathConfigIssuersHelpSyn = `Read and set the default issuer certificate for signing.` diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index df4d1c2008eea..573d1880d89b2 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -187,6 +187,14 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d } } + response := &logical.Response{ + Data: map[string]interface{}{ + "mapping": issuerKeyMap, + "imported_keys": createdKeys, + "imported_issuers": createdIssuers, + }, + } + if len(createdIssuers) > 0 { err := buildCRLs(ctx, b, req, true) if err != nil { @@ -194,13 +202,21 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d } } - return &logical.Response{ - Data: map[string]interface{}{ - "mapping": issuerKeyMap, - "imported_keys": createdKeys, - "imported_issuers": createdIssuers, - }, - }, nil + // While we're here, check if we should warn about a bad default key. We + // do this unconditionally if the issuer or key was modified, so the admin + // is always warned. But if unrelated key material was imported, we do + // not warn. + config, err := getIssuersConfig(ctx, req.Storage) + if err == nil && len(config.DefaultIssuerId) > 0 { + // We can use the mapping above to check the issuer mapping. + if keyId, ok := issuerKeyMap[string(config.DefaultIssuerId)]; !ok || len(keyId) == 0 { + msg := "The default issuer has no key associated with it. Some operations like issuing certificates and signing CRLs will be unavailable with the requested default issuer until a key is imported or the default issuer is changed." + response.AddWarning(msg) + b.Logger().Error(msg) + } + } + + return response, nil } const ( From 1b53ccdd67cfb65d611d1146ce057c0b4fbe797a Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 26 Apr 2022 17:39:29 -0400 Subject: [PATCH 61/76] Update PKI to new Operations framework (#15180) The backend Framework has updated Callbacks (used extensively in PKI) to become deprecated; Operations takes their place and clarifies forwarding of requests. We switch to the new format everywhere, updating some bad assumptions about forwarding along the way. Anywhere writes are handled (that should be propagated to all nodes in all clusters), we choose to forward the request all the way up to the performance primary cluster's primary node. This holds for issuers/keys, roles, and configs (such as CRL config, which is globally set for all clusters despite all clusters having their own separate CRL). Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 19 ++++++---- builtin/logical/pki/path_config_ca.go | 22 +++++++++--- builtin/logical/pki/path_config_crl.go | 13 +++++-- builtin/logical/pki/path_config_urls.go | 10 ++++-- builtin/logical/pki/path_fetch.go | 42 ++++++++++++++-------- builtin/logical/pki/path_fetch_issuers.go | 32 ++++++++++++----- builtin/logical/pki/path_issue_sign.go | 18 ++++++---- builtin/logical/pki/path_manage_issuers.go | 9 +++-- builtin/logical/pki/path_revoke.go | 20 ++++++++--- builtin/logical/pki/path_roles.go | 26 ++++++++++---- builtin/logical/pki/path_sign_issuers.go | 12 ++++--- 11 files changed, 161 insertions(+), 62 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 81df482afeb83..f79d1aec6eb63 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -25,22 +25,27 @@ const ( /* * PKI requests are a bit special to keep up with the various failure and load issues. - * The main ca and intermediate requests are always forwarded to the Primary cluster's active - * node to write and send the key material/config globally across all clusters. * - * CRL/Revocation and Issued certificate apis are handled by the active node within the cluster - * they originate. Which means if a request comes into a performance secondary cluster the writes + * Any requests to write/delete shared data (such as roles, issuers, keys, and configuration) + * are always forwarded to the Primary cluster's active node to write and send the key + * material/config globally across all clusters. Reads should be handled locally, to give a + * sense of where this cluster's replication state is at. + * + * CRL/Revocation and Fetch Certificate APIs are handled by the active node within the cluster + * they originate. This means, if a request comes into a performance secondary cluster, the writes * will be forwarded to that cluster's active node and not go all the way up to the performance primary's * active node. * - * If a certificate issue request has a role in which no_store is set to true that node itself - * will issue the certificate and not forward the request to the active node. + * If a certificate issue request has a role in which no_store is set to true, that node itself + * will issue the certificate and not forward the request to the active node, as this does not + * need to write to storage. * - * Following the same pattern if a managed key is involved to sign an issued certificate request + * Following the same pattern, if a managed key is involved to sign an issued certificate request * and the local node does not have access for some reason to it, the request will be forwarded to * the active node within the cluster only. * * To make sense of what goes where the following bits need to be analyzed within the codebase. + * * 1. The backend LocalStorage paths determine what storage paths will remain within a * cluster and not be forwarded to a performance primary * 2. Within each path's OperationHandler definition, check to see if ForwardPerformanceStandby & diff --git a/builtin/logical/pki/path_config_ca.go b/builtin/logical/pki/path_config_ca.go index 9cd4d1ca2544f..01d69321d2a56 100644 --- a/builtin/logical/pki/path_config_ca.go +++ b/builtin/logical/pki/path_config_ca.go @@ -18,8 +18,13 @@ secret key and certificate.`, }, }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.pathImportIssuers, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathImportIssuers, + // Read more about why these flags are set in backend.go. + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, }, HelpSynopsis: pathConfigCAHelpSyn, @@ -49,9 +54,16 @@ func pathConfigIssuers(b *backend) *framework.Path { }, }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathCAIssuersRead, - logical.UpdateOperation: b.pathCAIssuersWrite, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathCAIssuersRead, + }, + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathCAIssuersWrite, + // Read more about why these flags are set in backend.go. + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, }, HelpSynopsis: pathConfigIssuersHelpSyn, diff --git a/builtin/logical/pki/path_config_crl.go b/builtin/logical/pki/path_config_crl.go index e10c8d686b6b1..90dba1744a932 100644 --- a/builtin/logical/pki/path_config_crl.go +++ b/builtin/logical/pki/path_config_crl.go @@ -32,9 +32,16 @@ valid; defaults to 72 hours`, }, }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathCRLRead, - logical.UpdateOperation: b.pathCRLWrite, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathCRLRead, + }, + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathCRLWrite, + // Read more about why these flags are set in backend.go. + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, }, HelpSynopsis: pathConfigCRLHelpSyn, diff --git a/builtin/logical/pki/path_config_urls.go b/builtin/logical/pki/path_config_urls.go index c53d42d06973d..87e26a52d99c7 100644 --- a/builtin/logical/pki/path_config_urls.go +++ b/builtin/logical/pki/path_config_urls.go @@ -34,9 +34,13 @@ for the OCSP servers attribute. See also RFC 5280 Section 4.2.2.1.`, }, }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.pathWriteURL, - logical.ReadOperation: b.pathReadURL, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathWriteURL, + }, + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathReadURL, + }, }, HelpSynopsis: pathConfigURLsHelpSyn, diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index 70b8190dbf4b6..8b6d9d940cf7d 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -16,8 +16,10 @@ func pathFetchCA(b *backend) *framework.Path { return &framework.Path{ Pattern: `ca(/pem)?`, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathFetchRead, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathFetchRead, + }, }, HelpSynopsis: pathFetchHelpSyn, @@ -30,8 +32,10 @@ func pathFetchCAChain(b *backend) *framework.Path { return &framework.Path{ Pattern: `(cert/)?ca_chain`, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathFetchRead, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathFetchRead, + }, }, HelpSynopsis: pathFetchHelpSyn, @@ -44,8 +48,10 @@ func pathFetchCRL(b *backend) *framework.Path { return &framework.Path{ Pattern: `crl(/pem)?`, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathFetchRead, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathFetchRead, + }, }, HelpSynopsis: pathFetchHelpSyn, @@ -65,8 +71,10 @@ hyphen-separated octal`, }, }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathFetchRead, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathFetchRead, + }, }, HelpSynopsis: pathFetchHelpSyn, @@ -87,8 +95,10 @@ hyphen-separated octal`, }, }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathFetchRead, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathFetchRead, + }, }, HelpSynopsis: pathFetchHelpSyn, @@ -101,8 +111,10 @@ func pathFetchCRLViaCertPath(b *backend) *framework.Path { return &framework.Path{ Pattern: `cert/crl`, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathFetchRead, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathFetchRead, + }, }, HelpSynopsis: pathFetchHelpSyn, @@ -115,8 +127,10 @@ func pathFetchListCerts(b *backend) *framework.Path { return &framework.Path{ Pattern: "certs/?$", - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ListOperation: b.pathFetchCertList, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ListOperation: &framework.PathOperation{ + Callback: b.pathFetchCertList, + }, }, HelpSynopsis: pathFetchHelpSyn, diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 68a6d08d69c9f..b19ed30b0eae6 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -15,8 +15,10 @@ func pathListIssuers(b *backend) *framework.Path { return &framework.Path{ Pattern: "issuers/?$", - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ListOperation: b.pathListIssuersHandler, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ListOperation: &framework.PathOperation{ + Callback: b.pathListIssuersHandler, + }, }, HelpSynopsis: pathListIssuersHelpSyn, @@ -91,10 +93,22 @@ intermediate CAs and "permit" only for root CAs.`, Pattern: pattern, Fields: fields, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathGetIssuer, - logical.UpdateOperation: b.pathUpdateIssuer, - logical.DeleteOperation: b.pathDeleteIssuer, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathGetIssuer, + }, + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathUpdateIssuer, + // Read more about why these flags are set in backend.go. + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + logical.DeleteOperation: &framework.PathOperation{ + Callback: b.pathDeleteIssuer, + // Read more about why these flags are set in backend.go. + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, }, HelpSynopsis: pathGetIssuerHelpSyn, @@ -369,8 +383,10 @@ func buildPathGetIssuerCRL(b *backend, pattern string) *framework.Path { Pattern: pattern, Fields: fields, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathGetIssuerCRL, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathGetIssuerCRL, + }, }, HelpSynopsis: pathGetIssuerCRLHelpSyn, diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 8dade60aa7168..6039e2d41f75b 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -29,8 +29,10 @@ func buildPathIssue(b *backend, pattern string) *framework.Path { ret := &framework.Path{ Pattern: pattern, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.metricsWrap("issue", roleRequired, b.pathIssue), + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.metricsWrap("issue", roleRequired, b.pathIssue), + }, }, HelpSynopsis: pathIssueHelpSyn, @@ -55,8 +57,10 @@ func buildPathSign(b *backend, pattern string) *framework.Path { ret := &framework.Path{ Pattern: pattern, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.metricsWrap("sign", roleRequired, b.pathSign), + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.metricsWrap("sign", roleRequired, b.pathSign), + }, }, HelpSynopsis: pathSignHelpSyn, @@ -89,8 +93,10 @@ func buildPathIssuerSignVerbatim(b *backend, pattern string) *framework.Path { Pattern: pattern, Fields: map[string]*framework.FieldSchema{}, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.metricsWrap("sign-verbatim", roleOptional, b.pathSignVerbatim), + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.metricsWrap("sign-verbatim", roleOptional, b.pathSignVerbatim), + }, }, HelpSynopsis: pathIssuerSignVerbatimHelpSyn, diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index 573d1880d89b2..e252170679c04 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -84,8 +84,13 @@ secret-key (optional) and certificates.`, }, }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.pathImportIssuers, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathImportIssuers, + // Read more about why these flags are set in backend.go + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, }, HelpSynopsis: pathImportIssuersHelpSyn, diff --git a/builtin/logical/pki/path_revoke.go b/builtin/logical/pki/path_revoke.go index ee19a2e604306..47a3848ccb55b 100644 --- a/builtin/logical/pki/path_revoke.go +++ b/builtin/logical/pki/path_revoke.go @@ -22,8 +22,14 @@ hyphen-separated octal`, }, }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.metricsWrap("revoke", noRole, b.pathRevokeWrite), + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.metricsWrap("revoke", noRole, b.pathRevokeWrite), + // This should never be forwarded. See backend.go for more information. + // If this needs to write, the entire request will be forwarded to the + // active node of the current performance cluster, but we don't want to + // forward invalid revoke requests there. + }, }, HelpSynopsis: pathRevokeHelpSyn, @@ -35,8 +41,14 @@ func pathRotateCRL(b *backend) *framework.Path { return &framework.Path{ Pattern: `crl/rotate`, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathRotateCRLRead, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathRotateCRLRead, + // See backend.go; we will read a lot of data prior to calling write, + // so this request should be forwarded when it is first seen, not + // when it is ready to write. + ForwardPerformanceStandby: true, + }, }, HelpSynopsis: pathRotateCRLHelpSyn, diff --git a/builtin/logical/pki/path_roles.go b/builtin/logical/pki/path_roles.go index 2f2a29adce4d0..cd327df83f586 100644 --- a/builtin/logical/pki/path_roles.go +++ b/builtin/logical/pki/path_roles.go @@ -18,8 +18,10 @@ func pathListRoles(b *backend) *framework.Path { return &framework.Path{ Pattern: "roles/?$", - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ListOperation: b.pathRoleList, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ListOperation: &framework.PathOperation{ + Callback: b.pathRoleList, + }, }, HelpSynopsis: pathListRolesHelpSyn, @@ -413,10 +415,22 @@ serviced by this role.`, }, }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.pathRoleRead, - logical.UpdateOperation: b.pathRoleCreate, - logical.DeleteOperation: b.pathRoleDelete, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathRoleRead, + }, + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathRoleCreate, + // Read more about why these flags are set in backend.go. + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + logical.DeleteOperation: &framework.PathOperation{ + Callback: b.pathRoleDelete, + // Read more about why these flags are set in backend.go. + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, }, HelpSynopsis: pathRoleHelpSyn, diff --git a/builtin/logical/pki/path_sign_issuers.go b/builtin/logical/pki/path_sign_issuers.go index e54e45276e8f3..ae45245097b10 100644 --- a/builtin/logical/pki/path_sign_issuers.go +++ b/builtin/logical/pki/path_sign_issuers.go @@ -20,8 +20,10 @@ func pathIssuerSignIntermediateRaw(b *backend, pattern string) *framework.Path { path := &framework.Path{ Pattern: pattern, Fields: fields, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.pathIssuerSignIntermediate, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathIssuerSignIntermediate, + }, }, HelpSynopsis: pathIssuerSignIntermediateHelpSyn, @@ -98,8 +100,10 @@ func buildPathIssuerSignSelfIssued(b *backend, pattern string) *framework.Path { path := &framework.Path{ Pattern: pattern, Fields: fields, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.UpdateOperation: b.pathIssuerSignSelfIssued, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathIssuerSignSelfIssued, + }, }, HelpSynopsis: pathIssuerSignSelfIssuedHelpSyn, From c57c1ab942687f34f5d9996a93b65c038e20c652 Mon Sep 17 00:00:00 2001 From: kitography Date: Thu, 28 Apr 2022 11:35:06 -0400 Subject: [PATCH 62/76] Kitography/vault 5474 rebase (#15150) * These parts work (put in signature so that backend wouldn't break, but missing fields, desc, etc.) * Import and Generate API calls w/ needed additions to SDK. * make fmt * Add Help/Sync Text, fix some of internal/exported/kms code. * Fix PEM/DER Encoding issue. * make fmt * Standardize keyIdParam, keyNameParam, keyTypeParam * Add error response if key to be deleted is in use. * replaces all instances of "default" in code with defaultRef * Updates from Callbacks to Operations Function with explicit forwarding. * Fixes a panic with names not being updated everywhere. * add a logged error in addition to warning on deleting default key. * Normalize whitespace upon importing keys. Authored-by: Alexander Scheel * Fix isKeyInUse functionality. * Fixes tests associated with newline at end of key pem. --- builtin/logical/pki/backend.go | 7 + builtin/logical/pki/fields.go | 8 +- builtin/logical/pki/path_config_ca.go | 78 +++++- builtin/logical/pki/path_fetch_keys.go | 227 ++++++++++++++++++ builtin/logical/pki/path_manage_keys.go | 173 +++++++++++++ builtin/logical/pki/storage.go | 28 ++- .../logical/pki/storage_migrations_test.go | 2 +- builtin/logical/pki/storage_test.go | 4 +- builtin/logical/pki/util.go | 4 +- sdk/framework/backend_test.go | 1 + sdk/helper/certutil/helpers.go | 14 ++ sdk/helper/certutil/types.go | 33 +++ 12 files changed, 568 insertions(+), 11 deletions(-) create mode 100644 builtin/logical/pki/path_fetch_keys.go create mode 100644 builtin/logical/pki/path_manage_keys.go diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index f79d1aec6eb63..1a2493a0b015c 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -129,6 +129,13 @@ func Backend(conf *logical.BackendConfig) *backend { pathIssuerGenerateIntermediate(&b), pathConfigIssuers(&b), + // Key APIs + pathListKeys(&b), + pathKey(&b), + pathGenerateKey(&b), + pathImportKey(&b), + pathConfigKeys(&b), + // Fetch APIs have been lowered to favor the newer issuer API endpoints pathFetchCA(&b), pathFetchCAChain(&b), diff --git a/builtin/logical/pki/fields.go b/builtin/logical/pki/fields.go index 09a6eace39308..45c329bd8acf5 100644 --- a/builtin/logical/pki/fields.go +++ b/builtin/logical/pki/fields.go @@ -4,6 +4,10 @@ import "github.com/hashicorp/vault/sdk/framework" const ( issuerRefParam = "issuer_ref" + keyNameParam = "key_name" + keyRefParam = "key_ref" + keyIdParam = "key_id" + keyTypeParam = "key_type" ) // addIssueAndSignCommonFields adds fields common to both CA and non-CA issuing @@ -375,7 +379,7 @@ func addKeyRefNameFields(fields map[string]*framework.FieldSchema) map[string]*f } func addKeyNameField(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { - fields["key_name"] = &framework.FieldSchema{ + fields[keyNameParam] = &framework.FieldSchema{ Type: framework.TypeString, Description: `Provide a name for the key that will be generated, the name must be unique across all keys and not be the reserved value @@ -386,7 +390,7 @@ the name must be unique across all keys and not be the reserved value } func addKeyRefField(fields map[string]*framework.FieldSchema) map[string]*framework.FieldSchema { - fields["key_ref"] = &framework.FieldSchema{ + fields[keyRefParam] = &framework.FieldSchema{ Type: framework.TypeString, Description: `Reference to a existing key; either "default" for the configured default key, an identifier or the name assigned diff --git a/builtin/logical/pki/path_config_ca.go b/builtin/logical/pki/path_config_ca.go index 01d69321d2a56..8cebe94454336 100644 --- a/builtin/logical/pki/path_config_ca.go +++ b/builtin/logical/pki/path_config_ca.go @@ -48,7 +48,7 @@ func pathConfigIssuers(b *backend) *framework.Path { return &framework.Path{ Pattern: "config/issuers", Fields: map[string]*framework.FieldSchema{ - "default": { + defaultRef: { Type: framework.TypeString, Description: `Reference (name or identifier) to the default issuer.`, }, @@ -79,13 +79,13 @@ func (b *backend) pathCAIssuersRead(ctx context.Context, req *logical.Request, d return &logical.Response{ Data: map[string]interface{}{ - "default": config.DefaultIssuerId, + defaultRef: config.DefaultIssuerId, }, }, nil } func (b *backend) pathCAIssuersWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - newDefault := data.Get("default").(string) + newDefault := data.Get(defaultRef).(string) if len(newDefault) == 0 || newDefault == defaultRef { return logical.ErrorResponse("Invalid issuer specification; must be non-empty and can't be 'default'."), nil } @@ -130,6 +130,78 @@ accessible by the existing signing paths (/root/sign-intermediate, /root/sign-self-issued, /sign-verbatim, /sign/:role, and /issue/:role). ` +func pathConfigKeys(b *backend) *framework.Path { + return &framework.Path{ + Pattern: "config/keys", + Fields: map[string]*framework.FieldSchema{ + defaultRef: { + Type: framework.TypeString, + Description: `Reference (name or identifier) of the default key.`, + }, + }, + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathKeyDefaultWrite, + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathKeyDefaultRead, + ForwardPerformanceStandby: false, + ForwardPerformanceSecondary: false, + }, + }, + + HelpSynopsis: pathConfigKeysHelpSyn, + HelpDescription: pathConfigKeysHelpDesc, + } +} + +func (b *backend) pathKeyDefaultRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + config, err := getKeysConfig(ctx, req.Storage) + if err != nil { + return logical.ErrorResponse("Error loading keys configuration: " + err.Error()), nil + } + + return &logical.Response{ + Data: map[string]interface{}{ + defaultRef: config.DefaultKeyId, + }, + }, nil +} + +func (b *backend) pathKeyDefaultWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + newDefault := data.Get(defaultRef).(string) + if len(newDefault) == 0 || newDefault == defaultRef { + return logical.ErrorResponse("Invalid key specification; must be non-empty and can't be 'default'."), nil + } + + parsedKey, err := resolveKeyReference(ctx, req.Storage, newDefault) + if err != nil { + return logical.ErrorResponse("Error resolving issuer reference: " + err.Error()), nil + } + + err = updateDefaultKeyId(ctx, req.Storage, parsedKey) + if err != nil { + return logical.ErrorResponse("Error updating issuer configuration: " + err.Error()), nil + } + + return &logical.Response{ + Data: map[string]interface{}{ + defaultRef: parsedKey, + }, + }, nil +} + +const pathConfigKeysHelpSyn = `Read and set the default key used for signing` + +const pathConfigKeysHelpDesc = ` +This path allows configuration of key parameters. + +The "default" parameter controls which key is the default used by signing paths. +` + const pathConfigCAGenerateHelpSyn = ` Generate a new CA certificate and private key used for signing. ` diff --git a/builtin/logical/pki/path_fetch_keys.go b/builtin/logical/pki/path_fetch_keys.go new file mode 100644 index 0000000000000..21247ccf6955e --- /dev/null +++ b/builtin/logical/pki/path_fetch_keys.go @@ -0,0 +1,227 @@ +package pki + +import ( + "context" + "fmt" + + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/logical" +) + +func pathListKeys(b *backend) *framework.Path { + return &framework.Path{ + Pattern: "keys/?$", + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ListOperation: &framework.PathOperation{ + Callback: b.pathListKeysHandler, + ForwardPerformanceStandby: false, + ForwardPerformanceSecondary: false, + }, + }, + + HelpSynopsis: pathListKeysHelpSyn, + HelpDescription: pathListKeysHelpDesc, + } +} + +const ( + pathListKeysHelpSyn = `Fetch a list of all issuer keys` + pathListKeysHelpDesc = `This endpoint allows listing of known backing keys, returning +their identifier and their name (if set).` +) + +func (b *backend) pathListKeysHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + var responseKeys []string + responseInfo := make(map[string]interface{}) + + entries, err := listKeys(ctx, req.Storage) + if err != nil { + return nil, err + } + + for _, identifier := range entries { + key, err := fetchKeyById(ctx, req.Storage, identifier) + if err != nil { + return nil, err + } + + responseKeys = append(responseKeys, string(identifier)) + responseInfo[string(identifier)] = map[string]interface{}{ + keyNameParam: key.Name, + } + + } + return logical.ListResponseWithInfo(responseKeys, responseInfo), nil +} + +func pathKey(b *backend) *framework.Path { + pattern := "key/" + framework.GenericNameRegex(keyRefParam) + return buildPathKey(b, pattern) +} + +func buildPathKey(b *backend, pattern string) *framework.Path { + return &framework.Path{ + Pattern: pattern, + + Fields: map[string]*framework.FieldSchema{ + keyRefParam: { + Type: framework.TypeString, + Description: `Reference to key; either "default" for the configured default key, an identifier of a key, or the name assigned to the key.`, + Default: defaultRef, + }, + keyNameParam: { + Type: framework.TypeString, + Description: `Human-readable name for this key.`, + }, + }, + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.pathGetKeyHandler, + ForwardPerformanceStandby: false, + ForwardPerformanceSecondary: false, + }, + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathUpdateKeyHandler, + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + logical.DeleteOperation: &framework.PathOperation{ + Callback: b.pathDeleteKeyHandler, + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + }, + + HelpSynopsis: pathKeysHelpSyn, + HelpDescription: pathKeysHelpDesc, + } +} + +const ( + pathKeysHelpSyn = `Fetch a single issuer key` + pathKeysHelpDesc = `This allows fetching information associated with the underlying key. + +:ref can be either the literal value "default", in which case /config/keys +will be consulted for the present default key, an identifier of a key, +or its assigned name value. + +Writing to /key/:ref allows updating of the name field associated with +the certificate. +` +) + +func (b *backend) pathGetKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + keyRef := data.Get(keyRefParam).(string) + if len(keyRef) == 0 { + return logical.ErrorResponse("missing key reference"), nil + } + + keyId, err := resolveKeyReference(ctx, req.Storage, keyRef) + if err != nil { + return nil, err + } + if keyId == "" { + return logical.ErrorResponse("unable to resolve key id for reference" + keyRef), nil + } + + key, err := fetchKeyById(ctx, req.Storage, keyId) + if err != nil { + return nil, err + } + + return &logical.Response{ + Data: map[string]interface{}{ + keyIdParam: key.ID, + keyNameParam: key.Name, + keyTypeParam: key.PrivateKeyType, + }, + }, nil +} + +func (b *backend) pathUpdateKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + keyRef := data.Get(keyRefParam).(string) + if len(keyRef) == 0 { + return logical.ErrorResponse("missing key reference"), nil + } + + keyId, err := resolveKeyReference(ctx, req.Storage, keyRef) + if err != nil { + return nil, err + } + if keyId == "" { + return logical.ErrorResponse("unable to resolve key id for reference" + keyRef), nil + } + + key, err := fetchKeyById(ctx, req.Storage, keyId) + if err != nil { + return nil, err + } + + newName := data.Get(keyNameParam).(string) + if len(newName) > 0 && !nameMatcher.MatchString(newName) { + return logical.ErrorResponse("new key name outside of valid character limits"), nil + } + + if newName != key.Name { + key.Name = newName + + err := writeKey(ctx, req.Storage, *key) + if err != nil { + return nil, err + } + } + + resp := &logical.Response{ + Data: map[string]interface{}{ + keyIdParam: key.ID, + keyNameParam: key.Name, + keyTypeParam: key.PrivateKeyType, + }, + } + + if len(newName) == 0 { + resp.AddWarning("Name successfully deleted, you will now need to reference this key by it's Id: " + string(key.ID)) + } + + return resp, nil +} + +func (b *backend) pathDeleteKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + keyRef := data.Get(keyRefParam).(string) + if len(keyRef) == 0 { + return logical.ErrorResponse("missing key reference"), nil + } + + keyId, err := resolveKeyReference(ctx, req.Storage, keyRef) + if err != nil { + return nil, err + } + if keyId == "" { + return logical.ErrorResponse("unable to resolve key id for reference" + keyRef), nil + } + + keyInUse, issuerId, err := isKeyInUse(keyId.String(), ctx, req.Storage) + if err != nil { + return nil, err + } + if keyInUse { + return logical.ErrorResponse(fmt.Sprintf("Failed to Delete Key. Key in Use by Issuer: %s", issuerId)), nil + } + + wasDefault, err := deleteKey(ctx, req.Storage, keyId) + if err != nil { + return nil, err + } + + var response *logical.Response + if wasDefault { + msg := fmt.Sprintf("Deleted key %v (via key_ref %v); this was configured as the default key. Operations without an explicit key will not work until a new default is configured.", string(keyId), keyRef) + b.Logger().Error(msg) + response = &logical.Response{} + response.AddWarning(msg) + } + + return response, nil +} diff --git a/builtin/logical/pki/path_manage_keys.go b/builtin/logical/pki/path_manage_keys.go new file mode 100644 index 0000000000000..3ee6f0e61c5c7 --- /dev/null +++ b/builtin/logical/pki/path_manage_keys.go @@ -0,0 +1,173 @@ +package pki + +import ( + "context" + "strings" + + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/helper/certutil" + "github.com/hashicorp/vault/sdk/logical" +) + +func pathGenerateKey(b *backend) *framework.Path { + return &framework.Path{ + Pattern: "keys/generate/(internal|exported)", + + Fields: map[string]*framework.FieldSchema{ + keyNameParam: { + Type: framework.TypeString, + Description: "Optional name to be used for this key", + }, + keyTypeParam: { + Type: framework.TypeString, + Default: "rsa", + Description: `Type of the secret key to generate`, + }, + "key_bits": { + Type: framework.TypeInt, + Default: 2048, + Description: `Type of the secret key to generate`, + }, + }, + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.CreateOperation: &framework.PathOperation{ + Callback: b.pathGenerateKeyHandler, + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + }, + + HelpSynopsis: pathGenerateKeyHelpSyn, + HelpDescription: pathGenerateKeyHelpDesc, + } +} + +const ( + pathGenerateKeyHelpSyn = `Generate a new private key used for signing.` + pathGenerateKeyHelpDesc = `This endpoint will generate a new key pair of the specified type (internal, exported, or kms).` +) + +func (b *backend) pathGenerateKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + keyName, err := getKeyName(ctx, req.Storage, data) + if err != nil { // Fail Immediately if Key Name is in Use, etc... + return nil, err + } + keyType := data.Get(keyTypeParam).(string) + keyBits := data.Get("key_bits").(int) + + switch { + case strings.HasSuffix(req.Path, "/internal"): + // Internal key generation, stored in storage + keyBundle, err := certutil.GetKeyBundleFromKeyGenerator(keyType, keyBits, nil) + if err != nil { + return nil, err + } + privateKeyPemString, err := keyBundle.ToPrivateKeyPemString() + if err != nil { + return nil, err + } + key, _, err := importKey(ctx, req.Storage, privateKeyPemString, keyName) + if err != nil { + return nil, err + } + resp := logical.Response{ + Data: map[string]interface{}{ + keyIdParam: key.ID, + keyNameParam: key.Name, + keyTypeParam: key.PrivateKeyType, + }, + } + return &resp, nil + case strings.HasSuffix(req.Path, "/exported"): + // Same as internal key generation but we return the generated key + keyBundle, err := certutil.GetKeyBundleFromKeyGenerator(keyType, keyBits, nil) + if err != nil { + return nil, err + } + privateKeyPemString, err := keyBundle.ToPrivateKeyPemString() + if err != nil { + return nil, err + } + key, _, err := importKey(ctx, req.Storage, privateKeyPemString, keyName) + if err != nil { + return nil, err + } + resp := logical.Response{ + Data: map[string]interface{}{ + keyIdParam: key.ID, + keyNameParam: key.Name, + keyTypeParam: key.PrivateKeyType, + "private_key": privateKeyPemString, + }, + } + return &resp, nil + case strings.HasSuffix(req.Path, "/kms"): + return nil, errEntOnly + default: + return logical.ErrorResponse("Unknown type of key to generate"), nil + } +} + +func pathImportKey(b *backend) *framework.Path { + return &framework.Path{ + Pattern: "keys/import", + + Fields: map[string]*framework.FieldSchema{ + keyNameParam: { + Type: framework.TypeString, + Description: "Optional name to be used for this key", + }, + "pem_bundle": { + Type: framework.TypeString, + Description: `PEM-format, unencrypted secret key`, + }, + }, + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.CreateOperation: &framework.PathOperation{ + Callback: b.pathImportKeyHandler, + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + }, + + HelpSynopsis: pathImportKeyHelpSyn, + HelpDescription: pathImportKeyHelpDesc, + } +} + +const ( + pathImportKeyHelpSyn = `Import the specified key.` + pathImportKeyHelpDesc = `This endpoint allows importing a specified issuer key from a pem bundle. +If name is set, that will be set on the key.` +) + +func (b *backend) pathImportKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + keyValueInterface, isOk := data.GetOk("pem_bundle") + if !isOk { + return logical.ErrorResponse("keyValue must be set"), nil + } + keyValue := keyValueInterface.(string) + keyName := data.Get(keyNameParam).(string) + + key, existed, err := importKey(ctx, req.Storage, keyValue, keyName) + if err != nil { + return logical.ErrorResponse(err.Error()), nil + } + + resp := logical.Response{ + Data: map[string]interface{}{ + keyIdParam: key.ID, + keyNameParam: key.Name, + keyTypeParam: key.PrivateKeyType, + "backing": "", // This would show up as "Managed" in "type" + }, + } + + if existed { + resp.AddWarning("Key already imported, use key/ endpoint to update name.") + } + + return &resp, nil +} diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 514bd95d35f90..98821778a7c6e 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -158,6 +158,10 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName // and identifier); the last return field is whether or not an error // occurred. // + // Normalize whitespace before beginning. See note in importIssuer as to + // why we do this. + keyValue = strings.TrimSpace(keyValue) + "\n" + // // Before we can import a known key, we first need to know if the key // exists in storage already. This means iterating through all known // keys and comparing their private value against this value. @@ -205,7 +209,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName keyPublic := keySigner.Public() result.PrivateKeyType = certutil.GetPrivateKeyTypeFromSigner(keySigner) - // Finally we can write the key to storage. + // Finally, we can write the key to storage. if err := writeKey(ctx, s, result); err != nil { return nil, false, err } @@ -725,3 +729,25 @@ func genUuid() string { } return aUuid } + +func isKeyInUse(keyId string, ctx context.Context, s logical.Storage) (inUse bool, issuerId string, err error) { + knownIssuers, err := listIssuers(ctx, s) + if err != nil { + return true, "", err + } + + for _, issuerId := range knownIssuers { + issuerEntry, err := fetchIssuerById(ctx, s, issuerId) + if err != nil { + return true, issuerId.String(), errutil.InternalError{Err: fmt.Sprintf("unable to fetch pki issuer: %v", err)} + } + if issuerEntry == nil { + return true, issuerId.String(), errutil.InternalError{Err: fmt.Sprintf("Issuer listed: %s does not exist", issuerId.String())} + } + if issuerEntry.KeyID.String() == keyId { + return true, issuerId.String(), nil + } + } + + return false, "", nil +} diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index ccb826fd726be..d91b5d9c40b62 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -110,7 +110,7 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { // FIXME: Add tests for CAChain... require.Equal(t, keyId, key.ID) - require.Equal(t, bundle.PrivateKey, key.PrivateKey) + require.Equal(t, strings.TrimSpace(bundle.PrivateKey), strings.TrimSpace(key.PrivateKey)) require.Equal(t, bundle.PrivateKeyType, key.PrivateKeyType) // Make sure we kept the old bundle diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index 4d516ebd22f9f..39e32ef16a6e4 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -106,7 +106,7 @@ func Test_KeysIssuerImport(t *testing.T) { key1_ref1, existing, err := importKey(ctx, s, key1.PrivateKey, "key1") require.NoError(t, err) require.False(t, existing) - require.Equal(t, key1.PrivateKey, key1_ref1.PrivateKey) + require.Equal(t, strings.TrimSpace(key1.PrivateKey), strings.TrimSpace(key1_ref1.PrivateKey)) // Make sure if we attempt to re-import the same private key, no import/updates occur. // So the existing flag should be set to true and we do not update the existing Name field. @@ -166,7 +166,7 @@ func genIssuerAndKey(t *testing.T, b *backend) (issuerEntry, keyEntry) { pkiKey := keyEntry{ ID: keyId, PrivateKeyType: certBundle.PrivateKeyType, - PrivateKey: certBundle.PrivateKey, + PrivateKey: strings.TrimSpace(certBundle.PrivateKey) + "\n", } issuerId := genIssuerId() diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 0bd0acc05bcf6..77d777dae605a 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -158,7 +158,7 @@ func getIssuerName(ctx context.Context, s logical.Storage, data *framework.Field func getKeyName(ctx context.Context, s logical.Storage, data *framework.FieldData) (string, error) { keyName := "" - keyNameIface, ok := data.GetOk("key_name") + keyNameIface, ok := data.GetOk(keyNameParam) if ok { keyName = strings.TrimSpace(keyNameIface.(string)) @@ -186,7 +186,7 @@ func getIssuerRef(data *framework.FieldData) string { } func getKeyRef(data *framework.FieldData) string { - return extractRef(data, "key_ref") + return extractRef(data, keyRefParam) } func extractRef(data *framework.FieldData, paramName string) string { diff --git a/sdk/framework/backend_test.go b/sdk/framework/backend_test.go index c563a152b6057..9a2b5941457af 100644 --- a/sdk/framework/backend_test.go +++ b/sdk/framework/backend_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/go-secure-stdlib/strutil" + "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/logical" "github.com/stretchr/testify/require" diff --git a/sdk/helper/certutil/helpers.go b/sdk/helper/certutil/helpers.go index 819d96bb30f89..b9d3c61bc7f65 100644 --- a/sdk/helper/certutil/helpers.go +++ b/sdk/helper/certutil/helpers.go @@ -1228,3 +1228,17 @@ func GetPublicKeySize(key crypto.PublicKey) int { return -1 } + +func GetKeyBundleFromKeyGenerator(keyType string, keyBits int, keyGenerator KeyGenerator) (KeyBundle, error) { + result := KeyBundle{} + + if keyGenerator == nil { + keyGenerator = generatePrivateKey + } + + if err := keyGenerator(keyType, keyBits, &result, nil); err != nil { + return result, err + } + + return result, nil +} diff --git a/sdk/helper/certutil/types.go b/sdk/helper/certutil/types.go index 5b70a6c31d9b1..7f36c7ab5ea0a 100644 --- a/sdk/helper/certutil/types.go +++ b/sdk/helper/certutil/types.go @@ -138,6 +138,12 @@ type ParsedCSRBundle struct { CSR *x509.CertificateRequest } +type KeyBundle struct { + PrivateKeyType PrivateKeyType + PrivateKeyBytes []byte + PrivateKey crypto.Signer +} + func GetPrivateKeyTypeFromSigner(signer crypto.Signer) PrivateKeyType { switch signer.(type) { case *rsa.PrivateKey: @@ -852,3 +858,30 @@ func AddKeyUsages(data *CreationBundle, certTemplate *x509.Certificate) { certTemplate.ExtKeyUsage = append(certTemplate.ExtKeyUsage, x509.ExtKeyUsageMicrosoftKernelCodeSigning) } } + +// SetParsedPrivateKey sets the private key parameters on the bundle +func (p *KeyBundle) SetParsedPrivateKey(privateKey crypto.Signer, privateKeyType PrivateKeyType, privateKeyBytes []byte) { + p.PrivateKey = privateKey + p.PrivateKeyType = privateKeyType + p.PrivateKeyBytes = privateKeyBytes +} + +func (p *KeyBundle) ToPrivateKeyPemString() (string, error) { + block := pem.Block{} + + if p.PrivateKeyBytes != nil && len(p.PrivateKeyBytes) > 0 { + block.Bytes = p.PrivateKeyBytes + switch p.PrivateKeyType { + case RSAPrivateKey: + block.Type = "RSA PRIVATE KEY" + case ECPrivateKey: + block.Type = "EC PRIVATE KEY" + default: + block.Type = "PRIVATE KEY" + } + privateKeyPemString := strings.TrimSpace(string(pem.EncodeToMemory(&block))) + return privateKeyPemString, nil + } + + return "", errutil.InternalError{Err: "No Private Key Bytes to Wrap"} +} From e9abf67a00d90a7ea8447a6ab03c25629477f2ce Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Fri, 29 Apr 2022 10:11:03 -0400 Subject: [PATCH 63/76] Add alternative proposal PKI aliased paths (#15211) * Add aliased path for root/rotate/:exported This adds a user-friendly path name for generating a rotated root. We automatically choose the name "next" for the newly generated root at this path if it doesn't already exist. Signed-off-by: Alexander Scheel * Add aliased path for intermediate/cross-sign This allows cross-signatures to work. Signed-off-by: Alexander Scheel * Add path for replacing the current root This updates default to point to the value of the issuer with name "next" rather than its current value. Signed-off-by: Alexander Scheel * Remove plural issuers/ in signing paths These paths use a single issuer and thus shouldn't include the plural issuers/ as a path prefix, instead using the singular issuer/ path prefix. Signed-off-by: Alexander Scheel * Only warn if default issuer was imported When the default issuer was not (re-)imported, we'd fail to find it, causing an extraneous warning about missing keys, even though this issuer indeed had a key. Signed-off-by: Alexander Scheel * Add missing issuer sign/issue paths Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 5 ++++ builtin/logical/pki/chain_test.go | 2 +- builtin/logical/pki/path_config_ca.go | 28 ++++++++++++++++++++++ builtin/logical/pki/path_intermediate.go | 7 ++++++ builtin/logical/pki/path_issue_sign.go | 2 +- builtin/logical/pki/path_manage_issuers.go | 10 +++++++- builtin/logical/pki/path_root.go | 10 ++++++++ builtin/logical/pki/path_sign_issuers.go | 4 ++-- 8 files changed, 63 insertions(+), 5 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 1a2493a0b015c..2f1702d7ecf47 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -122,12 +122,17 @@ func Backend(conf *logical.BackendConfig) *backend { pathGetIssuer(&b), pathGetIssuerCRL(&b), pathImportIssuer(&b), + pathIssuerIssue(&b), + pathIssuerSign(&b), pathIssuerSignIntermediate(&b), pathIssuerSignSelfIssued(&b), pathIssuerSignVerbatim(&b), pathIssuerGenerateRoot(&b), + pathRotateRoot(&b), pathIssuerGenerateIntermediate(&b), + pathCrossSignIntermediate(&b), pathConfigIssuers(&b), + pathReplaceRoot(&b), // Key APIs pathListKeys(&b), diff --git a/builtin/logical/pki/chain_test.go b/builtin/logical/pki/chain_test.go index e0ff00d219021..95f1715864f47 100644 --- a/builtin/logical/pki/chain_test.go +++ b/builtin/logical/pki/chain_test.go @@ -117,7 +117,7 @@ func (c CBGenerateIntermediate) Run(t *testing.T, client *api.Client, mount stri csr := resp.Data["csr"].(string) // Sign CSR - url = fmt.Sprintf(mount+"/issuers/%s/sign-intermediate", c.Parent) + url = fmt.Sprintf(mount+"/issuer/%s/sign-intermediate", c.Parent) data = make(map[string]interface{}) data["csr"] = csr data["common_name"] = c.Name diff --git a/builtin/logical/pki/path_config_ca.go b/builtin/logical/pki/path_config_ca.go index 8cebe94454336..a7bdc002c4541 100644 --- a/builtin/logical/pki/path_config_ca.go +++ b/builtin/logical/pki/path_config_ca.go @@ -71,6 +71,31 @@ func pathConfigIssuers(b *backend) *framework.Path { } } +func pathReplaceRoot(b *backend) *framework.Path { + return &framework.Path{ + Pattern: "root/replace", + Fields: map[string]*framework.FieldSchema{ + "default": { + Type: framework.TypeString, + Description: `Reference (name or identifier) to the default issuer.`, + Default: "next", + }, + }, + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.pathCAIssuersWrite, + // Read more about why these flags are set in backend.go. + ForwardPerformanceStandby: true, + ForwardPerformanceSecondary: true, + }, + }, + + HelpSynopsis: pathConfigIssuersHelpSyn, + HelpDescription: pathConfigIssuersHelpDesc, + } +} + func (b *backend) pathCAIssuersRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { config, err := getIssuersConfig(ctx, req.Storage) if err != nil { @@ -128,6 +153,9 @@ This path allows configuration of issuer parameters. Presently, the "default" parameter controls which issuer is the default, accessible by the existing signing paths (/root/sign-intermediate, /root/sign-self-issued, /sign-verbatim, /sign/:role, and /issue/:role). + +The /root/replace path is aliased to this path, with default taking the +value of the issuer with the name "next", if it exists. ` func pathConfigKeys(b *backend) *framework.Path { diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index ed519b17adb83..e966afbe785fa 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -51,6 +51,13 @@ func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Req return logical.ErrorResponse("Can not create intermediate until migration has completed"), nil } + // Nasty hack :-) For cross-signing, we want to use the existing key, but + // this isn't _actually_ part of the path. Put it into the request + // parameters as if it was. + if req.Path == "intermediate/cross-sign" { + data.Raw["exported"] = "existing" + } + exported, format, role, errorResp := b.getGenerationParams(ctx, data, req.MountPoint) if errorResp != nil { return errorResp, nil diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 6039e2d41f75b..093c1181cc1e0 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -79,7 +79,7 @@ func buildPathSign(b *backend, pattern string) *framework.Path { } func pathIssuerSignVerbatim(b *backend) *framework.Path { - pattern := "issuers/" + framework.GenericNameRegex(issuerRefParam) + "/sign-verbatim" + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "/sign-verbatim" return buildPathIssuerSignVerbatim(b, pattern) } diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index e252170679c04..5eebbe296e3a3 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -16,6 +16,10 @@ func pathIssuerGenerateRoot(b *backend) *framework.Path { return buildPathGenerateRoot(b, "issuers/generate/root/"+framework.GenericNameRegex("exported")) } +func pathRotateRoot(b *backend) *framework.Path { + return buildPathGenerateRoot(b, "root/rotate/"+framework.GenericNameRegex("exported")) +} + func buildPathGenerateRoot(b *backend, pattern string) *framework.Path { ret := &framework.Path{ Pattern: pattern, @@ -44,6 +48,10 @@ func pathIssuerGenerateIntermediate(b *backend) *framework.Path { "issuers/generate/intermediate/"+framework.GenericNameRegex("exported")) } +func pathCrossSignIntermediate(b *backend) *framework.Path { + return buildPathGenerateIntermediate(b, "intermediate/cross-sign") +} + func buildPathGenerateIntermediate(b *backend, pattern string) *framework.Path { ret := &framework.Path{ Pattern: pattern, @@ -214,7 +222,7 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d config, err := getIssuersConfig(ctx, req.Storage) if err == nil && len(config.DefaultIssuerId) > 0 { // We can use the mapping above to check the issuer mapping. - if keyId, ok := issuerKeyMap[string(config.DefaultIssuerId)]; !ok || len(keyId) == 0 { + if keyId, ok := issuerKeyMap[string(config.DefaultIssuerId)]; ok && len(keyId) == 0 { msg := "The default issuer has no key associated with it. Some operations like issuing certificates and signing CRLs will be unavailable with the requested default issuer until a key is imported or the default issuer is changed." response.AddWarning(msg) b.Logger().Error(msg) diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index ba7c6f839a7a7..9d9c29d55deee 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -106,6 +106,16 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, if err != nil { return logical.ErrorResponse(err.Error()), nil } + // Handle the aliased path specifying the new issuer name as "next", but + // only do it if its not in use. + if strings.HasPrefix(req.Path, "root/rotate/") && len(issuerName) == 0 { + // err is nil when the issuer name is in use. + _, err = resolveIssuerReference(ctx, req.Storage, "next") + if err != nil { + issuerName = "next" + } + } + keyName, err := getKeyName(ctx, req.Storage, data) if err != nil { return logical.ErrorResponse(err.Error()), nil diff --git a/builtin/logical/pki/path_sign_issuers.go b/builtin/logical/pki/path_sign_issuers.go index ae45245097b10..96b7666065bb4 100644 --- a/builtin/logical/pki/path_sign_issuers.go +++ b/builtin/logical/pki/path_sign_issuers.go @@ -6,7 +6,7 @@ import ( ) func pathIssuerSignIntermediate(b *backend) *framework.Path { - pattern := "issuers/" + framework.GenericNameRegex(issuerRefParam) + "/sign-intermediate" + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "/sign-intermediate" return pathIssuerSignIntermediateRaw(b, pattern) } @@ -75,7 +75,7 @@ See the API documentation for more information about required parameters. ) func pathIssuerSignSelfIssued(b *backend) *framework.Path { - pattern := "issuers/" + framework.GenericNameRegex(issuerRefParam) + "/sign-self-issued" + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "/sign-self-issued" return buildPathIssuerSignSelfIssued(b, pattern) } From 06aed804725e3c5e0ec83482c0fe209580cce12a Mon Sep 17 00:00:00 2001 From: Steven Clark Date: Fri, 29 Apr 2022 10:35:36 -0400 Subject: [PATCH 64/76] Clean up various warnings within the PKI package (#15230) --- builtin/logical/pki/cert_util.go | 12 +----- builtin/logical/pki/path_config_ca.go | 38 +----------------- builtin/logical/pki/path_config_crl.go | 2 +- builtin/logical/pki/path_config_urls.go | 2 +- builtin/logical/pki/path_fetch.go | 2 +- builtin/logical/pki/path_fetch_keys.go | 2 +- builtin/logical/pki/path_revoke.go | 2 +- builtin/logical/pki/path_roles.go | 10 ++--- builtin/logical/pki/path_root.go | 2 +- builtin/logical/pki/path_tidy.go | 2 +- builtin/logical/pki/secret_certs.go | 2 +- builtin/logical/pki/storage.go | 10 ++--- builtin/logical/pki/storage_test.go | 52 ++++++++++++------------- builtin/logical/pki/util.go | 4 +- 14 files changed, 48 insertions(+), 94 deletions(-) diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index e0b8b816c7b83..cd05bdcd74944 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -64,19 +64,9 @@ var ( leftWildLabelRegex = regexp.MustCompile(`^(` + allWildRegex + `|` + startWildRegex + `|` + endWildRegex + `|` + middleWildRegex + `)$`) // OIDs for X.509 certificate extensions used below. - oidExtensionBasicConstraints = []int{2, 5, 29, 19} - oidExtensionSubjectAltName = []int{2, 5, 29, 17} + oidExtensionSubjectAltName = []int{2, 5, 29, 17} ) -func oidInExtensions(oid asn1.ObjectIdentifier, extensions []pkix.Extension) bool { - for _, e := range extensions { - if e.Id.Equal(oid) { - return true - } - } - return false -} - func getFormat(data *framework.FieldData) string { format := data.Get("format").(string) switch format { diff --git a/builtin/logical/pki/path_config_ca.go b/builtin/logical/pki/path_config_ca.go index a7bdc002c4541..a25d09f7adf5c 100644 --- a/builtin/logical/pki/path_config_ca.go +++ b/builtin/logical/pki/path_config_ca.go @@ -96,7 +96,7 @@ func pathReplaceRoot(b *backend) *framework.Path { } } -func (b *backend) pathCAIssuersRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathCAIssuersRead(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { config, err := getIssuersConfig(ctx, req.Storage) if err != nil { return logical.ErrorResponse("Error loading issuers configuration: " + err.Error()), nil @@ -186,7 +186,7 @@ func pathConfigKeys(b *backend) *framework.Path { } } -func (b *backend) pathKeyDefaultRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathKeyDefaultRead(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { config, err := getKeysConfig(ctx, req.Storage) if err != nil { return logical.ErrorResponse("Error loading keys configuration: " + err.Error()), nil @@ -229,37 +229,3 @@ This path allows configuration of key parameters. The "default" parameter controls which key is the default used by signing paths. ` - -const pathConfigCAGenerateHelpSyn = ` -Generate a new CA certificate and private key used for signing. -` - -const pathConfigCAGenerateHelpDesc = ` -This path generates a CA certificate and private key to be used for -credentials generated by this mount. The path can either -end in "internal" or "exported"; this controls whether the -unencrypted private key is exported after generation. This will -be your only chance to export the private key; for security reasons -it cannot be read or exported later. - -If the "type" option is set to "self-signed", the generated -certificate will be a self-signed root CA. Otherwise, this mount -will act as an intermediate CA; a CSR will be returned, to be signed -by your chosen CA (which could be another mount of this backend). -Note that the CRL path will be set to this mount's CRL path; if you -need further customization it is recommended that you create a CSR -separately and get it signed. Either way, use the "config/ca/set" -endpoint to load the signed certificate into Vault. -` - -const pathConfigCASignHelpSyn = ` -Generate a signed CA certificate from a CSR. -` - -const pathConfigCASignHelpDesc = ` -This path generates a CA certificate to be used for credentials -generated by the certificate's destination mount. - -Use the "config/ca/set" endpoint to load the signed certificate -into Vault another Vault mount. -` diff --git a/builtin/logical/pki/path_config_crl.go b/builtin/logical/pki/path_config_crl.go index 90dba1744a932..81ca2006a5693 100644 --- a/builtin/logical/pki/path_config_crl.go +++ b/builtin/logical/pki/path_config_crl.go @@ -66,7 +66,7 @@ func (b *backend) CRL(ctx context.Context, s logical.Storage) (*crlConfig, error return &result, nil } -func (b *backend) pathCRLRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathCRLRead(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { config, err := b.CRL(ctx, req.Storage) if err != nil { return nil, err diff --git a/builtin/logical/pki/path_config_urls.go b/builtin/logical/pki/path_config_urls.go index 87e26a52d99c7..2d18409971094 100644 --- a/builtin/logical/pki/path_config_urls.go +++ b/builtin/logical/pki/path_config_urls.go @@ -92,7 +92,7 @@ func writeURLs(ctx context.Context, req *logical.Request, entries *certutil.URLE return nil } -func (b *backend) pathReadURL(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathReadURL(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { entries, err := getURLs(ctx, req) if err != nil { return nil, err diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index 8b6d9d940cf7d..c3115a1f48e19 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -138,7 +138,7 @@ func pathFetchListCerts(b *backend) *framework.Path { } } -func (b *backend) pathFetchCertList(ctx context.Context, req *logical.Request, data *framework.FieldData) (response *logical.Response, retErr error) { +func (b *backend) pathFetchCertList(ctx context.Context, req *logical.Request, _ *framework.FieldData) (response *logical.Response, retErr error) { entries, err := req.Storage.List(ctx, "certs/") if err != nil { return nil, err diff --git a/builtin/logical/pki/path_fetch_keys.go b/builtin/logical/pki/path_fetch_keys.go index 21247ccf6955e..44820e0f37054 100644 --- a/builtin/logical/pki/path_fetch_keys.go +++ b/builtin/logical/pki/path_fetch_keys.go @@ -31,7 +31,7 @@ const ( their identifier and their name (if set).` ) -func (b *backend) pathListKeysHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathListKeysHandler(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { var responseKeys []string responseInfo := make(map[string]interface{}) diff --git a/builtin/logical/pki/path_revoke.go b/builtin/logical/pki/path_revoke.go index 47a3848ccb55b..910dc5126ccca 100644 --- a/builtin/logical/pki/path_revoke.go +++ b/builtin/logical/pki/path_revoke.go @@ -76,7 +76,7 @@ func (b *backend) pathRevokeWrite(ctx context.Context, req *logical.Request, dat return revokeCert(ctx, b, req, serial, false) } -func (b *backend) pathRotateCRLRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathRotateCRLRead(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { b.revokeStorageLock.RLock() defer b.revokeStorageLock.RUnlock() diff --git a/builtin/logical/pki/path_roles.go b/builtin/logical/pki/path_roles.go index cd327df83f586..60bccf0a6c213 100644 --- a/builtin/logical/pki/path_roles.go +++ b/builtin/logical/pki/path_roles.go @@ -600,7 +600,7 @@ func (b *backend) pathRoleRead(ctx context.Context, req *logical.Request, data * return resp, nil } -func (b *backend) pathRoleList(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathRoleList(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { entries, err := req.Storage.List(ctx, "role/") if err != nil { return nil, err @@ -710,16 +710,16 @@ func (b *backend) pathRoleCreate(ctx context.Context, req *logical.Request, data } } - allow_wildcard_certificates, present := data.GetOk("allow_wildcard_certificates") + allowWildcardCertificates, present := data.GetOk("allow_wildcard_certificates") if !present { // While not the most secure default, when AllowWildcardCertificates isn't // explicitly specified in the request, we automatically set it to true to // preserve compatibility with previous versions of Vault. - allow_wildcard_certificates = true + allowWildcardCertificates = true } - *entry.AllowWildcardCertificates = allow_wildcard_certificates.(bool) + *entry.AllowWildcardCertificates = allowWildcardCertificates.(bool) - // Ensure issuers ref is set to to a non-empty value. Note that we never + // Ensure issuers ref is set to a non-empty value. Note that we never // resolve the reference (to an issuerId) at role creation time; instead, // resolve it at use time. This allows values such as `default` or other // user-assigned names to "float" and change over time. diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 9d9c29d55deee..153477c09f96e 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -48,7 +48,7 @@ func pathDeleteRoot(b *backend) *framework.Path { return ret } -func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { issuers, err := listIssuers(ctx, req.Storage) if err != nil { return nil, err diff --git a/builtin/logical/pki/path_tidy.go b/builtin/logical/pki/path_tidy.go index 458ec30313789..11643336693c9 100644 --- a/builtin/logical/pki/path_tidy.go +++ b/builtin/logical/pki/path_tidy.go @@ -247,7 +247,7 @@ func (b *backend) pathTidyWrite(ctx context.Context, req *logical.Request, d *fr return logical.RespondWithStatusCode(resp, req, http.StatusAccepted) } -func (b *backend) pathTidyStatusRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathTidyStatusRead(_ context.Context, _ *logical.Request, _ *framework.FieldData) (*logical.Response, error) { // If this node is a performance secondary return an ErrReadOnly so that the request gets forwarded, // but only if the PKI backend is not a local mount. if b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary) && !b.System().LocalMount() { diff --git a/builtin/logical/pki/secret_certs.go b/builtin/logical/pki/secret_certs.go index bdbcd01ba1674..79fa410843cd0 100644 --- a/builtin/logical/pki/secret_certs.go +++ b/builtin/logical/pki/secret_certs.go @@ -35,7 +35,7 @@ reference`, } } -func (b *backend) secretCredsRevoke(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { +func (b *backend) secretCredsRevoke(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { if req.Secret == nil { return nil, fmt.Errorf("secret is nil in request") } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 98821778a7c6e..89175410b7896 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -132,13 +132,12 @@ func writeKey(ctx context.Context, s logical.Storage, key keyEntry) error { } func deleteKey(ctx context.Context, s logical.Storage, id keyID) (bool, error) { - wasDefault := false - config, err := getKeysConfig(ctx, s) if err != nil { - return wasDefault, err + return false, err } + wasDefault := false if config.DefaultKeyId == id { wasDefault = true config.DefaultKeyId = keyID("") @@ -368,13 +367,12 @@ func writeIssuer(ctx context.Context, s logical.Storage, issuer *issuerEntry) er } func deleteIssuer(ctx context.Context, s logical.Storage, id issuerID) (bool, error) { - wasDefault := false - config, err := getIssuersConfig(ctx, s) if err != nil { - return wasDefault, err + return false, err } + wasDefault := false if config.DefaultIssuerId == id { wasDefault = true config.DefaultIssuerId = issuerID("") diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index 39e32ef16a6e4..3ecf6a14e4056 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -103,36 +103,36 @@ func Test_KeysIssuerImport(t *testing.T) { issuer1.ID = "" issuer1.KeyID = "" - key1_ref1, existing, err := importKey(ctx, s, key1.PrivateKey, "key1") + key1Ref1, existing, err := importKey(ctx, s, key1.PrivateKey, "key1") require.NoError(t, err) require.False(t, existing) - require.Equal(t, strings.TrimSpace(key1.PrivateKey), strings.TrimSpace(key1_ref1.PrivateKey)) + require.Equal(t, strings.TrimSpace(key1.PrivateKey), strings.TrimSpace(key1Ref1.PrivateKey)) // Make sure if we attempt to re-import the same private key, no import/updates occur. - // So the existing flag should be set to true and we do not update the existing Name field. - key1_ref2, existing, err := importKey(ctx, s, key1.PrivateKey, "ignore-me") + // So the existing flag should be set to true, and we do not update the existing Name field. + key1Ref2, existing, err := importKey(ctx, s, key1.PrivateKey, "ignore-me") require.NoError(t, err) require.True(t, existing) - require.Equal(t, key1.PrivateKey, key1_ref1.PrivateKey) - require.Equal(t, key1_ref1.ID, key1_ref2.ID) - require.Equal(t, key1_ref1.Name, key1_ref2.Name) + require.Equal(t, key1.PrivateKey, key1Ref1.PrivateKey) + require.Equal(t, key1Ref1.ID, key1Ref2.ID) + require.Equal(t, key1Ref1.Name, key1Ref2.Name) - issuer1_ref1, existing, err := importIssuer(ctx, s, issuer1.Certificate, "issuer1") + issuer1Ref1, existing, err := importIssuer(ctx, s, issuer1.Certificate, "issuer1") require.NoError(t, err) require.False(t, existing) - require.Equal(t, strings.TrimSpace(issuer1.Certificate), strings.TrimSpace(issuer1_ref1.Certificate)) - require.Equal(t, key1_ref1.ID, issuer1_ref1.KeyID) - require.Equal(t, "issuer1", issuer1_ref1.Name) + require.Equal(t, strings.TrimSpace(issuer1.Certificate), strings.TrimSpace(issuer1Ref1.Certificate)) + require.Equal(t, key1Ref1.ID, issuer1Ref1.KeyID) + require.Equal(t, "issuer1", issuer1Ref1.Name) // Make sure if we attempt to re-import the same issuer, no import/updates occur. - // So the existing flag should be set to true and we do not update the existing Name field. - issuer1_ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate, "ignore-me") + // So the existing flag should be set to true, and we do not update the existing Name field. + issuer1Ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) - require.Equal(t, strings.TrimSpace(issuer1.Certificate), strings.TrimSpace(issuer1_ref1.Certificate)) - require.Equal(t, issuer1_ref1.ID, issuer1_ref2.ID) - require.Equal(t, key1_ref1.ID, issuer1_ref2.KeyID) - require.Equal(t, issuer1_ref1.Name, issuer1_ref2.Name) + require.Equal(t, strings.TrimSpace(issuer1.Certificate), strings.TrimSpace(issuer1Ref1.Certificate)) + require.Equal(t, issuer1Ref1.ID, issuer1Ref2.ID) + require.Equal(t, key1Ref1.ID, issuer1Ref2.KeyID) + require.Equal(t, issuer1Ref1.Name, issuer1Ref2.Name) err = writeIssuer(ctx, s, &issuer2) require.NoError(t, err) @@ -141,21 +141,21 @@ func Test_KeysIssuerImport(t *testing.T) { require.NoError(t, err) // Same double import tests as above, but make sure if the previous was created through writeIssuer not importIssuer. - issuer2_ref, existing, err := importIssuer(ctx, s, issuer2.Certificate, "ignore-me") + issuer2Ref, existing, err := importIssuer(ctx, s, issuer2.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) - require.Equal(t, strings.TrimSpace(issuer2.Certificate), strings.TrimSpace(issuer2_ref.Certificate)) - require.Equal(t, issuer2.ID, issuer2_ref.ID) - require.Equal(t, "", issuer2_ref.Name) - require.Equal(t, issuer2.KeyID, issuer2_ref.KeyID) + require.Equal(t, strings.TrimSpace(issuer2.Certificate), strings.TrimSpace(issuer2Ref.Certificate)) + require.Equal(t, issuer2.ID, issuer2Ref.ID) + require.Equal(t, "", issuer2Ref.Name) + require.Equal(t, issuer2.KeyID, issuer2Ref.KeyID) // Same double import tests as above, but make sure if the previous was created through writeKey not importKey. - key2_ref, existing, err := importKey(ctx, s, key2.PrivateKey, "ignore-me") + key2Ref, existing, err := importKey(ctx, s, key2.PrivateKey, "ignore-me") require.NoError(t, err) require.True(t, existing) - require.Equal(t, key2.PrivateKey, key2_ref.PrivateKey) - require.Equal(t, key2.ID, key2_ref.ID) - require.Equal(t, "", key2_ref.Name) + require.Equal(t, key2.PrivateKey, key2Ref.PrivateKey) + require.Equal(t, key2.ID, key2Ref.ID) + require.Equal(t, "", key2Ref.Name) } func genIssuerAndKey(t *testing.T, b *backend) (issuerEntry, keyEntry) { diff --git a/builtin/logical/pki/util.go b/builtin/logical/pki/util.go index 77d777dae605a..420fd2e24b877 100644 --- a/builtin/logical/pki/util.go +++ b/builtin/logical/pki/util.go @@ -169,12 +169,12 @@ func getKeyName(ctx context.Context, s logical.Storage, data *framework.FieldDat if !nameMatcher.MatchString(keyName) { return "", errutil.UserError{Err: "key name contained invalid characters"} } - key_id, err := resolveKeyReference(ctx, s, keyName) + keyId, err := resolveKeyReference(ctx, s, keyName) if err == nil { return "", errKeyNameInUse } - if err != nil && key_id != KeyRefNotFound { + if err != nil && keyId != KeyRefNotFound { return "", errutil.InternalError{Err: err.Error()} } } From 52087add7dfe8264c0d5d85a2051a5080555abe9 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Mon, 25 Apr 2022 14:57:50 -0400 Subject: [PATCH 65/76] Rebuild CRLs on secondary performance clusters post migration and on new/updated issuers - Hook into the backend invalidation function so that secondaries are notified of new/updated issuer or migrations occuring on the primary cluster. Upon notification schedule a CRL rebuild to take place upon the next process to read/update the CRL or within the periodic function if no request comes in. --- builtin/logical/pki/backend.go | 20 ++++++- builtin/logical/pki/cert_util.go | 5 +- builtin/logical/pki/cert_util_test.go | 6 +- builtin/logical/pki/crl_test.go | 49 ++++++++++++++- builtin/logical/pki/crl_util.go | 59 ++++++++++++++++++- builtin/logical/pki/path_config_crl.go | 2 +- builtin/logical/pki/path_fetch.go | 4 +- builtin/logical/pki/path_fetch_issuers.go | 4 ++ builtin/logical/pki/path_manage_issuers.go | 2 +- builtin/logical/pki/path_revoke.go | 2 +- builtin/logical/pki/path_root.go | 2 +- builtin/logical/pki/path_tidy.go | 2 +- builtin/logical/pki/storage_migrations.go | 4 +- .../logical/pki/storage_migrations_test.go | 2 +- 14 files changed, 142 insertions(+), 21 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 2f1702d7ecf47..424f00787dce3 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -158,6 +158,7 @@ func Backend(conf *logical.BackendConfig) *backend { BackendType: logical.TypeLogical, InitializeFunc: b.initialize, Invalidate: b.invalidate, + PeriodicFunc: b.periodicFunc, } b.crlLifetime = time.Hour * 72 @@ -167,6 +168,7 @@ func Backend(conf *logical.BackendConfig) *backend { b.pkiStorageVersion.Store(0) + b.crlBuilder = &crlBuilder{} return &b } @@ -182,6 +184,7 @@ type backend struct { tidyStatus *tidyStatus pkiStorageVersion atomic.Value + crlBuilder *crlBuilder } type ( @@ -285,7 +288,7 @@ func (b *backend) initialize(ctx context.Context, _ *logical.InitializationReque return nil } - if err := migrateStorage(ctx, b.storage, b.Logger()); err != nil { + if err := migrateStorage(ctx, b.crlBuilder, b.storage, b.Logger()); err != nil { b.Logger().Error("Error during migration of PKI mount: " + err.Error()) return err } @@ -320,8 +323,19 @@ func (b *backend) invalidate(ctx context.Context, key string) { switch { case strings.HasPrefix(key, legacyMigrationBundleLogKey): // This is for a secondary cluster to pick up that the migration has completed - // and reset its compatibility mode. + // and reset its compatibility mode and rebuild the CRL locally. b.updatePkiStorageVersion(ctx) + b.crlBuilder.requestRebuild() + case strings.HasPrefix(key, issuerPrefix): + // If an issuer has changed on the primary, we need to schedule an update of our CRL, + // the primary cluster would have done it already, but the CRL is cluster specific so + // force a rebuild of ours. + if !b.useLegacyBundleCaStorage() { + b.crlBuilder.requestRebuild() + } } - // FIXME: We need to hook into CRL generation here for issuer/bundle updates. +} + +func (b *backend) periodicFunc(ctx context.Context, request *logical.Request) error { + return b.crlBuilder.rebuildIfForced(ctx, b, request) } diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index cd05bdcd74944..0afd03d0fb2e7 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -156,7 +156,7 @@ func fetchCertBundle(ctx context.Context, b *backend, s logical.Storage, issuerR // // Support for fetching CA certificates was removed, due to the new issuers // changes. -func fetchCertBySerial(ctx context.Context, req *logical.Request, prefix, serial string) (*logical.StorageEntry, error) { +func fetchCertBySerial(ctx context.Context, b *backend, req *logical.Request, prefix, serial string) (*logical.StorageEntry, error) { var path, legacyPath string var err error var certEntry *logical.StorageEntry @@ -171,6 +171,9 @@ func fetchCertBySerial(ctx context.Context, req *logical.Request, prefix, serial legacyPath = "revoked/" + colonSerial path = "revoked/" + hyphenSerial case serial == "crl": + if err = b.crlBuilder.rebuildIfForced(ctx, b, req); err != nil { + return nil, err + } path, err = resolveIssuerCRLPath(ctx, req.Storage, defaultRef) if err != nil { return nil, err diff --git a/builtin/logical/pki/cert_util_test.go b/builtin/logical/pki/cert_util_test.go index d90963c079cfb..a631323724eaa 100644 --- a/builtin/logical/pki/cert_util_test.go +++ b/builtin/logical/pki/cert_util_test.go @@ -12,7 +12,7 @@ import ( ) func TestPki_FetchCertBySerial(t *testing.T) { - storage := &logical.InmemStorage{} + b, storage := createBackendWithStorage(t) cases := map[string]struct { Req *logical.Request @@ -46,7 +46,7 @@ func TestPki_FetchCertBySerial(t *testing.T) { t.Fatalf("error writing to storage on %s colon-based storage path: %s", name, err) } - certEntry, err := fetchCertBySerial(context.Background(), tc.Req, tc.Prefix, tc.Serial) + certEntry, err := fetchCertBySerial(context.Background(), b, tc.Req, tc.Prefix, tc.Serial) if err != nil { t.Fatalf("error on %s for colon-based storage path: %s", name, err) } @@ -81,7 +81,7 @@ func TestPki_FetchCertBySerial(t *testing.T) { t.Fatalf("error writing to storage on %s hyphen-based storage path: %s", name, err) } - certEntry, err := fetchCertBySerial(context.Background(), tc.Req, tc.Prefix, tc.Serial) + certEntry, err := fetchCertBySerial(context.Background(), b, tc.Req, tc.Prefix, tc.Serial) if err != nil || certEntry == nil { t.Fatalf("error on %s for hyphen-based storage path: err: %v, entry: %v", name, err, certEntry) } diff --git a/builtin/logical/pki/crl_test.go b/builtin/logical/pki/crl_test.go index 4f52050bba7bb..ea94fc07d5ab2 100644 --- a/builtin/logical/pki/crl_test.go +++ b/builtin/logical/pki/crl_test.go @@ -128,12 +128,57 @@ func TestBackend_CRL_EnableDisable(t *testing.T) { require.NotEqual(t, crlCreationTime1, crlCreationTime2) } +func TestBackend_Secondary_CRL_Rebuilding(t *testing.T) { + ctx := context.Background() + b, s := createBackendWithStorage(t) + + // Write out the issuer/key to storage without going through the api call as replication would. + bundle := genCertBundle(t, b) + issuer, _, err := writeCaBundle(ctx, s, bundle, "", "") + require.NoError(t, err) + + // Just to validate, before we call the invalidate function, make sure our CRL has not been generated + // and we get a nil response + resp := requestCrlFromBackend(t, s, b) + require.Nil(t, resp.Data["http_raw_body"]) + + // This should force any calls from now on to rebuild our CRL even a read + b.invalidate(ctx, issuerPrefix+issuer.ID.String()) + + // Perform the read operation again, we should have a valid CRL now... + resp = requestCrlFromBackend(t, s, b) + crl := parseCrlPemBytes(t, resp.Data["http_raw_body"].([]byte)) + require.Equal(t, 0, len(crl.RevokedCertificates)) +} + +func requestCrlFromBackend(t *testing.T, s logical.Storage, b *backend) *logical.Response { + crlReq := &logical.Request{ + Operation: logical.ReadOperation, + Path: "crl/pem", + Storage: s, + } + resp, err := b.HandleRequest(context.Background(), crlReq) + require.NoError(t, err, "crl req failed with an error") + require.NotNil(t, resp, "crl response was nil with no error") + require.False(t, resp.IsError(), "crl error response: %v", resp) + return resp +} + func getCrlCertificateList(t *testing.T, client *api.Client) pkix.TBSCertificateList { resp, err := client.Logical().ReadWithContext(context.Background(), "pki/cert/crl") - require.NoError(t, err) + require.NoError(t, err, "crl req failed with an error") + require.NotNil(t, resp, "crl response was nil with no error") crlPem := resp.Data["certificate"].(string) - certList, err := x509.ParseCRL([]byte(crlPem)) + return parseCrlPemString(t, crlPem) +} + +func parseCrlPemString(t *testing.T, crlPem string) pkix.TBSCertificateList { + return parseCrlPemBytes(t, []byte(crlPem)) +} + +func parseCrlPemBytes(t *testing.T, crlPem []byte) pkix.TBSCertificateList { + certList, err := x509.ParseCRL(crlPem) require.NoError(t, err) return certList.TBSCertList } diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index 1fc40efaaf701..1b7d0323f0934 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -10,6 +10,8 @@ import ( "fmt" "math/big" "strings" + "sync" + "sync/atomic" "time" "github.com/hashicorp/vault/sdk/helper/certutil" @@ -26,6 +28,57 @@ type revocationInfo struct { CertificateIssuer issuerID `json:"issuer_id"` } +// crlBuilder is gatekeeper for controlling various read/write operations to the storage of the CRL. +// The extra complexity arises from secondary performance clusters seeing various writes to its storage +// without the actual API calls. During the storage invalidation process, we do not have the required state +// to actually rebuild the CRLs, so we need to schedule it in a deferred fashion. This allows either +// read or write calls to perform the operation if required, or have the flag reset upon a write operation +type crlBuilder struct { + m sync.Mutex + forceRebuild uint32 +} + +const ( + _ignoreForceFlag = true + _enforceForceFlag = false +) + +// rebuildIfForced is to be called by readers or periodic functions that might need to trigger +// a refresh of the CRL before the read occurs. +func (cb *crlBuilder) rebuildIfForced(ctx context.Context, b *backend, request *logical.Request) error { + if atomic.LoadUint32(&cb.forceRebuild) == 1 { + return cb._doRebuild(ctx, b, request, true, _enforceForceFlag) + } + + return nil +} + +// rebuild is to be called by various write apis that know the CRL is to be updated and can be now. +func (cb *crlBuilder) rebuild(ctx context.Context, b *backend, request *logical.Request, forceNew bool) error { + return cb._doRebuild(ctx, b, request, forceNew, _ignoreForceFlag) +} + +// requestRebuild will schedule a rebuild of the CRL from the next reader or writer. +func (cb *crlBuilder) requestRebuild() { + cb.m.Lock() + defer cb.m.Unlock() + atomic.StoreUint32(&cb.forceRebuild, 1) +} + +func (cb *crlBuilder) _doRebuild(ctx context.Context, b *backend, request *logical.Request, forceNew bool, ignoreForceFlag bool) error { + cb.m.Lock() + defer cb.m.Unlock() + if cb.forceRebuild == 1 || ignoreForceFlag { + defer atomic.StoreUint32(&cb.forceRebuild, 0) + + // if forceRebuild was requested, that should force a complete rebuild even if requested not too by forceNew + myForceNew := cb.forceRebuild == 1 || forceNew + return buildCRLs(ctx, b, request, myForceNew) + } + + return nil +} + // Revokes a cert, and tries to be smart about error recovery func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial string, fromLease bool) (*logical.Response, error) { // As this backend is self-contained and this function does not hook into @@ -58,7 +111,7 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st alreadyRevoked := false var revInfo revocationInfo - revEntry, err := fetchCertBySerial(ctx, req, revokedPath, serial) + revEntry, err := fetchCertBySerial(ctx, b, req, revokedPath, serial) if err != nil { switch err.(type) { case errutil.UserError: @@ -77,7 +130,7 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st } if !alreadyRevoked { - certEntry, err := fetchCertBySerial(ctx, req, "certs/", serial) + certEntry, err := fetchCertBySerial(ctx, b, req, "certs/", serial) if err != nil { switch err.(type) { case errutil.UserError: @@ -133,7 +186,7 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st } } - crlErr := buildCRLs(ctx, b, req, false) + crlErr := b.crlBuilder.rebuild(ctx, b, req, false) if crlErr != nil { switch crlErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_config_crl.go b/builtin/logical/pki/path_config_crl.go index 81ca2006a5693..7555fc6833eb0 100644 --- a/builtin/logical/pki/path_config_crl.go +++ b/builtin/logical/pki/path_config_crl.go @@ -118,7 +118,7 @@ func (b *backend) pathCRLWrite(ctx context.Context, req *logical.Request, d *fra if oldDisable != config.Disable { // It wasn't disabled but now it is, rotate - crlErr := buildCRLs(ctx, b, req, true) + crlErr := b.crlBuilder.rebuild(ctx, b, req, true) if crlErr != nil { switch crlErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index c3115a1f48e19..6e30d6f9e7b97 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -248,7 +248,7 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data goto reply } - certEntry, funcErr = fetchCertBySerial(ctx, req, req.Path, serial) + certEntry, funcErr = fetchCertBySerial(ctx, b, req, req.Path, serial) if funcErr != nil { switch funcErr.(type) { case errutil.UserError: @@ -276,7 +276,7 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data certificate = []byte(strings.TrimSpace(string(pem.EncodeToMemory(&block)))) } - revokedEntry, funcErr = fetchCertBySerial(ctx, req, "revoked/", serial) + revokedEntry, funcErr = fetchCertBySerial(ctx, b, req, "revoked/", serial) if funcErr != nil { switch funcErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index b19ed30b0eae6..b1625a9d94d64 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -400,6 +400,10 @@ func (b *backend) pathGetIssuerCRL(ctx context.Context, req *logical.Request, da return logical.ErrorResponse("missing issuer reference"), nil } + if err := b.crlBuilder.rebuildIfForced(ctx, b, req); err != nil { + return nil, err + } + crlPath, err := resolveIssuerCRLPath(ctx, req.Storage, issuerName) if err != nil { return nil, err diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index 5eebbe296e3a3..fca56e3affb29 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -209,7 +209,7 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d } if len(createdIssuers) > 0 { - err := buildCRLs(ctx, b, req, true) + err := b.crlBuilder.rebuild(ctx, b, req, true) if err != nil { return nil, err } diff --git a/builtin/logical/pki/path_revoke.go b/builtin/logical/pki/path_revoke.go index 910dc5126ccca..52c5c63e7837c 100644 --- a/builtin/logical/pki/path_revoke.go +++ b/builtin/logical/pki/path_revoke.go @@ -80,7 +80,7 @@ func (b *backend) pathRotateCRLRead(ctx context.Context, req *logical.Request, _ b.revokeStorageLock.RLock() defer b.revokeStorageLock.RUnlock() - crlErr := buildCRLs(ctx, b, req, false) + crlErr := b.crlBuilder.rebuild(ctx, b, req, false) if crlErr != nil { switch crlErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 153477c09f96e..67ae2828223bf 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -205,7 +205,7 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, } // Build a fresh CRL - err = buildCRLs(ctx, b, req, true) + err = b.crlBuilder.rebuild(ctx, b, req, true) if err != nil { return nil, err } diff --git a/builtin/logical/pki/path_tidy.go b/builtin/logical/pki/path_tidy.go index 11643336693c9..3075fc5627144 100644 --- a/builtin/logical/pki/path_tidy.go +++ b/builtin/logical/pki/path_tidy.go @@ -225,7 +225,7 @@ func (b *backend) pathTidyWrite(ctx context.Context, req *logical.Request, d *fr } if rebuildCRL { - if err := buildCRLs(ctx, b, req, false); err != nil { + if err := b.crlBuilder.rebuild(ctx, b, req, false); err != nil { return err } } diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index 1796d34a39c42..143d5bc11a30c 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -64,7 +64,7 @@ func getMigrationInfo(ctx context.Context, s logical.Storage) (migrationInfo, er return migrationInfo, nil } -func migrateStorage(ctx context.Context, s logical.Storage, logger log.Logger) error { +func migrateStorage(ctx context.Context, cb *crlBuilder, s logical.Storage, logger log.Logger) error { migrationInfo, err := getMigrationInfo(ctx, s) if err != nil { return err @@ -88,6 +88,8 @@ func migrateStorage(ctx context.Context, s logical.Storage, logger log.Logger) e logger.Debug("No legacy CA certs found, no migration required.") } + cb.requestRebuild() + // We always want to write out this log entry as the secondary clusters leverage this path to wake up // if they were upgraded prior to the primary cluster's migration occurred. err = setLegacyBundleMigrationLog(ctx, s, &legacyBundleMigrationLog{ diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index d91b5d9c40b62..498cdfe5cb356 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -128,7 +128,7 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { require.Equal(t, &issuerConfigEntry{DefaultIssuerId: issuerId}, issuersConfig) // Make sure if we attempt to re-run the migration nothing happens... - err = migrateStorage(ctx, s, b.Logger()) + err = migrateStorage(ctx, b.crlBuilder, s, b.Logger()) require.NoError(t, err) logEntry2, err := getLegacyBundleMigrationLog(ctx, s) require.NoError(t, err) From db405142d72514346cec4f6f134b73753ddb1007 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Wed, 27 Apr 2022 15:01:10 -0400 Subject: [PATCH 66/76] Schedule rebuilding PKI CRLs on active nodes only - Address an issue that we were scheduling the rebuilding of a CRL on standby nodes, which would not be able to write to storage. - Fix an issue with standby nodes not correctly determining that a migration previously occurred. --- builtin/logical/pki/backend.go | 11 ++++++++--- builtin/logical/pki/ca_util.go | 8 ++++---- builtin/logical/pki/cert_util.go | 1 + builtin/logical/pki/crl_test.go | 2 +- builtin/logical/pki/crl_util.go | 14 ++++++++++++-- builtin/logical/pki/path_intermediate.go | 2 +- builtin/logical/pki/path_root.go | 2 +- builtin/logical/pki/storage_migrations.go | 17 +++++++++-------- builtin/logical/pki/storage_migrations_test.go | 4 ++-- builtin/logical/pki/storage_test.go | 18 +++++++++--------- 10 files changed, 48 insertions(+), 31 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 424f00787dce3..95b05edf6ad33 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -281,6 +281,9 @@ func (b *backend) metricsWrap(callType string, roleMode int, ofunc roleOperation // initialize is used to perform a possible PKI storage migration if needed func (b *backend) initialize(ctx context.Context, _ *logical.InitializationRequest) error { + // Load up our current pki storage state, no matter the host type we are on. + b.updatePkiStorageVersion(ctx) + // Early exit if not a primary cluster or performance secondary with a local mount. if b.System().ReplicationState().HasState(consts.ReplicationDRSecondary|consts.ReplicationPerformanceStandby) || (!b.System().LocalMount() && b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary)) { @@ -288,7 +291,7 @@ func (b *backend) initialize(ctx context.Context, _ *logical.InitializationReque return nil } - if err := migrateStorage(ctx, b.crlBuilder, b.storage, b.Logger()); err != nil { + if err := migrateStorage(ctx, b, b.storage); err != nil { b.Logger().Error("Error during migration of PKI mount: " + err.Error()) return err } @@ -325,13 +328,15 @@ func (b *backend) invalidate(ctx context.Context, key string) { // This is for a secondary cluster to pick up that the migration has completed // and reset its compatibility mode and rebuild the CRL locally. b.updatePkiStorageVersion(ctx) - b.crlBuilder.requestRebuild() + b.crlBuilder.requestRebuildOnActiveNode(b) case strings.HasPrefix(key, issuerPrefix): // If an issuer has changed on the primary, we need to schedule an update of our CRL, // the primary cluster would have done it already, but the CRL is cluster specific so // force a rebuild of ours. if !b.useLegacyBundleCaStorage() { - b.crlBuilder.requestRebuild() + b.crlBuilder.requestRebuildOnActiveNode(b) + } else { + b.Logger().Debug("Ignoring invalidation updates for issuer as the PKI migration has yet to complete.") } } } diff --git a/builtin/logical/pki/ca_util.go b/builtin/logical/pki/ca_util.go index 3265a2aae7090..5e81c7fe6031c 100644 --- a/builtin/logical/pki/ca_util.go +++ b/builtin/logical/pki/ca_util.go @@ -18,7 +18,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -func (b *backend) getGenerationParams(ctx context.Context, data *framework.FieldData, mountPoint string) (exported bool, format string, role *roleEntry, errorResp *logical.Response) { +func (b *backend) getGenerationParams(ctx context.Context, storage logical.Storage, data *framework.FieldData, mountPoint string) (exported bool, format string, role *roleEntry, errorResp *logical.Response) { exportedStr := data.Get("exported").(string) switch exportedStr { case "exported": @@ -39,7 +39,7 @@ func (b *backend) getGenerationParams(ctx context.Context, data *framework.Field return } - keyType, keyBits, err := getKeyTypeAndBitsForRole(ctx, b, data, mountPoint) + keyType, keyBits, err := getKeyTypeAndBitsForRole(ctx, b, storage, data, mountPoint) if err != nil { errorResp = logical.ErrorResponse(err.Error()) return @@ -114,7 +114,7 @@ func parseCABundle(ctx context.Context, b *backend, req *logical.Request, bundle return bundle.ToParsedCertBundle() } -func getKeyTypeAndBitsForRole(ctx context.Context, b *backend, data *framework.FieldData, mountPoint string) (string, int, error) { +func getKeyTypeAndBitsForRole(ctx context.Context, b *backend, storage logical.Storage, data *framework.FieldData, mountPoint string) (string, int, error) { exportedStr := data.Get("exported").(string) var keyType string var keyBits int @@ -146,7 +146,7 @@ func getKeyTypeAndBitsForRole(ctx context.Context, b *backend, data *framework.F } if existingKeyRequestedFromFieldData(data) { - existingPubKey, err := getExistingPublicKey(ctx, b.storage, data) + existingPubKey, err := getExistingPublicKey(ctx, storage, data) if err != nil { return "", 0, errors.New("failed to lookup public key from existing key: " + err.Error()) } diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index 0afd03d0fb2e7..093a7ce812ca0 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -139,6 +139,7 @@ func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRe func fetchCertBundle(ctx context.Context, b *backend, s logical.Storage, issuerRef string) (*issuerEntry, *certutil.CertBundle, error) { if b.useLegacyBundleCaStorage() { // We have not completed the migration so attempt to load the bundle from the legacy location + b.Logger().Info("Using legacy CA bundle") return getLegacyCertBundle(ctx, s) } diff --git a/builtin/logical/pki/crl_test.go b/builtin/logical/pki/crl_test.go index ea94fc07d5ab2..368a412ca8a60 100644 --- a/builtin/logical/pki/crl_test.go +++ b/builtin/logical/pki/crl_test.go @@ -133,7 +133,7 @@ func TestBackend_Secondary_CRL_Rebuilding(t *testing.T) { b, s := createBackendWithStorage(t) // Write out the issuer/key to storage without going through the api call as replication would. - bundle := genCertBundle(t, b) + bundle := genCertBundle(t, b, s) issuer, _, err := writeCaBundle(ctx, s, bundle, "", "") require.NoError(t, err) diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index 1b7d0323f0934..b395a6a7d820f 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -14,6 +14,8 @@ import ( "sync/atomic" "time" + "github.com/hashicorp/vault/sdk/helper/consts" + "github.com/hashicorp/vault/sdk/helper/certutil" "github.com/hashicorp/vault/sdk/helper/errutil" "github.com/hashicorp/vault/sdk/logical" @@ -58,8 +60,16 @@ func (cb *crlBuilder) rebuild(ctx context.Context, b *backend, request *logical. return cb._doRebuild(ctx, b, request, forceNew, _ignoreForceFlag) } -// requestRebuild will schedule a rebuild of the CRL from the next reader or writer. -func (cb *crlBuilder) requestRebuild() { +// requestRebuildOnActiveNode will schedule a rebuild of the CRL from the next read or write api call assuming we are the active node of a cluster +func (cb *crlBuilder) requestRebuildOnActiveNode(b *backend) { + // Only schedule us on active nodes, ignoring secondary nodes, the active can/should rebuild the CRL. + if b.System().ReplicationState().HasState(consts.ReplicationPerformanceStandby) || + b.System().ReplicationState().HasState(consts.ReplicationDRSecondary) { + b.Logger().Debug("Ignoring request to schedule a CRL rebuild, not on active node.") + return + } + + b.Logger().Info("Scheduling PKI CRL rebuild.") cb.m.Lock() defer cb.m.Unlock() atomic.StoreUint32(&cb.forceRebuild, 1) diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index e966afbe785fa..828900a72641f 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -58,7 +58,7 @@ func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Req data.Raw["exported"] = "existing" } - exported, format, role, errorResp := b.getGenerationParams(ctx, data, req.MountPoint) + exported, format, role, errorResp := b.getGenerationParams(ctx, req.Storage, data, req.MountPoint) if errorResp != nil { return errorResp, nil } diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 67ae2828223bf..1b9c43c463fee 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -91,7 +91,7 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, return logical.ErrorResponse("Can not create root CA until migration has completed"), nil } - exported, format, role, errorResp := b.getGenerationParams(ctx, data, req.MountPoint) + exported, format, role, errorResp := b.getGenerationParams(ctx, req.Storage, data, req.MountPoint) if errorResp != nil { return errorResp, nil } diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index 143d5bc11a30c..c5d17570b1e71 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "time" - log "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/sdk/helper/certutil" "github.com/hashicorp/vault/sdk/logical" ) @@ -64,7 +63,7 @@ func getMigrationInfo(ctx context.Context, s logical.Storage) (migrationInfo, er return migrationInfo, nil } -func migrateStorage(ctx context.Context, cb *crlBuilder, s logical.Storage, logger log.Logger) error { +func migrateStorage(ctx context.Context, b *backend, s logical.Storage) error { migrationInfo, err := getMigrationInfo(ctx, s) if err != nil { return err @@ -72,23 +71,25 @@ func migrateStorage(ctx context.Context, cb *crlBuilder, s logical.Storage, logg if !migrationInfo.isRequired { // No migration was deemed to be required. - logger.Debug("existing migration found and was considered valid, skipping migration.") + b.Logger().Debug("existing migration found and was considered valid, skipping migration.") return nil } - logger.Info("performing PKI migration to new keys/issuers layout") + b.Logger().Info("performing PKI migration to new keys/issuers layout") if migrationInfo.legacyBundle != nil { anIssuer, aKey, err := writeCaBundle(ctx, s, migrationInfo.legacyBundle, "current", "current") if err != nil { return err } - logger.Debug("Migration generated the following ids and set them as defaults", + b.Logger().Debug("Migration generated the following ids and set them as defaults", "issuer id", anIssuer.ID, "key id", aKey.ID) } else { - logger.Debug("No legacy CA certs found, no migration required.") + b.Logger().Debug("No legacy CA certs found, no migration required.") } - cb.requestRebuild() + // Since we do not have all the mount information available we must schedule + // the CRL to be rebuilt at a later time. + b.crlBuilder.requestRebuildOnActiveNode(b) // We always want to write out this log entry as the secondary clusters leverage this path to wake up // if they were upgraded prior to the primary cluster's migration occurred. @@ -101,7 +102,7 @@ func migrateStorage(ctx context.Context, cb *crlBuilder, s logical.Storage, logg return err } - logger.Info("successfully completed migration to new keys/issuers layout") + b.Logger().Info("successfully completed migration to new keys/issuers layout") return nil } diff --git a/builtin/logical/pki/storage_migrations_test.go b/builtin/logical/pki/storage_migrations_test.go index 498cdfe5cb356..cea35628590aa 100644 --- a/builtin/logical/pki/storage_migrations_test.go +++ b/builtin/logical/pki/storage_migrations_test.go @@ -64,7 +64,7 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { b.pkiStorageVersion.Store(0) require.True(t, b.useLegacyBundleCaStorage(), "pre migration we should have been told to use legacy storage.") - bundle := genCertBundle(t, b) + bundle := genCertBundle(t, b, s) json, err := logical.StorageEntryJSON(legacyCertBundlePath, bundle) require.NoError(t, err) err = s.Put(ctx, json) @@ -128,7 +128,7 @@ func Test_migrateStorageSimpleBundle(t *testing.T) { require.Equal(t, &issuerConfigEntry{DefaultIssuerId: issuerId}, issuersConfig) // Make sure if we attempt to re-run the migration nothing happens... - err = migrateStorage(ctx, b.crlBuilder, s, b.Logger()) + err = migrateStorage(ctx, b, s) require.NoError(t, err) logEntry2, err := getLegacyBundleMigrationLog(ctx, s) require.NoError(t, err) diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index 3ecf6a14e4056..8b752cd73d00a 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -50,8 +50,8 @@ func Test_ConfigsRoundTrip(t *testing.T) { func Test_IssuerRoundTrip(t *testing.T) { b, s := createBackendWithStorage(t) - issuer1, key1 := genIssuerAndKey(t, b) - issuer2, key2 := genIssuerAndKey(t, b) + issuer1, key1 := genIssuerAndKey(t, b, s) + issuer2, key2 := genIssuerAndKey(t, b, s) // We get an error when issuer id not found _, err := fetchIssuerById(ctx, s, issuer1.ID) @@ -94,8 +94,8 @@ func Test_IssuerRoundTrip(t *testing.T) { func Test_KeysIssuerImport(t *testing.T) { b, s := createBackendWithStorage(t) - issuer1, key1 := genIssuerAndKey(t, b) - issuer2, key2 := genIssuerAndKey(t, b) + issuer1, key1 := genIssuerAndKey(t, b, s) + issuer2, key2 := genIssuerAndKey(t, b, s) // Key 1 before Issuer 1; Issuer 2 before Key 2. // Remove KeyIDs from non-written entities before beginning. @@ -158,8 +158,8 @@ func Test_KeysIssuerImport(t *testing.T) { require.Equal(t, "", key2Ref.Name) } -func genIssuerAndKey(t *testing.T, b *backend) (issuerEntry, keyEntry) { - certBundle := genCertBundle(t, b) +func genIssuerAndKey(t *testing.T, b *backend, s logical.Storage) (issuerEntry, keyEntry) { + certBundle := genCertBundle(t, b, s) keyId := genKeyId() @@ -182,7 +182,7 @@ func genIssuerAndKey(t *testing.T, b *backend) (issuerEntry, keyEntry) { return pkiIssuer, pkiKey } -func genCertBundle(t *testing.T, b *backend) *certutil.CertBundle { +func genCertBundle(t *testing.T, b *backend, s logical.Storage) *certutil.CertBundle { // Pretty gross just to generate a cert bundle, but fields := addCACommonFields(map[string]*framework.FieldSchema{}) fields = addCAKeyGenerationFields(fields) @@ -195,14 +195,14 @@ func genCertBundle(t *testing.T, b *backend) *certutil.CertBundle { "ttl": 3600, }, } - _, _, role, respErr := b.getGenerationParams(ctx, apiData, "/pki") + _, _, role, respErr := b.getGenerationParams(ctx, s, apiData, "/pki") require.Nil(t, respErr) input := &inputBundle{ req: &logical.Request{ Operation: logical.UpdateOperation, Path: "issue/testrole", - Storage: b.storage, + Storage: s, }, apiData: apiData, role: role, From 9157497ef1367dccc7fc867ad735149111965594 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Thu, 28 Apr 2022 13:16:01 -0400 Subject: [PATCH 67/76] Return legacy CRL storage path when no migration has occurred. --- builtin/logical/pki/backend.go | 4 ++-- builtin/logical/pki/cert_util.go | 4 ++-- builtin/logical/pki/crl_util.go | 4 ++-- builtin/logical/pki/path_fetch_issuers.go | 2 +- builtin/logical/pki/storage.go | 6 +++++- builtin/logical/pki/storage_migrations.go | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 95b05edf6ad33..e463a89cb0e00 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -328,13 +328,13 @@ func (b *backend) invalidate(ctx context.Context, key string) { // This is for a secondary cluster to pick up that the migration has completed // and reset its compatibility mode and rebuild the CRL locally. b.updatePkiStorageVersion(ctx) - b.crlBuilder.requestRebuildOnActiveNode(b) + b.crlBuilder.requestRebuildIfActiveNode(b) case strings.HasPrefix(key, issuerPrefix): // If an issuer has changed on the primary, we need to schedule an update of our CRL, // the primary cluster would have done it already, but the CRL is cluster specific so // force a rebuild of ours. if !b.useLegacyBundleCaStorage() { - b.crlBuilder.requestRebuildOnActiveNode(b) + b.crlBuilder.requestRebuildIfActiveNode(b) } else { b.Logger().Debug("Ignoring invalidation updates for issuer as the PKI migration has yet to complete.") } diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index 093a7ce812ca0..11fa043be5d38 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -139,7 +139,7 @@ func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRe func fetchCertBundle(ctx context.Context, b *backend, s logical.Storage, issuerRef string) (*issuerEntry, *certutil.CertBundle, error) { if b.useLegacyBundleCaStorage() { // We have not completed the migration so attempt to load the bundle from the legacy location - b.Logger().Info("Using legacy CA bundle") + b.Logger().Info("Using legacy CA bundle as PKI migration has not completed.") return getLegacyCertBundle(ctx, s) } @@ -175,7 +175,7 @@ func fetchCertBySerial(ctx context.Context, b *backend, req *logical.Request, pr if err = b.crlBuilder.rebuildIfForced(ctx, b, req); err != nil { return nil, err } - path, err = resolveIssuerCRLPath(ctx, req.Storage, defaultRef) + path, err = resolveIssuerCRLPath(ctx, b, req.Storage, defaultRef) if err != nil { return nil, err } diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index b395a6a7d820f..28fb2f3d4f626 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -60,8 +60,8 @@ func (cb *crlBuilder) rebuild(ctx context.Context, b *backend, request *logical. return cb._doRebuild(ctx, b, request, forceNew, _ignoreForceFlag) } -// requestRebuildOnActiveNode will schedule a rebuild of the CRL from the next read or write api call assuming we are the active node of a cluster -func (cb *crlBuilder) requestRebuildOnActiveNode(b *backend) { +// requestRebuildIfActiveNode will schedule a rebuild of the CRL from the next read or write api call assuming we are the active node of a cluster +func (cb *crlBuilder) requestRebuildIfActiveNode(b *backend) { // Only schedule us on active nodes, ignoring secondary nodes, the active can/should rebuild the CRL. if b.System().ReplicationState().HasState(consts.ReplicationPerformanceStandby) || b.System().ReplicationState().HasState(consts.ReplicationDRSecondary) { diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index b1625a9d94d64..717c29a5c667f 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -404,7 +404,7 @@ func (b *backend) pathGetIssuerCRL(ctx context.Context, req *logical.Request, da return nil, err } - crlPath, err := resolveIssuerCRLPath(ctx, req.Storage, issuerName) + crlPath, err := resolveIssuerCRLPath(ctx, b, req.Storage, issuerName) if err != nil { return nil, err } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 89175410b7896..74812364a5e59 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -643,7 +643,11 @@ func resolveIssuerReference(ctx context.Context, s logical.Storage, reference st return IssuerRefNotFound, errutil.UserError{Err: fmt.Sprintf("unable to find PKI issuer for reference: %v", reference)} } -func resolveIssuerCRLPath(ctx context.Context, s logical.Storage, reference string) (string, error) { +func resolveIssuerCRLPath(ctx context.Context, b *backend, s logical.Storage, reference string) (string, error) { + if b.useLegacyBundleCaStorage() { + return "crl", nil + } + issuer, err := resolveIssuerReference(ctx, s, reference) if err != nil { return "crl", err diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index c5d17570b1e71..c5679b53c1d72 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -89,7 +89,7 @@ func migrateStorage(ctx context.Context, b *backend, s logical.Storage) error { // Since we do not have all the mount information available we must schedule // the CRL to be rebuilt at a later time. - b.crlBuilder.requestRebuildOnActiveNode(b) + b.crlBuilder.requestRebuildIfActiveNode(b) // We always want to write out this log entry as the secondary clusters leverage this path to wake up // if they were upgraded prior to the primary cluster's migration occurred. From 94169e53da46804f22cae4dad7a852b080954e88 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Fri, 29 Apr 2022 13:08:41 -0400 Subject: [PATCH 68/76] Handle issuer, keys locking (#15227) * Handle locking of issuers during writes We need a write lock around writes to ensure serialization of modifications. We use a single lock for both issuer and key updates, in part because certain operations (like deletion) will potentially affect both. Signed-off-by: Alexander Scheel * Add missing b.useLegacyBundleCaStorage guards Several locations needed to guard against early usage of the new issuers endpoint pre-migration. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 6 +++ builtin/logical/pki/path_config_ca.go | 10 +++++ builtin/logical/pki/path_fetch_issuers.go | 34 +++++++++++++++++ builtin/logical/pki/path_fetch_keys.go | 26 +++++++++++++ builtin/logical/pki/path_intermediate.go | 5 +++ builtin/logical/pki/path_manage_issuers.go | 5 +++ builtin/logical/pki/path_manage_keys.go | 10 +++++ builtin/logical/pki/path_root.go | 44 ++++++++++++++-------- 8 files changed, 124 insertions(+), 16 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index e463a89cb0e00..3629dab1572ec 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -185,6 +185,9 @@ type backend struct { pkiStorageVersion atomic.Value crlBuilder *crlBuilder + + // Write lock around issuers and keys. + issuersLock sync.RWMutex } type ( @@ -291,6 +294,9 @@ func (b *backend) initialize(ctx context.Context, _ *logical.InitializationReque return nil } + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + if err := migrateStorage(ctx, b, b.storage); err != nil { b.Logger().Error("Error during migration of PKI mount: " + err.Error()) return err diff --git a/builtin/logical/pki/path_config_ca.go b/builtin/logical/pki/path_config_ca.go index a25d09f7adf5c..50d1dc5959cb4 100644 --- a/builtin/logical/pki/path_config_ca.go +++ b/builtin/logical/pki/path_config_ca.go @@ -110,6 +110,11 @@ func (b *backend) pathCAIssuersRead(ctx context.Context, req *logical.Request, _ } func (b *backend) pathCAIssuersWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + newDefault := data.Get(defaultRef).(string) if len(newDefault) == 0 || newDefault == defaultRef { return logical.ErrorResponse("Invalid issuer specification; must be non-empty and can't be 'default'."), nil @@ -200,6 +205,11 @@ func (b *backend) pathKeyDefaultRead(ctx context.Context, req *logical.Request, } func (b *backend) pathKeyDefaultWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating keys here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + newDefault := data.Get(defaultRef).(string) if len(newDefault) == 0 || newDefault == defaultRef { return logical.ErrorResponse("Invalid key specification; must be non-empty and can't be 'default'."), nil diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 717c29a5c667f..3b4694e92f09c 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -27,6 +27,10 @@ func pathListIssuers(b *backend) *framework.Path { } func (b *backend) pathListIssuersHandler(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not list issuers until migration has completed"), nil + } + var responseKeys []string responseInfo := make(map[string]interface{}) @@ -122,6 +126,10 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data return b.pathGetRawIssuer(ctx, req, data) } + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not get issuer until migration has completed"), nil + } + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil @@ -159,6 +167,15 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data } func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not update issuer until migration has completed"), nil + } + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil @@ -277,6 +294,10 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da } func (b *backend) pathGetRawIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not get issuer until migration has completed"), nil + } + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil @@ -324,6 +345,15 @@ func (b *backend) pathGetRawIssuer(ctx context.Context, req *logical.Request, da } func (b *backend) pathDeleteIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not delete issuer until migration has completed"), nil + } + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil @@ -395,6 +425,10 @@ func buildPathGetIssuerCRL(b *backend, pattern string) *framework.Path { } func (b *backend) pathGetIssuerCRL(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not get issuer's CRL until migration has completed"), nil + } + issuerName := getIssuerRef(data) if len(issuerName) == 0 { return logical.ErrorResponse("missing issuer reference"), nil diff --git a/builtin/logical/pki/path_fetch_keys.go b/builtin/logical/pki/path_fetch_keys.go index 44820e0f37054..6bb58b0a3e8df 100644 --- a/builtin/logical/pki/path_fetch_keys.go +++ b/builtin/logical/pki/path_fetch_keys.go @@ -32,6 +32,10 @@ their identifier and their name (if set).` ) func (b *backend) pathListKeysHandler(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not list keys until migration has completed"), nil + } + var responseKeys []string responseInfo := make(map[string]interface{}) @@ -113,6 +117,10 @@ the certificate. ) func (b *backend) pathGetKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not get keys until migration has completed"), nil + } + keyRef := data.Get(keyRefParam).(string) if len(keyRef) == 0 { return logical.ErrorResponse("missing key reference"), nil @@ -141,6 +149,15 @@ func (b *backend) pathGetKeyHandler(ctx context.Context, req *logical.Request, d } func (b *backend) pathUpdateKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating keys here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not update keys until migration has completed"), nil + } + keyRef := data.Get(keyRefParam).(string) if len(keyRef) == 0 { return logical.ErrorResponse("missing key reference"), nil @@ -189,6 +206,15 @@ func (b *backend) pathUpdateKeyHandler(ctx context.Context, req *logical.Request } func (b *backend) pathDeleteKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + + if b.useLegacyBundleCaStorage() { + return logical.ErrorResponse("Can not delete keys until migration has completed"), nil + } + keyRef := data.Get(keyRefParam).(string) if len(keyRef) == 0 { return logical.ErrorResponse("missing key reference"), nil diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index 828900a72641f..a0b69e1a481c8 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -45,6 +45,11 @@ appended to the bundle.`, } func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + var err error if b.useLegacyBundleCaStorage() { diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index fca56e3affb29..0138755c8196e 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -107,6 +107,11 @@ secret-key (optional) and certificates.`, } func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + keysAllowed := strings.HasSuffix(req.Path, "bundle") || req.Path == "config/ca" if b.useLegacyBundleCaStorage() { diff --git a/builtin/logical/pki/path_manage_keys.go b/builtin/logical/pki/path_manage_keys.go index 3ee6f0e61c5c7..04195aae2ed52 100644 --- a/builtin/logical/pki/path_manage_keys.go +++ b/builtin/logical/pki/path_manage_keys.go @@ -49,6 +49,11 @@ const ( ) func (b *backend) pathGenerateKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + keyName, err := getKeyName(ctx, req.Storage, data) if err != nil { // Fail Immediately if Key Name is in Use, etc... return nil, err @@ -144,6 +149,11 @@ If name is set, that will be set on the key.` ) func (b *backend) pathImportKeyHandler(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + keyValueInterface, isOk := data.GetOk("pem_bundle") if !isOk { return logical.ErrorResponse("keyValue must be set"), nil diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 1b9c43c463fee..66ac5b0a26663 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -49,27 +49,34 @@ func pathDeleteRoot(b *backend) *framework.Path { } func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { - issuers, err := listIssuers(ctx, req.Storage) - if err != nil { - return nil, err - } - - keys, err := listKeys(ctx, req.Storage) - if err != nil { - return nil, err - } + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() - // Delete all issuers and keys. Ignore deleting the default since we're - // explicitly deleting everything. - for _, issuer := range issuers { - if _, err = deleteIssuer(ctx, req.Storage, issuer); err != nil { + if !b.useLegacyBundleCaStorage() { + issuers, err := listIssuers(ctx, req.Storage) + if err != nil { return nil, err } - } - for _, key := range keys { - if _, err = deleteKey(ctx, req.Storage, key); err != nil { + + keys, err := listKeys(ctx, req.Storage) + if err != nil { return nil, err } + + // Delete all issuers and keys. Ignore deleting the default since we're + // explicitly deleting everything. + for _, issuer := range issuers { + if _, err = deleteIssuer(ctx, req.Storage, issuer); err != nil { + return nil, err + } + } + for _, key := range keys { + if _, err = deleteKey(ctx, req.Storage, key); err != nil { + return nil, err + } + } } // Delete legacy CA bundle; but don't error if it doesn't exist. @@ -85,6 +92,11 @@ func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, _ } func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + // Since we're planning on updating issuers here, grab the lock so we've + // got a consistent view. + b.issuersLock.Lock() + defer b.issuersLock.Unlock() + var err error if b.useLegacyBundleCaStorage() { From 358e1b81ef43cbe9fc2906380501631852df2209 Mon Sep 17 00:00:00 2001 From: Steven Clark Date: Mon, 2 May 2022 16:58:55 -0400 Subject: [PATCH 69/76] Address PKI to properly support managed keys (#15256) * Address codebase for managed key fixes * Add proper public key comparison for better managed key support to importKeys * Remove redundant public key fetching within PKI importKeys --- builtin/logical/pki/backend.go | 2 + builtin/logical/pki/ca_util.go | 131 +++++++++++---------- builtin/logical/pki/crl_test.go | 3 +- builtin/logical/pki/fields.go | 1 + builtin/logical/pki/key_util.go | 126 ++++++++++++++++++++ builtin/logical/pki/managed_key_util.go | 16 ++- builtin/logical/pki/path_intermediate.go | 2 +- builtin/logical/pki/path_manage_issuers.go | 5 +- builtin/logical/pki/path_manage_keys.go | 98 ++++++++------- builtin/logical/pki/path_root.go | 2 +- builtin/logical/pki/storage.go | 111 +++++++++-------- builtin/logical/pki/storage_migrations.go | 3 +- builtin/logical/pki/storage_test.go | 17 +-- sdk/helper/certutil/helpers.go | 19 +-- 14 files changed, 359 insertions(+), 177 deletions(-) create mode 100644 builtin/logical/pki/key_util.go diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 3629dab1572ec..e168bb8ea94d2 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -165,6 +165,7 @@ func Backend(conf *logical.BackendConfig) *backend { b.tidyCASGuard = new(uint32) b.tidyStatus = &tidyStatus{state: tidyStatusInactive} b.storage = conf.StorageView + b.backendUuid = conf.BackendUUID b.pkiStorageVersion.Store(0) @@ -175,6 +176,7 @@ func Backend(conf *logical.BackendConfig) *backend { type backend struct { *framework.Backend + backendUuid string storage logical.Storage crlLifetime time.Duration revokeStorageLock sync.RWMutex diff --git a/builtin/logical/pki/ca_util.go b/builtin/logical/pki/ca_util.go index 5e81c7fe6031c..680139a56c137 100644 --- a/builtin/logical/pki/ca_util.go +++ b/builtin/logical/pki/ca_util.go @@ -5,7 +5,6 @@ import ( "crypto" "crypto/ecdsa" "crypto/rsa" - "encoding/pem" "errors" "fmt" "io" @@ -38,8 +37,8 @@ func (b *backend) getGenerationParams(ctx context.Context, storage logical.Stora `the "format" path parameter must be "pem", "der", or "pem_bundle"`) return } - - keyType, keyBits, err := getKeyTypeAndBitsForRole(ctx, b, storage, data, mountPoint) + mkc := newManagedKeyContext(ctx, b, mountPoint) + keyType, keyBits, err := getKeyTypeAndBitsForRole(mkc, storage, data) if err != nil { errorResp = logical.ErrorResponse(err.Error()) return @@ -77,7 +76,11 @@ func (b *backend) getGenerationParams(ctx context.Context, storage logical.Stora func generateCABundle(ctx context.Context, b *backend, input *inputBundle, data *certutil.CreationBundle, randomSource io.Reader) (*certutil.ParsedCertBundle, error) { if kmsRequested(input) { - return generateManagedKeyCABundle(ctx, b, input, data, randomSource) + keyId, err := getManagedKeyId(input.apiData) + if err != nil { + return nil, err + } + return generateManagedKeyCABundle(ctx, b, input, keyId, data, randomSource) } if existingKeyRequested(input) { @@ -85,7 +88,21 @@ func generateCABundle(ctx context.Context, b *backend, input *inputBundle, data if err != nil { return nil, err } - return certutil.CreateCertificateWithKeyGenerator(data, randomSource, existingGeneratePrivateKey(ctx, input.req.Storage, keyRef)) + + keyEntry, err := getExistingKeyFromRef(ctx, input.req.Storage, keyRef) + if err != nil { + return nil, err + } + + if keyEntry.isManagedPrivateKey() { + keyId, err := keyEntry.getManagedKeyUUID() + if err != nil { + return nil, err + } + return generateManagedKeyCABundle(ctx, b, input, keyId, data, randomSource) + } + + return certutil.CreateCertificateWithKeyGenerator(data, randomSource, existingKeyGeneratorFromBytes(keyEntry)) } return certutil.CreateCertificateWithRandomSource(data, randomSource) @@ -93,7 +110,12 @@ func generateCABundle(ctx context.Context, b *backend, input *inputBundle, data func generateCSRBundle(ctx context.Context, b *backend, input *inputBundle, data *certutil.CreationBundle, addBasicConstraints bool, randomSource io.Reader) (*certutil.ParsedCSRBundle, error) { if kmsRequested(input) { - return generateManagedKeyCSRBundle(ctx, b, input, data, addBasicConstraints, randomSource) + keyId, err := getManagedKeyId(input.apiData) + if err != nil { + return nil, err + } + + return generateManagedKeyCSRBundle(ctx, b, input, keyId, data, addBasicConstraints, randomSource) } if existingKeyRequested(input) { @@ -101,7 +123,21 @@ func generateCSRBundle(ctx context.Context, b *backend, input *inputBundle, data if err != nil { return nil, err } - return certutil.CreateCSRWithKeyGenerator(data, addBasicConstraints, randomSource, existingGeneratePrivateKey(ctx, input.req.Storage, keyRef)) + + key, err := getExistingKeyFromRef(ctx, input.req.Storage, keyRef) + if err != nil { + return nil, err + } + + if key.isManagedPrivateKey() { + keyId, err := key.getManagedKeyUUID() + if err != nil { + return nil, err + } + return generateManagedKeyCSRBundle(ctx, b, input, keyId, data, addBasicConstraints, randomSource) + } + + return certutil.CreateCSRWithKeyGenerator(data, addBasicConstraints, randomSource, existingKeyGeneratorFromBytes(key)) } return certutil.CreateCSRWithRandomSource(data, addBasicConstraints, randomSource) @@ -114,7 +150,7 @@ func parseCABundle(ctx context.Context, b *backend, req *logical.Request, bundle return bundle.ToParsedCertBundle() } -func getKeyTypeAndBitsForRole(ctx context.Context, b *backend, storage logical.Storage, data *framework.FieldData, mountPoint string) (string, int, error) { +func getKeyTypeAndBitsForRole(mkc managedKeyContext, storage logical.Storage, data *framework.FieldData) (string, int, error) { exportedStr := data.Get("exported").(string) var keyType string var keyBits int @@ -138,7 +174,12 @@ func getKeyTypeAndBitsForRole(ctx context.Context, b *backend, storage logical.S var pubKey crypto.PublicKey if kmsRequestedFromFieldData(data) { - pubKeyManagedKey, err := getManagedKeyPublicKey(ctx, b, data, mountPoint) + keyId, err := getManagedKeyId(data) + if err != nil { + return "", 0, errors.New("unable to determine managed key id" + err.Error()) + } + + pubKeyManagedKey, err := getManagedKeyPublicKey(mkc, keyId) if err != nil { return "", 0, errors.New("failed to lookup public key from managed key: " + err.Error()) } @@ -146,95 +187,67 @@ func getKeyTypeAndBitsForRole(ctx context.Context, b *backend, storage logical.S } if existingKeyRequestedFromFieldData(data) { - existingPubKey, err := getExistingPublicKey(ctx, storage, data) + existingPubKey, err := getExistingPublicKey(mkc, storage, data) if err != nil { return "", 0, errors.New("failed to lookup public key from existing key: " + err.Error()) } pubKey = existingPubKey } - return getKeyTypeAndBitsFromPublicKeyForRole(pubKey) + privateKeyType, keyBits, err := getKeyTypeAndBitsFromPublicKeyForRole(pubKey) + return string(privateKeyType), keyBits, err } -func getExistingPublicKey(ctx context.Context, s logical.Storage, data *framework.FieldData) (crypto.PublicKey, error) { +func getExistingPublicKey(mkc managedKeyContext, s logical.Storage, data *framework.FieldData) (crypto.PublicKey, error) { keyRef, err := getKeyRefWithErr(data) if err != nil { return nil, err } - id, err := resolveKeyReference(ctx, s, keyRef) - if err != nil { - return nil, err - } - key, err := fetchKeyById(ctx, s, id) + id, err := resolveKeyReference(mkc.ctx, s, keyRef) if err != nil { return nil, err } - signer, err := key.GetSigner() + key, err := fetchKeyById(mkc.ctx, s, id) if err != nil { return nil, err } - return signer.Public(), nil + return getPublicKey(mkc, key) } -func getKeyTypeAndBitsFromPublicKeyForRole(pubKey crypto.PublicKey) (string, int, error) { - var keyType string +func getKeyTypeAndBitsFromPublicKeyForRole(pubKey crypto.PublicKey) (certutil.PrivateKeyType, int, error) { + var keyType certutil.PrivateKeyType var keyBits int switch pubKey.(type) { case *rsa.PublicKey: - keyType = "rsa" + keyType = certutil.RSAPrivateKey keyBits = certutil.GetPublicKeySize(pubKey) case *ecdsa.PublicKey: - keyType = "ec" + keyType = certutil.ECPrivateKey case *ed25519.PublicKey: - keyType = "ed25519" + keyType = certutil.Ed25519PrivateKey default: - return "", 0, fmt.Errorf("unsupported public key: %#v", pubKey) + return certutil.UnknownPrivateKey, 0, fmt.Errorf("unsupported public key: %#v", pubKey) } return keyType, keyBits, nil } -func getManagedKeyPublicKey(ctx context.Context, b *backend, data *framework.FieldData, mountPoint string) (crypto.PublicKey, error) { - keyId, err := getManagedKeyId(data) +func getExistingKeyFromRef(ctx context.Context, s logical.Storage, keyRef string) (*keyEntry, error) { + keyId, err := resolveKeyReference(ctx, s, keyRef) if err != nil { - return nil, errors.New("unable to determine managed key id") - } - // Determine key type and key bits from the managed public key - var pubKey crypto.PublicKey - err = withManagedPKIKey(ctx, b, keyId, mountPoint, func(ctx context.Context, key logical.ManagedSigningKey) error { - pubKey, err = key.GetPublicKey(ctx) - if err != nil { - return err - } - - return nil - }) - if err != nil { - return nil, errors.New("failed to lookup public key from managed key: " + err.Error()) + return nil, err } - return pubKey, nil + return fetchKeyById(ctx, s, keyId) } -func existingGeneratePrivateKey(ctx context.Context, s logical.Storage, keyRef string) certutil.KeyGenerator { - return func(keyType string, keyBits int, container certutil.ParsedPrivateKeyContainer, _ io.Reader) error { - keyId, err := resolveKeyReference(ctx, s, keyRef) - if err != nil { - return err - } - key, err := fetchKeyById(ctx, s, keyId) +func existingKeyGeneratorFromBytes(key *keyEntry) certutil.KeyGenerator { + return func(_ string, _ int, container certutil.ParsedPrivateKeyContainer, _ io.Reader) error { + signer, _, pemBytes, err := getSignerFromKeyEntryBytes(key) if err != nil { return err } - signer, err := key.GetSigner() - if err != nil { - return err - } - privateKeyType := certutil.GetPrivateKeyTypeFromSigner(signer) - if privateKeyType == certutil.UnknownPrivateKey { - return errors.New("unknown private key type loaded from key id: " + keyId.String()) - } - blk, _ := pem.Decode([]byte(key.PrivateKey)) - container.SetParsedPrivateKey(signer, privateKeyType, blk.Bytes) + + container.SetParsedPrivateKey(signer, key.PrivateKeyType, pemBytes.Bytes) return nil } } diff --git a/builtin/logical/pki/crl_test.go b/builtin/logical/pki/crl_test.go index 368a412ca8a60..20ed3403e15de 100644 --- a/builtin/logical/pki/crl_test.go +++ b/builtin/logical/pki/crl_test.go @@ -131,10 +131,11 @@ func TestBackend_CRL_EnableDisable(t *testing.T) { func TestBackend_Secondary_CRL_Rebuilding(t *testing.T) { ctx := context.Background() b, s := createBackendWithStorage(t) + mkc := newManagedKeyContext(ctx, b, "test") // Write out the issuer/key to storage without going through the api call as replication would. bundle := genCertBundle(t, b, s) - issuer, _, err := writeCaBundle(ctx, s, bundle, "", "") + issuer, _, err := writeCaBundle(mkc, s, bundle, "", "") require.NoError(t, err) // Just to validate, before we call the invalidate function, make sure our CRL has not been generated diff --git a/builtin/logical/pki/fields.go b/builtin/logical/pki/fields.go index 45c329bd8acf5..b53ddea366645 100644 --- a/builtin/logical/pki/fields.go +++ b/builtin/logical/pki/fields.go @@ -8,6 +8,7 @@ const ( keyRefParam = "key_ref" keyIdParam = "key_id" keyTypeParam = "key_type" + keyBitsParam = "key_bits" ) // addIssueAndSignCommonFields adds fields common to both CA and non-CA issuing diff --git a/builtin/logical/pki/key_util.go b/builtin/logical/pki/key_util.go new file mode 100644 index 0000000000000..3aa7d670dbb13 --- /dev/null +++ b/builtin/logical/pki/key_util.go @@ -0,0 +1,126 @@ +package pki + +import ( + "context" + "crypto" + "encoding/pem" + "errors" + "fmt" + + "github.com/hashicorp/vault/sdk/helper/certutil" + "github.com/hashicorp/vault/sdk/helper/errutil" + "github.com/hashicorp/vault/sdk/logical" +) + +type managedKeyContext struct { + ctx context.Context + b *backend + mountPoint string +} + +func newManagedKeyContext(ctx context.Context, b *backend, mountPoint string) managedKeyContext { + return managedKeyContext{ + ctx: ctx, + b: b, + mountPoint: mountPoint, + } +} + +func comparePublicKey(ctx managedKeyContext, key *keyEntry, publicKey crypto.PublicKey) (bool, error) { + publicKeyForKeyEntry, err := getPublicKey(ctx, key) + if err != nil { + return false, err + } + + return certutil.ComparePublicKeysAndType(publicKeyForKeyEntry, publicKey) +} + +func getPublicKey(mkc managedKeyContext, key *keyEntry) (crypto.PublicKey, error) { + if key.PrivateKeyType == certutil.ManagedPrivateKey { + keyId, err := extractManagedKeyId([]byte(key.PrivateKey)) + if err != nil { + return nil, err + } + return getManagedKeyPublicKey(mkc, keyId) + } + + signer, _, _, err := getSignerFromKeyEntryBytes(key) + if err != nil { + return nil, err + } + return signer.Public(), nil +} + +func getSignerFromKeyEntryBytes(key *keyEntry) (crypto.Signer, certutil.BlockType, *pem.Block, error) { + if key.PrivateKeyType == certutil.UnknownPrivateKey { + return nil, certutil.UnknownBlock, nil, errutil.InternalError{Err: fmt.Sprintf("unsupported unknown private key type for key: %s (%s)", key.ID, key.Name)} + } + + if key.PrivateKeyType == certutil.ManagedPrivateKey { + return nil, certutil.UnknownBlock, nil, errutil.InternalError{Err: fmt.Sprintf("can not get a signer from a managed key: %s (%s)", key.ID, key.Name)} + } + + bytes, blockType, blk, err := getSignerFromBytes([]byte(key.PrivateKey)) + if err != nil { + return nil, certutil.UnknownBlock, nil, errutil.InternalError{Err: fmt.Sprintf("failed parsing key entry bytes for key id: %s (%s): %s", key.ID, key.Name, err.Error())} + } + + return bytes, blockType, blk, nil +} + +func getSignerFromBytes(keyBytes []byte) (crypto.Signer, certutil.BlockType, *pem.Block, error) { + pemBlock, _ := pem.Decode(keyBytes) + if pemBlock == nil { + return nil, certutil.UnknownBlock, pemBlock, errutil.InternalError{Err: "no data found in PEM block"} + } + + signer, blk, err := certutil.ParseDERKey(pemBlock.Bytes) + if err != nil { + return nil, certutil.UnknownBlock, pemBlock, errutil.InternalError{Err: fmt.Sprintf("failed to parse PEM block: %s", err.Error())} + } + return signer, blk, pemBlock, nil +} + +func getManagedKeyPublicKey(mkc managedKeyContext, keyId managedKeyId) (crypto.PublicKey, error) { + // Determine key type and key bits from the managed public key + var pubKey crypto.PublicKey + err := withManagedPKIKey(mkc.ctx, mkc.b, keyId, mkc.mountPoint, func(ctx context.Context, key logical.ManagedSigningKey) error { + var myErr error + pubKey, myErr = key.GetPublicKey(ctx) + if myErr != nil { + return myErr + } + + return nil + }) + if err != nil { + return nil, errors.New("failed to lookup public key from managed key: " + err.Error()) + } + return pubKey, nil +} + +func getPublicKeyFromBytes(keyBytes []byte) (crypto.PublicKey, error) { + signer, _, _, err := getSignerFromBytes(keyBytes) + if err != nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("failed parsing key bytes: %s", err.Error())} + } + + return signer.Public(), nil +} + +func importKeyFromBytes(mkc managedKeyContext, s logical.Storage, keyValue string, keyName string) (*keyEntry, bool, error) { + signer, _, _, err := getSignerFromBytes([]byte(keyValue)) + if err != nil { + return nil, false, err + } + privateKeyType := certutil.GetPrivateKeyTypeFromSigner(signer) + if privateKeyType == certutil.UnknownPrivateKey { + return nil, false, errors.New("unsupported private key type within pem bundle") + } + + key, existed, err := importKey(mkc, s, keyValue, keyName, privateKeyType) + if err != nil { + return nil, false, err + } + return key, existed, nil +} diff --git a/builtin/logical/pki/managed_key_util.go b/builtin/logical/pki/managed_key_util.go index 45d80d643dcd7..a69ac056bfc45 100644 --- a/builtin/logical/pki/managed_key_util.go +++ b/builtin/logical/pki/managed_key_util.go @@ -13,18 +13,26 @@ import ( var errEntOnly = errors.New("managed keys are supported within enterprise edition only") -func generateManagedKeyCABundle(_ context.Context, _ *backend, _ *inputBundle, _ *certutil.CreationBundle, _ io.Reader) (*certutil.ParsedCertBundle, error) { +func generateManagedKeyCABundle(ctx context.Context, b *backend, input *inputBundle, keyId managedKeyId, data *certutil.CreationBundle, randomSource io.Reader) (bundle *certutil.ParsedCertBundle, err error) { return nil, errEntOnly } -func generateManagedKeyCSRBundle(_ context.Context, _ *backend, _ *inputBundle, _ *certutil.CreationBundle, _ bool, _ io.Reader) (*certutil.ParsedCSRBundle, error) { +func generateManagedKeyCSRBundle(ctx context.Context, b *backend, input *inputBundle, keyId managedKeyId, data *certutil.CreationBundle, addBasicConstraints bool, randomSource io.Reader) (bundle *certutil.ParsedCSRBundle, err error) { return nil, errEntOnly } -func parseManagedKeyCABundle(_ context.Context, _ *backend, _ *logical.Request, _ *certutil.CertBundle) (*certutil.ParsedCertBundle, error) { +func parseManagedKeyCABundle(ctx context.Context, b *backend, req *logical.Request, bundle *certutil.CertBundle) (*certutil.ParsedCertBundle, error) { return nil, errEntOnly } -func withManagedPKIKey(_ context.Context, _ *backend, _ managedKeyId, _ string, _ logical.ManagedSigningKeyConsumer) error { +func withManagedPKIKey(ctx context.Context, b *backend, keyId managedKeyId, mountPoint string, f logical.ManagedSigningKeyConsumer) error { return errEntOnly } + +func extractManagedKeyId(privateKeyBytes []byte) (UUIDKey, error) { + return "", errEntOnly +} + +func createKmsKeyBundle(mkc managedKeyContext, keyId managedKeyId) (certutil.KeyBundle, certutil.PrivateKeyType, error) { + return certutil.KeyBundle{}, certutil.UnknownPrivateKey, errEntOnly +} diff --git a/builtin/logical/pki/path_intermediate.go b/builtin/logical/pki/path_intermediate.go index a0b69e1a481c8..bf0b416bf4a4a 100644 --- a/builtin/logical/pki/path_intermediate.go +++ b/builtin/logical/pki/path_intermediate.go @@ -130,7 +130,7 @@ func (b *backend) pathGenerateIntermediate(ctx context.Context, req *logical.Req } } - myKey, _, err := importKey(ctx, req.Storage, csrb.PrivateKey, keyName) + myKey, _, err := importKey(newManagedKeyContext(ctx, b, req.MountPoint), req.Storage, csrb.PrivateKey, keyName, csrb.PrivateKeyType) if err != nil { return nil, err } diff --git a/builtin/logical/pki/path_manage_issuers.go b/builtin/logical/pki/path_manage_issuers.go index 0138755c8196e..ea7abcfa08c1f 100644 --- a/builtin/logical/pki/path_manage_issuers.go +++ b/builtin/logical/pki/path_manage_issuers.go @@ -181,9 +181,10 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d return logical.ErrorResponse("private keys found in the PEM bundle but not allowed by the path; use /issuers/import/bundle"), nil } + mkc := newManagedKeyContext(ctx, b, req.MountPoint) for keyIndex, keyPem := range keys { // Handle import of private key. - key, existing, err := importKey(ctx, req.Storage, keyPem, "") + key, existing, err := importKeyFromBytes(mkc, req.Storage, keyPem, "") if err != nil { return logical.ErrorResponse(fmt.Sprintf("Error parsing key %v: %v", keyIndex, err)), nil } @@ -194,7 +195,7 @@ func (b *backend) pathImportIssuers(ctx context.Context, req *logical.Request, d } for certIndex, certPem := range issuers { - cert, existing, err := importIssuer(ctx, req.Storage, certPem, "") + cert, existing, err := importIssuer(mkc, req.Storage, certPem, "") if err != nil { return logical.ErrorResponse(fmt.Sprintf("Error parsing issuer %v: %v\n%v", certIndex, err, certPem)), nil } diff --git a/builtin/logical/pki/path_manage_keys.go b/builtin/logical/pki/path_manage_keys.go index 04195aae2ed52..cce0fde52bd15 100644 --- a/builtin/logical/pki/path_manage_keys.go +++ b/builtin/logical/pki/path_manage_keys.go @@ -11,7 +11,7 @@ import ( func pathGenerateKey(b *backend) *framework.Path { return &framework.Path{ - Pattern: "keys/generate/(internal|exported)", + Pattern: "keys/generate/(internal|exported|kms)", Fields: map[string]*framework.FieldSchema{ keyNameParam: { @@ -23,15 +23,27 @@ func pathGenerateKey(b *backend) *framework.Path { Default: "rsa", Description: `Type of the secret key to generate`, }, - "key_bits": { + keyBitsParam: { Type: framework.TypeInt, Default: 2048, Description: `Type of the secret key to generate`, }, + "managed_key_name": { + Type: framework.TypeString, + Description: `The name of the managed key to use when the exported +type is kms. When kms type is the key type, this field or managed_key_id +is required. Ignored for other types.`, + }, + "managed_key_id": { + Type: framework.TypeString, + Description: `The name of the managed key to use when the exported +type is kms. When kms type is the key type, this field or managed_key_name +is required. Ignored for other types.`, + }, }, Operations: map[logical.Operation]framework.OperationHandler{ - logical.CreateOperation: &framework.PathOperation{ + logical.UpdateOperation: &framework.PathOperation{ Callback: b.pathGenerateKeyHandler, ForwardPerformanceStandby: true, ForwardPerformanceSecondary: true, @@ -58,60 +70,60 @@ func (b *backend) pathGenerateKeyHandler(ctx context.Context, req *logical.Reque if err != nil { // Fail Immediately if Key Name is in Use, etc... return nil, err } - keyType := data.Get(keyTypeParam).(string) - keyBits := data.Get("key_bits").(int) + mkc := newManagedKeyContext(ctx, b, req.MountPoint) + exportPrivateKey := false + var keyBundle certutil.KeyBundle + var actualPrivateKeyType certutil.PrivateKeyType switch { + case strings.HasSuffix(req.Path, "/exported"): + exportPrivateKey = true + fallthrough case strings.HasSuffix(req.Path, "/internal"): + keyType := data.Get(keyTypeParam).(string) + keyBits := data.Get(keyBitsParam).(int) + // Internal key generation, stored in storage - keyBundle, err := certutil.GetKeyBundleFromKeyGenerator(keyType, keyBits, nil) - if err != nil { - return nil, err - } - privateKeyPemString, err := keyBundle.ToPrivateKeyPemString() - if err != nil { - return nil, err - } - key, _, err := importKey(ctx, req.Storage, privateKeyPemString, keyName) - if err != nil { - return nil, err - } - resp := logical.Response{ - Data: map[string]interface{}{ - keyIdParam: key.ID, - keyNameParam: key.Name, - keyTypeParam: key.PrivateKeyType, - }, - } - return &resp, nil - case strings.HasSuffix(req.Path, "/exported"): - // Same as internal key generation but we return the generated key - keyBundle, err := certutil.GetKeyBundleFromKeyGenerator(keyType, keyBits, nil) + keyBundle, err = certutil.CreateKeyBundle(keyType, keyBits, b.GetRandomReader()) if err != nil { return nil, err } - privateKeyPemString, err := keyBundle.ToPrivateKeyPemString() + + actualPrivateKeyType = keyBundle.PrivateKeyType + case strings.HasSuffix(req.Path, "/kms"): + keyId, err := getManagedKeyId(data) if err != nil { return nil, err } - key, _, err := importKey(ctx, req.Storage, privateKeyPemString, keyName) + + keyBundle, actualPrivateKeyType, err = createKmsKeyBundle(mkc, keyId) if err != nil { return nil, err } - resp := logical.Response{ - Data: map[string]interface{}{ - keyIdParam: key.ID, - keyNameParam: key.Name, - keyTypeParam: key.PrivateKeyType, - "private_key": privateKeyPemString, - }, - } - return &resp, nil - case strings.HasSuffix(req.Path, "/kms"): - return nil, errEntOnly default: return logical.ErrorResponse("Unknown type of key to generate"), nil } + + privateKeyPemString, err := keyBundle.ToPrivateKeyPemString() + if err != nil { + return nil, err + } + + key, _, err := importKey(mkc, req.Storage, privateKeyPemString, keyName, keyBundle.PrivateKeyType) + if err != nil { + return nil, err + } + responseData := map[string]interface{}{ + keyIdParam: key.ID, + keyNameParam: key.Name, + keyTypeParam: string(actualPrivateKeyType), + } + if exportPrivateKey { + responseData["private_key"] = privateKeyPemString + } + return &logical.Response{ + Data: responseData, + }, nil } func pathImportKey(b *backend) *framework.Path { @@ -161,7 +173,8 @@ func (b *backend) pathImportKeyHandler(ctx context.Context, req *logical.Request keyValue := keyValueInterface.(string) keyName := data.Get(keyNameParam).(string) - key, existed, err := importKey(ctx, req.Storage, keyValue, keyName) + mkc := newManagedKeyContext(ctx, b, req.MountPoint) + key, existed, err := importKeyFromBytes(mkc, req.Storage, keyValue, keyName) if err != nil { return logical.ErrorResponse(err.Error()), nil } @@ -171,7 +184,6 @@ func (b *backend) pathImportKeyHandler(ctx context.Context, req *logical.Request keyIdParam: key.ID, keyNameParam: key.Name, keyTypeParam: key.PrivateKeyType, - "backing": "", // This would show up as "Managed" in "type" }, } diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 66ac5b0a26663..121c7c4b5b39f 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -199,7 +199,7 @@ func (b *backend) pathCAGenerateRoot(ctx context.Context, req *logical.Request, } // Store it as the CA bundle - myIssuer, myKey, err := writeCaBundle(ctx, req.Storage, cb, issuerName, keyName) + myIssuer, myKey, err := writeCaBundle(newManagedKeyContext(ctx, b, req.MountPoint), req.Storage, cb, issuerName, keyName) if err != nil { return nil, err } diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 74812364a5e59..9908bb75fccc7 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -55,6 +55,17 @@ type keyEntry struct { PrivateKey string `json:"private_key" structs:"private_key" mapstructure:"private_key"` } +func (e keyEntry) getManagedKeyUUID() (UUIDKey, error) { + if !e.isManagedPrivateKey() { + return "", errutil.InternalError{Err: "getManagedKeyId called on a key id %s (%s) "} + } + return extractManagedKeyId([]byte(e.PrivateKey)) +} + +func (e keyEntry) isManagedPrivateKey() bool { + return e.PrivateKeyType == certutil.ManagedPrivateKey +} + type issuerEntry struct { ID issuerID `json:"id" structs:"id" mapstructure:"id"` Name string `json:"name" structs:"name" mapstructure:"name"` @@ -79,11 +90,6 @@ type issuerConfigEntry struct { DefaultIssuerId issuerID `json:"default" structs:"default" mapstructure:"default"` } -func (k keyEntry) GetSigner() (crypto.Signer, error) { - signer, _, err := certutil.ParsePEMKey(k.PrivateKey) - return signer, err -} - func listKeys(ctx context.Context, s logical.Storage) ([]keyID, error) { strList, err := s.List(ctx, keyPrefix) if err != nil { @@ -149,7 +155,7 @@ func deleteKey(ctx context.Context, s logical.Storage, id keyID) (bool, error) { return wasDefault, s.Delete(ctx, keyPrefix+id.String()) } -func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName string) (*keyEntry, bool, error) { +func importKey(mkc managedKeyContext, s logical.Storage, keyValue string, keyName string, keyType certutil.PrivateKeyType) (*keyEntry, bool, error) { // importKey imports the specified PEM-format key (from keyValue) into // the new PKI storage format. The first return field is a reference to // the new key; the second is whether or not the key already existed @@ -164,27 +170,40 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName // Before we can import a known key, we first need to know if the key // exists in storage already. This means iterating through all known // keys and comparing their private value against this value. - knownKeys, err := listKeys(ctx, s) + knownKeys, err := listKeys(mkc.ctx, s) if err != nil { return nil, false, err } - // Before we return below, we need to iterate over _all_ issuers and see if - // one of them has a missing KeyId link, and if so, point it back to - // ourselves. We fetch the list of issuers up front, even when don't need - // it, to give ourselves a better chance of succeeding below. - knownIssuers, err := listIssuers(ctx, s) - if err != nil { - return nil, false, err + // Get our public key from the current inbound key, to compare against all the other keys. + var pkForImportingKey crypto.PublicKey + if keyType == certutil.ManagedPrivateKey { + managedKeyUUID, err := extractManagedKeyId([]byte(keyValue)) + if err != nil { + return nil, false, errutil.InternalError{Err: fmt.Sprintf("failed extracting managed key uuid from key: %v", err)} + } + pkForImportingKey, err = getManagedKeyPublicKey(mkc, managedKeyUUID) + if err != nil { + return nil, false, err + } + } else { + pkForImportingKey, err = getPublicKeyFromBytes([]byte(keyValue)) + if err != nil { + return nil, false, err + } } for _, identifier := range knownKeys { - existingKey, err := fetchKeyById(ctx, s, identifier) + existingKey, err := fetchKeyById(mkc.ctx, s, identifier) + if err != nil { + return nil, false, err + } + areEqual, err := comparePublicKey(mkc, existingKey, pkForImportingKey) if err != nil { return nil, false, err } - if existingKey.PrivateKey == keyValue { + if areEqual { // Here, we don't need to stitch together the issuer entries, // because the last run should've done that for us (or, when // importing an issuer). @@ -197,25 +216,25 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName result.ID = genKeyId() result.Name = keyName result.PrivateKey = keyValue + result.PrivateKeyType = keyType - // Extracting the signer is necessary for two reasons: first, to get the - // public key for comparison with existing issuers; second, to get the - // corresponding private key type. - keySigner, err := result.GetSigner() - if err != nil { + // Finally, we can write the key to storage. + if err := writeKey(mkc.ctx, s, result); err != nil { return nil, false, err } - keyPublic := keySigner.Public() - result.PrivateKeyType = certutil.GetPrivateKeyTypeFromSigner(keySigner) - // Finally, we can write the key to storage. - if err := writeKey(ctx, s, result); err != nil { + // Before we return below, we need to iterate over _all_ issuers and see if + // one of them has a missing KeyId link, and if so, point it back to + // ourselves. We fetch the list of issuers up front, even when don't need + // it, to give ourselves a better chance of succeeding below. + knownIssuers, err := listIssuers(mkc.ctx, s) + if err != nil { return nil, false, err } // Now, for each issuer, try and compute the issuer<->key link if missing. for _, identifier := range knownIssuers { - existingIssuer, err := fetchIssuerById(ctx, s, identifier) + existingIssuer, err := fetchIssuerById(mkc.ctx, s, identifier) if err != nil { return nil, false, err } @@ -234,7 +253,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName return nil, false, err } - equal, err := certutil.ComparePublicKeysAndType(cert.PublicKey, keyPublic) + equal, err := certutil.ComparePublicKeysAndType(cert.PublicKey, pkForImportingKey) if err != nil { return nil, false, err } @@ -243,7 +262,7 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName // These public keys are equal, so this key entry must be the // corresponding private key to this issuer; update it accordingly. existingIssuer.KeyID = result.ID - if err := writeIssuer(ctx, s, existingIssuer); err != nil { + if err := writeIssuer(mkc.ctx, s, existingIssuer); err != nil { return nil, false, err } } @@ -251,12 +270,12 @@ func importKey(ctx context.Context, s logical.Storage, keyValue string, keyName // If there was no prior default value set and/or we had no known // keys when we started, set this key as default. - keyDefaultSet, err := isDefaultKeySet(ctx, s) + keyDefaultSet, err := isDefaultKeySet(mkc.ctx, s) if err != nil { return nil, false, err } if len(knownKeys) == 0 || !keyDefaultSet { - if err = updateDefaultKeyId(ctx, s, result.ID); err != nil { + if err = updateDefaultKeyId(mkc.ctx, s, result.ID); err != nil { return nil, false, err } } @@ -384,7 +403,7 @@ func deleteIssuer(ctx context.Context, s logical.Storage, id issuerID) (bool, er return wasDefault, s.Delete(ctx, issuerPrefix+id.String()) } -func importIssuer(ctx context.Context, s logical.Storage, certValue string, issuerName string) (*issuerEntry, bool, error) { +func importIssuer(ctx managedKeyContext, s logical.Storage, certValue string, issuerName string) (*issuerEntry, bool, error) { // importIssuers imports the specified PEM-format certificate (from // certValue) into the new PKI storage format. The first return field is a // reference to the new issuer; the second is whether or not the issuer @@ -409,7 +428,7 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu // Before we can import a known issuer, we first need to know if the issuer // exists in storage already. This means iterating through all known // issuers and comparing their private value against this value. - knownIssuers, err := listIssuers(ctx, s) + knownIssuers, err := listIssuers(ctx.ctx, s) if err != nil { return nil, false, err } @@ -418,13 +437,13 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu // one of them a public key matching this certificate, and if so, update our // link accordingly. We fetch the list of keys up front, even may not need // it, to give ourselves a better chance of succeeding below. - knownKeys, err := listKeys(ctx, s) + knownKeys, err := listKeys(ctx.ctx, s) if err != nil { return nil, false, err } for _, identifier := range knownIssuers { - existingIssuer, err := fetchIssuerById(ctx, s, identifier) + existingIssuer, err := fetchIssuerById(ctx.ctx, s, identifier) if err != nil { return nil, false, err } @@ -470,18 +489,12 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu // writing issuer to storage as we won't need to update the key, only // the issuer. for _, identifier := range knownKeys { - existingKey, err := fetchKeyById(ctx, s, identifier) - if err != nil { - return nil, false, err - } - - // Fetch the signer for its Public() value. - signer, err := existingKey.GetSigner() + existingKey, err := fetchKeyById(ctx.ctx, s, identifier) if err != nil { return nil, false, err } - equal, err := certutil.ComparePublicKeysAndType(issuerCert.PublicKey, signer.Public()) + equal, err := comparePublicKey(ctx, existingKey, issuerCert.PublicKey) if err != nil { return nil, false, err } @@ -498,18 +511,18 @@ func importIssuer(ctx context.Context, s logical.Storage, certValue string, issu // Finally, rebuild the chains. In this process, because the provided // reference issuer is non-nil, we'll save this issuer to storage. - if err := rebuildIssuersChains(ctx, s, &result); err != nil { + if err := rebuildIssuersChains(ctx.ctx, s, &result); err != nil { return nil, false, err } // If there was no prior default value set and/or we had no known // issuers when we started, set this issuer as default. - issuerDefaultSet, err := isDefaultIssuerSet(ctx, s) + issuerDefaultSet, err := isDefaultIssuerSet(ctx.ctx, s) if err != nil { return nil, false, err } if len(knownIssuers) == 0 || !issuerDefaultSet { - if err = updateDefaultIssuerId(ctx, s, result.ID); err != nil { + if err = updateDefaultIssuerId(ctx.ctx, s, result.ID); err != nil { return nil, false, err } } @@ -692,19 +705,19 @@ func fetchCertBundleByIssuerId(ctx context.Context, s logical.Storage, id issuer return issuer, &bundle, nil } -func writeCaBundle(ctx context.Context, s logical.Storage, caBundle *certutil.CertBundle, issuerName string, keyName string) (*issuerEntry, *keyEntry, error) { - myKey, _, err := importKey(ctx, s, caBundle.PrivateKey, keyName) +func writeCaBundle(mkc managedKeyContext, s logical.Storage, caBundle *certutil.CertBundle, issuerName string, keyName string) (*issuerEntry, *keyEntry, error) { + myKey, _, err := importKey(mkc, s, caBundle.PrivateKey, keyName, caBundle.PrivateKeyType) if err != nil { return nil, nil, err } - myIssuer, _, err := importIssuer(ctx, s, caBundle.Certificate, issuerName) + myIssuer, _, err := importIssuer(mkc, s, caBundle.Certificate, issuerName) if err != nil { return nil, nil, err } for _, cert := range caBundle.CAChain { - if _, _, err = importIssuer(ctx, s, cert, ""); err != nil { + if _, _, err = importIssuer(mkc, s, cert, ""); err != nil { return nil, nil, err } } diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index c5679b53c1d72..3b3f75267465b 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -77,7 +77,8 @@ func migrateStorage(ctx context.Context, b *backend, s logical.Storage) error { b.Logger().Info("performing PKI migration to new keys/issuers layout") if migrationInfo.legacyBundle != nil { - anIssuer, aKey, err := writeCaBundle(ctx, s, migrationInfo.legacyBundle, "current", "current") + mkc := newManagedKeyContext(ctx, b, b.backendUuid) + anIssuer, aKey, err := writeCaBundle(mkc, s, migrationInfo.legacyBundle, "current", "current") if err != nil { return err } diff --git a/builtin/logical/pki/storage_test.go b/builtin/logical/pki/storage_test.go index 8b752cd73d00a..1d2d93acc783c 100644 --- a/builtin/logical/pki/storage_test.go +++ b/builtin/logical/pki/storage_test.go @@ -2,7 +2,6 @@ package pki import ( "context" - "crypto/rand" "strings" "testing" @@ -94,6 +93,8 @@ func Test_IssuerRoundTrip(t *testing.T) { func Test_KeysIssuerImport(t *testing.T) { b, s := createBackendWithStorage(t) + mkc := newManagedKeyContext(ctx, b, "test") + issuer1, key1 := genIssuerAndKey(t, b, s) issuer2, key2 := genIssuerAndKey(t, b, s) @@ -103,21 +104,21 @@ func Test_KeysIssuerImport(t *testing.T) { issuer1.ID = "" issuer1.KeyID = "" - key1Ref1, existing, err := importKey(ctx, s, key1.PrivateKey, "key1") + key1Ref1, existing, err := importKey(mkc, s, key1.PrivateKey, "key1", key1.PrivateKeyType) require.NoError(t, err) require.False(t, existing) require.Equal(t, strings.TrimSpace(key1.PrivateKey), strings.TrimSpace(key1Ref1.PrivateKey)) // Make sure if we attempt to re-import the same private key, no import/updates occur. // So the existing flag should be set to true, and we do not update the existing Name field. - key1Ref2, existing, err := importKey(ctx, s, key1.PrivateKey, "ignore-me") + key1Ref2, existing, err := importKey(mkc, s, key1.PrivateKey, "ignore-me", key1.PrivateKeyType) require.NoError(t, err) require.True(t, existing) require.Equal(t, key1.PrivateKey, key1Ref1.PrivateKey) require.Equal(t, key1Ref1.ID, key1Ref2.ID) require.Equal(t, key1Ref1.Name, key1Ref2.Name) - issuer1Ref1, existing, err := importIssuer(ctx, s, issuer1.Certificate, "issuer1") + issuer1Ref1, existing, err := importIssuer(mkc, s, issuer1.Certificate, "issuer1") require.NoError(t, err) require.False(t, existing) require.Equal(t, strings.TrimSpace(issuer1.Certificate), strings.TrimSpace(issuer1Ref1.Certificate)) @@ -126,7 +127,7 @@ func Test_KeysIssuerImport(t *testing.T) { // Make sure if we attempt to re-import the same issuer, no import/updates occur. // So the existing flag should be set to true, and we do not update the existing Name field. - issuer1Ref2, existing, err := importIssuer(ctx, s, issuer1.Certificate, "ignore-me") + issuer1Ref2, existing, err := importIssuer(mkc, s, issuer1.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) require.Equal(t, strings.TrimSpace(issuer1.Certificate), strings.TrimSpace(issuer1Ref1.Certificate)) @@ -141,7 +142,7 @@ func Test_KeysIssuerImport(t *testing.T) { require.NoError(t, err) // Same double import tests as above, but make sure if the previous was created through writeIssuer not importIssuer. - issuer2Ref, existing, err := importIssuer(ctx, s, issuer2.Certificate, "ignore-me") + issuer2Ref, existing, err := importIssuer(mkc, s, issuer2.Certificate, "ignore-me") require.NoError(t, err) require.True(t, existing) require.Equal(t, strings.TrimSpace(issuer2.Certificate), strings.TrimSpace(issuer2Ref.Certificate)) @@ -150,7 +151,7 @@ func Test_KeysIssuerImport(t *testing.T) { require.Equal(t, issuer2.KeyID, issuer2Ref.KeyID) // Same double import tests as above, but make sure if the previous was created through writeKey not importKey. - key2Ref, existing, err := importKey(ctx, s, key2.PrivateKey, "ignore-me") + key2Ref, existing, err := importKey(mkc, s, key2.PrivateKey, "ignore-me", key2.PrivateKeyType) require.NoError(t, err) require.True(t, existing) require.Equal(t, key2.PrivateKey, key2Ref.PrivateKey) @@ -207,7 +208,7 @@ func genCertBundle(t *testing.T, b *backend, s logical.Storage) *certutil.CertBu apiData: apiData, role: role, } - parsedCertBundle, err := generateCert(ctx, b, input, nil, true, rand.Reader) + parsedCertBundle, err := generateCert(ctx, b, input, nil, true, b.GetRandomReader()) require.NoError(t, err) certBundle, err := parsedCertBundle.ToCertBundle() diff --git a/sdk/helper/certutil/helpers.go b/sdk/helper/certutil/helpers.go index b9d3c61bc7f65..99bed25402ce6 100644 --- a/sdk/helper/certutil/helpers.go +++ b/sdk/helper/certutil/helpers.go @@ -1229,16 +1229,19 @@ func GetPublicKeySize(key crypto.PublicKey) int { return -1 } -func GetKeyBundleFromKeyGenerator(keyType string, keyBits int, keyGenerator KeyGenerator) (KeyBundle, error) { - result := KeyBundle{} - - if keyGenerator == nil { - keyGenerator = generatePrivateKey - } +// CreateKeyBundle create a KeyBundle struct object which includes a generated key +// of keyType with keyBits leveraging the randomness from randReader. +func CreateKeyBundle(keyType string, keyBits int, randReader io.Reader) (KeyBundle, error) { + return CreateKeyBundleWithKeyGenerator(keyType, keyBits, randReader, generatePrivateKey) +} - if err := keyGenerator(keyType, keyBits, &result, nil); err != nil { +// CreateKeyBundleWithKeyGenerator create a KeyBundle struct object which includes +// a generated key of keyType with keyBits leveraging the randomness from randReader and +// delegates the actual key generation to keyGenerator +func CreateKeyBundleWithKeyGenerator(keyType string, keyBits int, randReader io.Reader, keyGenerator KeyGenerator) (KeyBundle, error) { + result := KeyBundle{} + if err := keyGenerator(keyType, keyBits, &result, randReader); err != nil { return result, err } - return result, nil } From 6ac18d18dea98ab19b6cbdcf41836406a5c91e25 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 28 Apr 2022 12:49:19 -0400 Subject: [PATCH 70/76] Correctly handle rebuilding remaining chains When deleting a specific issuer, we might impact the chains. From a consistency perspective, we need to ensure the remaining chains are correct and don't refer to the since-deleted issuer, so trigger a full rebuild here. We don't need to call this in the delete-the-world (DELETE /root) code path, as there shouldn't be any remaining issuers or chains to build. Signed-off-by: Alexander Scheel --- builtin/logical/pki/path_fetch_issuers.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 3b4694e92f09c..4c7b2d646d8ef 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -378,6 +378,15 @@ func (b *backend) pathDeleteIssuer(ctx context.Context, req *logical.Request, da response.AddWarning(fmt.Sprintf("Deleted issuer %v (via issuer_ref %v); this was configured as the default issuer. Operations without an explicit issuer will not work until a new default is configured.", ref, issuerName)) } + // Since we've deleted an issuer, the chains might've changed. Call the + // rebuild code. We shouldn't technically err (as the issuer was deleted + // successfully), but log a warning (and to the response) if this fails. + if err := rebuildIssuersChains(ctx, req.Storage, nil); err != nil { + msg := fmt.Sprintf("Failed to rebuild remaining issuers' chains: %v", err) + b.Logger().Error(msg) + response.AddWarning(msg) + } + return response, nil } From 39b8093d9d213d918064ce95dc11f6d9ff6fab83 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 28 Apr 2022 12:54:25 -0400 Subject: [PATCH 71/76] Remove legacy CRL bundle on world deletion When calling DELETE /root, we should remove the legacy CRL bundle, since we're deleting the legacy CA issuer bundle as well. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 2 +- builtin/logical/pki/cert_util.go | 2 +- builtin/logical/pki/path_fetch.go | 4 ++-- builtin/logical/pki/path_root.go | 7 ++++++- builtin/logical/pki/storage.go | 7 ++++--- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index e168bb8ea94d2..753343d84ec8c 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -81,7 +81,7 @@ func Backend(conf *logical.BackendConfig) *backend { LocalStorage: []string{ "revoked/", - "crl", + legacyCRLPath, "crls/", "certs/", }, diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index 11fa043be5d38..c5bc87020404f 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -171,7 +171,7 @@ func fetchCertBySerial(ctx context.Context, b *backend, req *logical.Request, pr case strings.HasPrefix(prefix, "revoked/"): legacyPath = "revoked/" + colonSerial path = "revoked/" + hyphenSerial - case serial == "crl": + case serial == legacyCRLPath: if err = b.crlBuilder.rebuildIfForced(ctx, b, req); err != nil { return nil, err } diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index 6e30d6f9e7b97..dd474cc06e05a 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -179,14 +179,14 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data contentType = "application/pkix-cert" } case req.Path == "crl" || req.Path == "crl/pem": - serial = "crl" + serial = legacyCRLPath contentType = "application/pkix-crl" if req.Path == "crl/pem" { pemType = "X509 CRL" contentType = "application/x-pem-file" } case req.Path == "cert/crl": - serial = "crl" + serial = legacyCRLPath pemType = "X509 CRL" case strings.HasSuffix(req.Path, "/pem") || strings.HasSuffix(req.Path, "/raw"): serial = data.Get("serial").(string) diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index 121c7c4b5b39f..aa46e65b73f07 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -79,11 +79,16 @@ func (b *backend) pathCADeleteRoot(ctx context.Context, req *logical.Request, _ } } - // Delete legacy CA bundle; but don't error if it doesn't exist. + // Delete legacy CA bundle. if err := req.Storage.Delete(ctx, legacyCertBundlePath); err != nil { return nil, err } + // Delete legacy CRL bundle. + if err := req.Storage.Delete(ctx, legacyCRLPath); err != nil { + return nil, err + } + // Return a warning about preferring to delete issuers and keys // explicitly versus deleting everything. resp := &logical.Response{} diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 9908bb75fccc7..0c6725291a307 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -23,6 +23,7 @@ const ( legacyMigrationBundleLogKey = "config/legacyMigrationBundleLog" legacyCertBundlePath = "config/ca_bundle" + legacyCRLPath = "crl" ) type keyID string @@ -663,19 +664,19 @@ func resolveIssuerCRLPath(ctx context.Context, b *backend, s logical.Storage, re issuer, err := resolveIssuerReference(ctx, s, reference) if err != nil { - return "crl", err + return legacyCRLPath, err } crlConfig, err := getLocalCRLConfig(ctx, s) if err != nil { - return "crl", err + return legacyCRLPath, err } if crlId, ok := crlConfig.IssuerIDCRLMap[issuer]; ok && len(crlId) > 0 { return fmt.Sprintf("crls/%v", crlId), nil } - return "crl", fmt.Errorf("unable to find CRL for issuer: id:%v/ref:%v", issuer, reference) + return legacyCRLPath, fmt.Errorf("unable to find CRL for issuer: id:%v/ref:%v", issuer, reference) } // Builds a certutil.CertBundle from the specified issuer identifier, From 2b4d2568b04115b3b68a6bce19839ca657876267 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Thu, 28 Apr 2022 13:01:57 -0400 Subject: [PATCH 72/76] Remove deleted issuers' CRL entries Since CRLs are no longer resolvable after deletion (due to missing issuer ID, which will cause resolution to fail regardless of if an ID or a name/default reference was used), we should delete these CRLs from storage to avoid leaking them. In the event that this issuer comes back (with key material), we can simply rebuild the CRL at that time (from the remaining revoked storage entries). Signed-off-by: Alexander Scheel --- builtin/logical/pki/crl_util.go | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index 28fb2f3d4f626..e043e605a96a4 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -360,6 +360,40 @@ func buildCRLs(ctx context.Context, b *backend, req *logical.Request, forceNew b } } + // Before persisting our updated CRL config, check to see if we have + // any dangling references. If we have any issuers that don't exist, + // remove them, remembering their CRLs IDs. If we've completely removed + // all issuers pointing to that CRL number, we can remove it from the + // number map and from storage. + for mapIssuerId := range crlConfig.IssuerIDCRLMap { + stillHaveIssuer := false + for _, listedIssuerId := range issuers { + if mapIssuerId == listedIssuerId { + stillHaveIssuer = true + break + } + } + + if !stillHaveIssuer { + delete(crlConfig.IssuerIDCRLMap, mapIssuerId) + } + } + for crlId := range crlConfig.CRLNumberMap { + stillHaveIssuerForID := false + for _, remainingCRL := range crlConfig.IssuerIDCRLMap { + if remainingCRL == crlId { + stillHaveIssuerForID = true + break + } + } + + if !stillHaveIssuerForID { + if err := req.Storage.Delete(ctx, "crls/"+crlId.String()); err != nil { + return fmt.Errorf("error building CRLs: unable to clean up deleted issuers' CRL: %v", err) + } + } + } + // Finally, persist our potentially updated local CRL config if err := setLocalCRLConfig(ctx, req.Storage, crlConfig); err != nil { return fmt.Errorf("error building CRLs: unable to persist updated cluster-local CRL config: %v", err) From f82d3ef70bbc59292576c5a1b521d9de20729c44 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 2 May 2022 09:48:46 -0400 Subject: [PATCH 73/76] Add unauthed JSON fetching of CRLs, Issuers (#15253) Default to fetching JSON CRL for consistency This makes the bare issuer-specific CRL fetching endpoint return the JSON-wrapped CRL by default, moving the DER CRL to a specific endpoint. Signed-off-by: Alexander Scheel Add JSON-specific endpoint for fetching issuers Unlike the unqualified /issuer/:ref endpoint (which also returns JSON), we have a separate /issuer/:ref/json endpoint to return _only_ the PEM-encoded certificate and the chain, mirroring the existing /cert/ca endpoint but for a specific issuer. This allows us to make the endpoint unauthenticated, whereas the bare endpoint would remain authenticated and usually privileged. Signed-off-by: Alexander Scheel Add tests for raw JSON endpoints Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend_test.go | 2 +- builtin/logical/pki/ca_test.go | 64 ++++++++++++++++----- builtin/logical/pki/path_fetch_issuers.go | 69 +++++++++++++++-------- 3 files changed, 98 insertions(+), 37 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index bdaa6942a2b77..bb95880d8238c 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -4034,7 +4034,7 @@ func getParsedCrl(t *testing.T, client *api.Client, mountPoint string) *pkix.Cer } func getParsedCrlForIssuer(t *testing.T, client *api.Client, mountPoint string, issuer string) *pkix.CertificateList { - path := fmt.Sprintf("/v1/%v/issuer/%v/crl", mountPoint, issuer) + path := fmt.Sprintf("/v1/%v/issuer/%v/crl/der", mountPoint, issuer) crl := getParsedCrlAtPath(t, client, path) // Now fetch the issuer as well and verify the certificate diff --git a/builtin/logical/pki/ca_test.go b/builtin/logical/pki/ca_test.go index b74026b131a72..ec3220ba37c82 100644 --- a/builtin/logical/pki/ca_test.go +++ b/builtin/logical/pki/ca_test.go @@ -290,23 +290,29 @@ func runSteps(t *testing.T, rootB, intB *backend, client *api.Client, rootName, prevToken := client.Token() client.SetToken("") - // cert/ca path - { - resp, err := client.Logical().Read(rootName + "cert/ca") + // cert/ca and issuer/default/json path + for _, path := range []string{"cert/ca", "issuer/default/json"} { + resp, err := client.Logical().Read(rootName + path) if err != nil { t.Fatal(err) } if resp == nil { t.Fatal("nil response") } - if diff := deep.Equal(resp.Data["certificate"].(string), caCert); diff != nil { + expected := caCert + if path == "issuer/default/json" { + // Preserves the new line. + expected += "\n" + } + if diff := deep.Equal(resp.Data["certificate"].(string), expected); diff != nil { t.Fatal(diff) } } - // ca/pem path (raw string) - { + + // ca/pem and issuer/default/pem path (raw string) + for _, path := range []string{"ca/pem", "issuer/default/pem"} { req := &logical.Request{ - Path: "ca/pem", + Path: path, Operation: logical.ReadOperation, Storage: rootB.storage, } @@ -317,7 +323,12 @@ func runSteps(t *testing.T, rootB, intB *backend, client *api.Client, rootName, if resp == nil { t.Fatal("nil response") } - if diff := deep.Equal(resp.Data["http_raw_body"].([]byte), []byte(caCert)); diff != nil { + expected := []byte(caCert) + if path == "issuer/default/pem" { + // Preserves the new line. + expected = []byte(caCert + "\n") + } + if diff := deep.Equal(resp.Data["http_raw_body"].([]byte), expected); diff != nil { t.Fatal(diff) } if resp.Data["http_content_type"].(string) != "application/pem-certificate-chain" { @@ -325,10 +336,10 @@ func runSteps(t *testing.T, rootB, intB *backend, client *api.Client, rootName, } } - // ca (raw DER bytes) - { + // ca and issuer/default/der (raw DER bytes) + for _, path := range []string{"ca", "issuer/default/der"} { req := &logical.Request{ - Path: "ca", + Path: path, Operation: logical.ReadOperation, Storage: rootB.storage, } @@ -521,9 +532,16 @@ func runSteps(t *testing.T, rootB, intB *backend, client *api.Client, rootName, } // Fetch the CRL and make sure it shows up - { + for path, derPemOrJSON := range map[string]int{ + "crl": 0, + "issuer/default/crl/der": 0, + "crl/pem": 1, + "issuer/default/crl/pem": 1, + "cert/crl": 2, + "issuer/default/crl": 3, + } { req := &logical.Request{ - Path: "crl", + Path: path, Operation: logical.ReadOperation, Storage: rootB.storage, } @@ -534,7 +552,25 @@ func runSteps(t *testing.T, rootB, intB *backend, client *api.Client, rootName, if resp == nil { t.Fatal("nil response") } - crlBytes := resp.Data["http_raw_body"].([]byte) + + var crlBytes []byte + if derPemOrJSON == 2 { + // Old endpoint + crlBytes = []byte(resp.Data["certificate"].(string)) + } else if derPemOrJSON == 3 { + // New endpoint + crlBytes = []byte(resp.Data["crl"].(string)) + } else { + // DER or PEM + crlBytes = resp.Data["http_raw_body"].([]byte) + } + + if derPemOrJSON >= 1 { + // Do for both PEM and JSON endpoints + pemBlock, _ := pem.Decode(crlBytes) + crlBytes = pemBlock.Bytes + } + certList, err := x509.ParseCRL(crlBytes) if err != nil { t.Fatal(err) diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 4c7b2d646d8ef..42d0055897b7e 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -66,7 +66,7 @@ their identifier and their name (if set). ) func pathGetIssuer(b *backend) *framework.Path { - pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "(/der|/pem)?" + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "(/der|/pem|/json)?" return buildPathGetIssuer(b, pattern) } @@ -122,7 +122,7 @@ intermediate CAs and "permit" only for root CAs.`, func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { // Handle raw issuers first. - if strings.HasSuffix(req.Path, "/der") || strings.HasSuffix(req.Path, "/pem") { + if strings.HasSuffix(req.Path, "/der") || strings.HasSuffix(req.Path, "/pem") || strings.HasSuffix(req.Path, "/json") { return b.pathGetRawIssuer(ctx, req, data) } @@ -317,11 +317,15 @@ func (b *backend) pathGetRawIssuer(ctx context.Context, req *logical.Request, da } certificate := []byte(issuer.Certificate) - contentType := "application/pem-certificate-chain" - if strings.HasSuffix(req.Path, "/der") { + var contentType string + if strings.HasSuffix(req.Path, "/pem") { + contentType = "application/pem-certificate-chain" + } else if strings.HasSuffix(req.Path, "/der") { contentType = "application/pkix-cert" + } + if strings.HasSuffix(req.Path, "/der") { pemBlock, _ := pem.Decode(certificate) if pemBlock == nil { return nil, err @@ -335,13 +339,22 @@ func (b *backend) pathGetRawIssuer(ctx context.Context, req *logical.Request, da statusCode = 204 } - return &logical.Response{ - Data: map[string]interface{}{ - logical.HTTPContentType: contentType, - logical.HTTPRawBody: certificate, - logical.HTTPStatusCode: statusCode, - }, - }, nil + if strings.HasSuffix(req.Path, "/pem") || strings.HasSuffix(req.Path, "/der") { + return &logical.Response{ + Data: map[string]interface{}{ + logical.HTTPContentType: contentType, + logical.HTTPRawBody: certificate, + logical.HTTPStatusCode: statusCode, + }, + }, nil + } else { + return &logical.Response{ + Data: map[string]interface{}{ + "certificate": string(certificate), + "ca_chain": issuer.CAChain, + }, + }, nil + } } func (b *backend) pathDeleteIssuer(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { @@ -409,7 +422,7 @@ the certificate. ) func pathGetIssuerCRL(b *backend) *framework.Path { - pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "/crl(/pem)?" + pattern := "issuer/" + framework.GenericNameRegex(issuerRefParam) + "/crl(/pem|/der)?" return buildPathGetIssuerCRL(b, pattern) } @@ -462,13 +475,16 @@ func (b *backend) pathGetIssuerCRL(ctx context.Context, req *logical.Request, da certificate = []byte(crlEntry.Value) } - contentType := "application/pkix-crl" - - if strings.HasSuffix(req.Path, "/pem") { + var contentType string + if strings.HasSuffix(req.Path, "/der") { + contentType = "application/pkix-crl" + } else if strings.HasSuffix(req.Path, "/pem") { contentType = "application/x-pem-file" + } - // Rather return an empty response rather than an empty - // PEM blob. + if !strings.HasSuffix(req.Path, "/der") { + // Rather return an empty response rather than an empty PEM blob. + // We build this PEM block for both the JSON and PEM endpoints. if len(certificate) > 0 { pemBlock := pem.Block{ Type: "X509 CRL", @@ -484,11 +500,19 @@ func (b *backend) pathGetIssuerCRL(ctx context.Context, req *logical.Request, da statusCode = 204 } + if strings.HasSuffix(req.Path, "/der") || strings.HasSuffix(req.Path, "/pem") { + return &logical.Response{ + Data: map[string]interface{}{ + logical.HTTPContentType: contentType, + logical.HTTPRawBody: certificate, + logical.HTTPStatusCode: statusCode, + }, + }, nil + } + return &logical.Response{ Data: map[string]interface{}{ - logical.HTTPContentType: contentType, - logical.HTTPRawBody: certificate, - logical.HTTPStatusCode: statusCode, + "crl": string(certificate), }, }, nil } @@ -507,7 +531,8 @@ they have the same Subject value. will be consulted for the present default issuer, an identifier of an issuer, or its assigned name value. -Use /issuer/:ref/crl/pem to return just the certificate in PEM form; the raw -/issuer/:ref/crl is in DER form. + - /issuer/:ref/crl is JSON encoded and contains a PEM CRL, + - /issuer/:ref/crl/pem contains the PEM-encoded CRL, + - /issuer/:ref/crl/DER contains the raw DER-encoded (binary) CRL. ` ) From 725bd2e2117caa0c8d9d8ac954440f85e02cc95c Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Mon, 2 May 2022 12:07:46 -0400 Subject: [PATCH 74/76] Add unauthenticated issuers endpoints to PKI table This adds the unauthenticated issuers endpoints? - LIST /issuers, - Fetching _just_ the issuer certificates (in JSON/DER/PEM form), and - Fetching the CRL of this issuer (in JSON/DER/PEM form). Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/builtin/logical/pki/backend.go b/builtin/logical/pki/backend.go index 753343d84ec8c..d81bf84bcb70c 100644 --- a/builtin/logical/pki/backend.go +++ b/builtin/logical/pki/backend.go @@ -77,6 +77,13 @@ func Backend(conf *logical.BackendConfig) *backend { "ca", "crl/pem", "crl", + "issuer/+/crl/der", + "issuer/+/crl/pem", + "issuer/+/crl", + "issuer/+/pem", + "issuer/+/der", + "issuer/+/json", + "issuers", }, LocalStorage: []string{ From 15bc5981717a5bd18af93a6a789733bf1ebe70d7 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Tue, 3 May 2022 16:40:26 -0400 Subject: [PATCH 75/76] Add issuer usage restrictions bitset This allows issuers to have usage restrictions, limiting whether they can be used to issue certificates or if they can generate CRLs. This allows certain issuers to not generate a CRL (if the global config is with the CRL enabled) or allows the issuer to not issue new certificates (but potentially letting the CRL generation continue). Setting both fields to false effectively forms a soft delete capability. Signed-off-by: Alexander Scheel --- builtin/logical/pki/backend_test.go | 2 +- builtin/logical/pki/cert_util.go | 6 +- builtin/logical/pki/crl_util.go | 13 +++- builtin/logical/pki/path_fetch.go | 2 +- builtin/logical/pki/path_fetch_issuers.go | 25 +++++++ builtin/logical/pki/path_issue_sign.go | 2 +- builtin/logical/pki/path_root.go | 4 +- builtin/logical/pki/storage.go | 88 +++++++++++++++++++++++ builtin/logical/pki/storage_migrations.go | 8 ++- 9 files changed, 142 insertions(+), 8 deletions(-) diff --git a/builtin/logical/pki/backend_test.go b/builtin/logical/pki/backend_test.go index bb95880d8238c..1fc8ea38233a2 100644 --- a/builtin/logical/pki/backend_test.go +++ b/builtin/logical/pki/backend_test.go @@ -2660,7 +2660,7 @@ func TestBackend_SignSelfIssued(t *testing.T) { t.Fatal(err) } - signingBundle, err := fetchCAInfo(context.Background(), b, &logical.Request{Storage: storage}, defaultRef) + signingBundle, err := fetchCAInfo(context.Background(), b, &logical.Request{Storage: storage}, defaultRef, ReadOnlyUsage) if err != nil { t.Fatal(err) } diff --git a/builtin/logical/pki/cert_util.go b/builtin/logical/pki/cert_util.go index c5bc87020404f..46d2e579000ba 100644 --- a/builtin/logical/pki/cert_util.go +++ b/builtin/logical/pki/cert_util.go @@ -80,7 +80,7 @@ func getFormat(data *framework.FieldData) string { } // fetchCAInfo will fetch the CA info, will return an error if no ca info exists. -func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRef string) (*certutil.CAInfoBundle, error) { +func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRef string, usage issuerUsage) (*certutil.CAInfoBundle, error) { entry, bundle, err := fetchCertBundle(ctx, b, req.Storage, issuerRef) if err != nil { switch err.(type) { @@ -93,6 +93,10 @@ func fetchCAInfo(ctx context.Context, b *backend, req *logical.Request, issuerRe } } + if err := entry.EnsureUsage(usage); err != nil { + return nil, errutil.InternalError{Err: fmt.Sprintf("error while attempting to use issuer %v: %v", issuerRef, err)} + } + if bundle == nil { return nil, errutil.UserError{Err: "no CA information is present"} } diff --git a/builtin/logical/pki/crl_util.go b/builtin/logical/pki/crl_util.go index e043e605a96a4..5ff47fb15efb5 100644 --- a/builtin/logical/pki/crl_util.go +++ b/builtin/logical/pki/crl_util.go @@ -100,7 +100,7 @@ func revokeCert(ctx context.Context, b *backend, req *logical.Request, serial st return nil, nil } - signingBundle, caErr := fetchCAInfo(ctx, b, req, defaultRef) + signingBundle, caErr := fetchCAInfo(ctx, b, req, defaultRef, ReadOnlyUsage) if caErr != nil { switch caErr.(type) { case errutil.UserError: @@ -270,6 +270,11 @@ func buildCRLs(ctx context.Context, b *backend, req *logical.Request, forceNew b continue } + // Skip entries which aren't enabled for CRL signing. + if err := thisEntry.EnsureUsage(CRLSigningUsage); err != nil { + continue + } + issuerIDEntryMap[issuer] = thisEntry thisCert, err := thisEntry.GetCertificate() @@ -365,6 +370,12 @@ func buildCRLs(ctx context.Context, b *backend, req *logical.Request, forceNew b // remove them, remembering their CRLs IDs. If we've completely removed // all issuers pointing to that CRL number, we can remove it from the // number map and from storage. + // + // Note that we persist the last generated CRL for a specified issuer + // if it is later disabled for CRL generation. This mirrors the old + // root deletion behavior, but using soft issuer deletes. If there is an + // alternate, equivalent issuer however, we'll keep updating the shared + // CRL; all equivalent issuers must have their CRLs disabled. for mapIssuerId := range crlConfig.IssuerIDCRLMap { stillHaveIssuer := false for _, listedIssuerId := range issuers { diff --git a/builtin/logical/pki/path_fetch.go b/builtin/logical/pki/path_fetch.go index dd474cc06e05a..f4a2a6632c0f1 100644 --- a/builtin/logical/pki/path_fetch.go +++ b/builtin/logical/pki/path_fetch.go @@ -206,7 +206,7 @@ func (b *backend) pathFetchRead(ctx context.Context, req *logical.Request, data // Prefer fetchCAInfo to fetchCertBySerial for CA certificates. if serial == "ca_chain" || serial == "ca" { - caInfo, err := fetchCAInfo(ctx, b, req, defaultRef) + caInfo, err := fetchCAInfo(ctx, b, req, defaultRef, ReadOnlyUsage) if err != nil { switch err.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_fetch_issuers.go b/builtin/logical/pki/path_fetch_issuers.go index 42d0055897b7e..0bfd362750beb 100644 --- a/builtin/logical/pki/path_fetch_issuers.go +++ b/builtin/logical/pki/path_fetch_issuers.go @@ -91,6 +91,14 @@ the entire validity period. It is suggested to use "truncate" for intermediate CAs and "permit" only for root CAs.`, Default: "err", } + fields["usage"] = &framework.FieldSchema{ + Type: framework.TypeCommaStringSlice, + Description: `Comma-separated list (or string slice) of usages for +this issuer; valid values are "read-only", "issuing-certificates", and +"crl-signing". Multiple values may be specified. Read-only is implicit +and always set.`, + Default: []string{"read-only", "issuing-certificates", "crl-signing"}, + } return &framework.Path{ // Returns a JSON entry. @@ -162,6 +170,7 @@ func (b *backend) pathGetIssuer(ctx context.Context, req *logical.Request, data "manual_chain": respManualChain, "ca_chain": issuer.CAChain, "leaf_not_after_behavior": issuer.LeafNotAfterBehavior, + "usage": issuer.Usage.Names(), }, }, nil } @@ -202,6 +211,10 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da // errs should still be surfaced, however. return logical.ErrorResponse(err.Error()), nil } + if err == errIssuerNameInUse && issuer.Name != newName { + // When the new name is in use but isn't this name, throw an error. + return logical.ErrorResponse(err.Error()), nil + } newPath := data.Get("manual_chain").([]string) rawLeafBehavior := data.Get("leaf_not_after_behavior").(string) @@ -217,6 +230,12 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da return logical.ErrorResponse("Unknown value for field `leaf_not_after_behavior`. Possible values are `err`, `truncate`, and `permit`."), nil } + rawUsage := data.Get("usage").([]string) + newUsage, err := NewIssuerUsageFromNames(rawUsage) + if err != nil { + return logical.ErrorResponse(fmt.Sprintf("Unable to parse specified usages: %v - valid values are %v", rawUsage, AllIssuerUsages.Names())), nil + } + modified := false if newName != issuer.Name { @@ -229,6 +248,11 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da modified = true } + if newUsage != issuer.Usage { + issuer.Usage = newUsage + modified = true + } + var updateChain bool var constructedChain []issuerID for index, newPathRef := range newPath { @@ -289,6 +313,7 @@ func (b *backend) pathUpdateIssuer(ctx context.Context, req *logical.Request, da "manual_chain": respManualChain, "ca_chain": issuer.CAChain, "leaf_not_after_behavior": issuer.LeafNotAfterBehavior, + "usage": issuer.Usage.Names(), }, }, nil } diff --git a/builtin/logical/pki/path_issue_sign.go b/builtin/logical/pki/path_issue_sign.go index 093c1181cc1e0..041fd596ce28b 100644 --- a/builtin/logical/pki/path_issue_sign.go +++ b/builtin/logical/pki/path_issue_sign.go @@ -262,7 +262,7 @@ func (b *backend) pathIssueSignCert(ctx context.Context, req *logical.Request, d } var caErr error - signingBundle, caErr := fetchCAInfo(ctx, b, req, issuerName) + signingBundle, caErr := fetchCAInfo(ctx, b, req, issuerName, IssuanceUsage) if caErr != nil { switch caErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/path_root.go b/builtin/logical/pki/path_root.go index aa46e65b73f07..f20ef96959fa1 100644 --- a/builtin/logical/pki/path_root.go +++ b/builtin/logical/pki/path_root.go @@ -276,7 +276,7 @@ func (b *backend) pathIssuerSignIntermediate(ctx context.Context, req *logical.R } var caErr error - signingBundle, caErr := fetchCAInfo(ctx, b, req, issuerName) + signingBundle, caErr := fetchCAInfo(ctx, b, req, issuerName, IssuanceUsage) if caErr != nil { switch caErr.(type) { case errutil.UserError: @@ -417,7 +417,7 @@ func (b *backend) pathIssuerSignSelfIssued(ctx context.Context, req *logical.Req } var caErr error - signingBundle, caErr := fetchCAInfo(ctx, b, req, issuerName) + signingBundle, caErr := fetchCAInfo(ctx, b, req, issuerName, IssuanceUsage) if caErr != nil { switch caErr.(type) { case errutil.UserError: diff --git a/builtin/logical/pki/storage.go b/builtin/logical/pki/storage.go index 0c6725291a307..7237b96b3d4d5 100644 --- a/builtin/logical/pki/storage.go +++ b/builtin/logical/pki/storage.go @@ -67,6 +67,68 @@ func (e keyEntry) isManagedPrivateKey() bool { return e.PrivateKeyType == certutil.ManagedPrivateKey } +type issuerUsage uint + +const ( + ReadOnlyUsage issuerUsage = iota + IssuanceUsage issuerUsage = 1 << iota + CRLSigningUsage issuerUsage = 1 << iota + + // When adding a new usage in the future, we'll need to create a usage + // mask field on the IssuerEntry and handle migrations to a newer mask, + // inferring a value for the new bits. + AllIssuerUsages issuerUsage = ReadOnlyUsage | IssuanceUsage | CRLSigningUsage +) + +var namedIssuerUsages = map[string]issuerUsage{ + "read-only": ReadOnlyUsage, + "issuing-certificates": IssuanceUsage, + "crl-signing": CRLSigningUsage, +} + +func (i *issuerUsage) ToggleUsage(usages ...issuerUsage) { + for _, usage := range usages { + *i ^= usage + } +} + +func (i issuerUsage) HasUsage(usage issuerUsage) bool { + return (i & usage) == usage +} + +func (i issuerUsage) Names() string { + var names []string + var builtUsage issuerUsage + + for name, usage := range namedIssuerUsages { + if i.HasUsage(usage) { + names = append(names, name) + builtUsage.ToggleUsage(usage) + } + } + + if i != builtUsage { + // Found some unknown usage, we should indicate this in the names. + names = append(names, fmt.Sprintf("unknown:%v", i^builtUsage)) + } + + return strings.Join(names, ",") +} + +func NewIssuerUsageFromNames(names []string) (issuerUsage, error) { + var result issuerUsage + for index, name := range names { + usage, ok := namedIssuerUsages[name] + if !ok { + return ReadOnlyUsage, fmt.Errorf("unknown name for usage at index %v: %v", index, name) + } + + result.ToggleUsage(usage) + } + + return result, nil +} + type issuerEntry struct { ID issuerID `json:"id" structs:"id" mapstructure:"id"` Name string `json:"name" structs:"name" mapstructure:"name"` @@ -76,6 +138,7 @@ type issuerEntry struct { ManualChain []issuerID `json:"manual_chain" structs:"manual_chain" mapstructure:"manual_chain"` SerialNumber string `json:"serial_number" structs:"serial_number" mapstructure:"serial_number"` LeafNotAfterBehavior certutil.NotAfterBehavior `json:"not_after_behavior" structs:"not_after_behavior" mapstructure:"not_after_behavior"` + Usage issuerUsage `json:"usage" structs:"usage" mapstructure:"usage"` } type localCRLConfigEntry struct { @@ -297,6 +360,30 @@ func (i issuerEntry) GetCertificate() (*x509.Certificate, error) { return x509.ParseCertificate(block.Bytes) } +func (i issuerEntry) EnsureUsage(usage issuerUsage) error { + // We want to spit out a nice error message about missing usages. + if i.Usage.HasUsage(usage) { + return nil + } + + issuerRef := fmt.Sprintf("id:%v", i.ID) + if len(i.Name) > 0 { + issuerRef = fmt.Sprintf("%v / name:%v", issuerRef, i.Name) + } + + // These usages differ at some point in time. We've gotta find the first + // usage that differs and return a logical-sounding error message around + // that difference. + for name, candidate := range namedIssuerUsages { + if usage.HasUsage(candidate) && !i.Usage.HasUsage(candidate) { + return fmt.Errorf("requested usage %v for issuer [%v] but only had usage %v", name, issuerRef, i.Usage.Names()) + } + } + + // Maybe we have an unnamed usage that's requested. + return fmt.Errorf("unknown delta between usages: %v -> %v / for issuer [%v]", usage.Names(), i.Usage.Names(), issuerRef) +} + func listIssuers(ctx context.Context, s logical.Storage) ([]issuerID, error) { strList, err := s.List(ctx, issuerPrefix) if err != nil { @@ -464,6 +551,7 @@ func importIssuer(ctx managedKeyContext, s logical.Storage, certValue string, is result.Name = issuerName result.Certificate = certValue result.LeafNotAfterBehavior = certutil.ErrNotAfterBehavior + result.Usage.ToggleUsage(IssuanceUsage, CRLSigningUsage) // We shouldn't add CSRs or multiple certificates in this countCertificates := strings.Count(result.Certificate, "-BEGIN ") diff --git a/builtin/logical/pki/storage_migrations.go b/builtin/logical/pki/storage_migrations.go index 3b3f75267465b..18c17ecea3e83 100644 --- a/builtin/logical/pki/storage_migrations.go +++ b/builtin/logical/pki/storage_migrations.go @@ -13,7 +13,10 @@ import ( // This allows us to record the version of the migration code within the log entry // in case we find out in the future that something was horribly wrong with the migration, // and we need to perform it again... -const latestMigrationVersion = 1 +const ( + latestMigrationVersion = 1 + legacyBundleShimID = issuerID("legacy-entry-shim-id") +) type legacyBundleMigrationLog struct { Hash string `json:"hash" structs:"hash" mapstructure:"hash"` @@ -173,8 +176,11 @@ func getLegacyCertBundle(ctx context.Context, s logical.Storage) (*issuerEntry, // Fake a storage entry with backwards compatibility in mind. We only need // the fields in the CAInfoBundle; everything else doesn't matter. issuer := &issuerEntry{ + ID: legacyBundleShimID, + Name: "legacy-entry-shim", LeafNotAfterBehavior: certutil.ErrNotAfterBehavior, } + issuer.Usage.ToggleUsage(IssuanceUsage, CRLSigningUsage) return issuer, cb, nil } From bba4bcb19515e8ca5ea35931c15bff216951d5e0 Mon Sep 17 00:00:00 2001 From: kitography Date: Wed, 4 May 2022 12:09:37 -0400 Subject: [PATCH 76/76] PKI Pod rotation Add Base Changelog (#15283) * PKI Pod rotation changelog. * Use feature release-note formatting of changelog. --- changelog/15277.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/15277.txt diff --git a/changelog/15277.txt b/changelog/15277.txt new file mode 100644 index 0000000000000..3b23d0e44ff68 --- /dev/null +++ b/changelog/15277.txt @@ -0,0 +1,11 @@ +```release-note:feature +**Allows Multiple Issuer Certificates to enable Non-Disruptive +Intermediate/Root Certificate Rotation**: This introduces /keys and /issuers +endpoints to allow import, generation and configuration of any number of keys +or issuers that can be used to issue and revoke certificates. Keys and Issuers +can be referred to by (a) a unique UUID; (b) a name; (c) “default”. If an +issuer existed prior to this feature, that issuer will be tagged by a migration +as “default” to allow backwards compatible calls which don’t specify an issuer. +Creation of new roles will assume an issuer of “default” unless otherwise +specified. This default can be configured at /config/issuers and /config/keys. +```