Skip to content

Commit

Permalink
Fix: rotate root credentials for database plugins using WAL (#8782)
Browse files Browse the repository at this point in the history
* fix: rotate root credentials for database plugins using WAL

* test: adds a test for WAL rollback logic

* fix: progress on wal rollback

* docs: updates some comments

* docs: updates some comments

* test: adds additional test coverage for WAL rollback

* chore: remove unneeded log

* style: error handling, imports, signature line wraps

* fix: always close db plugin connection
  • Loading branch information
austingebauer committed Apr 22, 2020
1 parent 8f834b3 commit 7807d45
Show file tree
Hide file tree
Showing 9 changed files with 611 additions and 27 deletions.
24 changes: 16 additions & 8 deletions builtin/logical/database/backend.go
Expand Up @@ -6,6 +6,7 @@ import (
"net/rpc"
"strings"
"sync"
"time"

log "github.com/hashicorp/go-hclog"

Expand All @@ -24,6 +25,7 @@ const (
databaseConfigPath = "database/config/"
databaseRolePath = "role/"
databaseStaticRolePath = "static-role/"
minRootCredRollbackAge = 1 * time.Minute
)

type dbPluginInstance struct {
Expand Down Expand Up @@ -93,9 +95,11 @@ func Backend(conf *logical.BackendConfig) *databaseBackend {
Secrets: []*framework.Secret{
secretCreds(&b),
},
Clean: b.clean,
Invalidate: b.invalidate,
BackendType: logical.TypeLogical,
Clean: b.clean,
Invalidate: b.invalidate,
WALRollback: b.walRollback,
WALRollbackMinAge: minRootCredRollbackAge,
BackendType: logical.TypeLogical,
}

b.logger = conf.Logger
Expand Down Expand Up @@ -223,6 +227,15 @@ func (b *databaseBackend) invalidate(ctx context.Context, key string) {
}

func (b *databaseBackend) GetConnection(ctx context.Context, s logical.Storage, name string) (*dbPluginInstance, error) {
config, err := b.DatabaseConfig(ctx, s, name)
if err != nil {
return nil, err
}

return b.GetConnectionWithConfig(ctx, name, config)
}

func (b *databaseBackend) GetConnectionWithConfig(ctx context.Context, name string, config *DatabaseConfig) (*dbPluginInstance, error) {
b.RLock()
unlockFunc := b.RUnlock
defer func() { unlockFunc() }()
Expand All @@ -242,11 +255,6 @@ func (b *databaseBackend) GetConnection(ctx context.Context, s logical.Storage,
return db, nil
}

config, err := b.DatabaseConfig(ctx, s, name)
if err != nil {
return nil, err
}

dbp, err := dbplugin.PluginFactory(ctx, config.PluginName, b.System(), b.logger)
if err != nil {
return nil, err
Expand Down
2 changes: 1 addition & 1 deletion builtin/logical/database/backend_test.go
Expand Up @@ -71,7 +71,7 @@ func preparePostgresTestContainer(t *testing.T, s logical.Storage, b logical.Bac
})
if err != nil || (resp != nil && resp.IsError()) {
// It's likely not up and running yet, so return error and try again
return fmt.Errorf("err:%#v resp:%#v", err, resp)
return fmt.Errorf("err:%#v resp:%+v", err, resp)
}
if resp == nil {
t.Fatal("expected warning")
Expand Down
60 changes: 52 additions & 8 deletions builtin/logical/database/path_rotate_credentials.go
Expand Up @@ -5,6 +5,10 @@ import (
"fmt"
"time"

"github.com/hashicorp/vault/sdk/database/dbplugin"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"github.com/hashicorp/vault/sdk/framework"
"github.com/hashicorp/vault/sdk/logical"
"github.com/hashicorp/vault/sdk/queue"
Expand Down Expand Up @@ -72,6 +76,16 @@ func (b *databaseBackend) pathRotateCredentialsUpdate() framework.OperationFunc
return nil, err
}

defer func() {
// Close the plugin
db.closed = true
if err := db.Database.Close(); err != nil {
b.Logger().Error("error closing the database plugin connection", "err", err)
}
// Even on error, still remove the connection
delete(b.connections, name)
}()

// Take out the backend lock since we are swapping out the connection
b.Lock()
defer b.Unlock()
Expand All @@ -80,12 +94,44 @@ func (b *databaseBackend) pathRotateCredentialsUpdate() framework.OperationFunc
db.Lock()
defer db.Unlock()

connectionDetails, err := db.RotateRootCredentials(ctx, config.RootCredentialsRotateStatements)
// Generate new credentials
userName := config.ConnectionDetails["username"].(string)
oldPassword := config.ConnectionDetails["password"].(string)
newPassword, err := db.GenerateCredentials(ctx)
if err != nil {
return nil, err
}
config.ConnectionDetails["password"] = newPassword

// Write a WAL entry
walID, err := framework.PutWAL(ctx, req.Storage, rotateRootWALKey, &rotateRootCredentialsWAL{
ConnectionName: name,
UserName: userName,
OldPassword: oldPassword,
NewPassword: newPassword,
})
if err != nil {
return nil, err
}

config.ConnectionDetails = connectionDetails
// Attempt to use SetCredentials for the root credential rotation
statements := dbplugin.Statements{Rotation: config.RootCredentialsRotateStatements}
userConfig := dbplugin.StaticUserConfig{
Username: userName,
Password: newPassword,
}
if _, _, err := db.SetCredentials(ctx, statements, userConfig); err != nil {
if status.Code(err) == codes.Unimplemented {
// Fall back to using RotateRootCredentials if unimplemented
config.ConnectionDetails, err = db.RotateRootCredentials(ctx,
config.RootCredentialsRotateStatements)
}
if err != nil {
return nil, err
}
}

// Update storage with the new root credentials
entry, err := logical.StorageEntryJSON(fmt.Sprintf("config/%s", name), config)
if err != nil {
return nil, err
Expand All @@ -94,17 +140,15 @@ func (b *databaseBackend) pathRotateCredentialsUpdate() framework.OperationFunc
return nil, err
}

// Close the plugin
db.closed = true
if err := db.Database.Close(); err != nil {
b.Logger().Error("error closing the database plugin connection", "err", err)
// Delete the WAL entry after successfully rotating root credentials
if err := framework.DeleteWAL(ctx, req.Storage, walID); err != nil {
b.Logger().Warn("unable to delete WAL", "error", err, "WAL ID", walID)
}
// Even on error, still remove the connection
delete(b.connections, name)

return nil, nil
}
}

func (b *databaseBackend) pathRotateRoleCredentialsUpdate() framework.OperationFunc {
return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
name := data.Get("name").(string)
Expand Down
112 changes: 112 additions & 0 deletions builtin/logical/database/rollback.go
@@ -0,0 +1,112 @@
package database

import (
"context"
"errors"

"github.com/hashicorp/vault/sdk/database/dbplugin"
"github.com/hashicorp/vault/sdk/logical"
"github.com/mitchellh/mapstructure"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)

// WAL storage key used for the rollback of root database credentials
const rotateRootWALKey = "rotateRootWALKey"

// WAL entry used for the rollback of root database credentials
type rotateRootCredentialsWAL struct {
ConnectionName string
UserName string
NewPassword string
OldPassword string
}

// walRollback handles WAL entries that result from partial failures
// to rotate the root credentials of a database. It is responsible
// for rolling back root database credentials when doing so would
// reconcile the credentials with Vault storage.
func (b *databaseBackend) walRollback(ctx context.Context, req *logical.Request, kind string, data interface{}) error {
if kind != rotateRootWALKey {
return errors.New("unknown type to rollback")
}

// Decode the WAL data
var entry rotateRootCredentialsWAL
if err := mapstructure.Decode(data, &entry); err != nil {
return err
}

// Get the current database configuration from storage
config, err := b.DatabaseConfig(ctx, req.Storage, entry.ConnectionName)
if err != nil {
return err
}

// The password in storage doesn't match the new password
// in the WAL entry. This means there was a partial failure
// to update either the database or storage.
if config.ConnectionDetails["password"] != entry.NewPassword {
// Clear any cached connection to inform the rollback decision
err := b.ClearConnection(entry.ConnectionName)
if err != nil {
return err
}

// Attempt to get a connection with the current configuration.
// If successful, the WAL entry can be deleted. This means
// the root credentials are the same according to the database
// and storage.
_, err = b.GetConnection(ctx, req.Storage, entry.ConnectionName)
if err == nil {
return nil
}

return b.rollbackDatabaseCredentials(ctx, config, entry)
}

// The password in storage matches the new password in
// the WAL entry, so there is nothing to roll back. This
// means the new password was successfully updated in the
// database and storage, but the WAL wasn't deleted.
return nil
}

// rollbackDatabaseCredentials rolls back root database credentials for
// the connection associated with the passed WAL entry. It will creates
// a connection to the database using the WAL entry new password in
// order to alter the password to be the WAL entry old password.
func (b *databaseBackend) rollbackDatabaseCredentials(ctx context.Context, config *DatabaseConfig, entry rotateRootCredentialsWAL) error {
// Attempt to get a connection with the WAL entry new password.
config.ConnectionDetails["password"] = entry.NewPassword
dbc, err := b.GetConnectionWithConfig(ctx, entry.ConnectionName, config)
if err != nil {
return err
}

// Ensure the connection used to roll back the database password is not cached
defer func() {
if err := b.ClearConnection(entry.ConnectionName); err != nil {
b.Logger().Error("error closing database plugin connection", "err", err)
}
}()

// Roll back the database password to the WAL entry old password
statements := dbplugin.Statements{Rotation: config.RootCredentialsRotateStatements}
userConfig := dbplugin.StaticUserConfig{
Username: entry.UserName,
Password: entry.OldPassword,
}
if _, _, err := dbc.SetCredentials(ctx, statements, userConfig); err != nil {
// If the database plugin doesn't implement SetCredentials, the root
// credentials can't be rolled back. This means the root credential
// rotation happened via the plugin RotateRootCredentials RPC.
if status.Code(err) == codes.Unimplemented {
return nil
}

return err
}

return nil
}

0 comments on commit 7807d45

Please sign in to comment.