New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Provide 'sys/step-down' and 'vault step-down' #1146
Changes from 1 commit
6b0c692
ef4466d
d0ec85f
2a347d2
1ae2f2f
4e964a6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
package api | ||
|
||
func (c *Sys) StepDown() error { | ||
r := c.c.NewRequest("PUT", "/v1/sys/step-down") | ||
resp, err := c.c.RawRequest(r) | ||
if err == nil { | ||
defer resp.Body.Close() | ||
} | ||
return err | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
package command | ||
|
||
import ( | ||
"fmt" | ||
"strings" | ||
) | ||
|
||
// StepDownCommand is a Command that seals the vault. | ||
type StepDownCommand struct { | ||
Meta | ||
} | ||
|
||
func (c *StepDownCommand) Run(args []string) int { | ||
flags := c.Meta.FlagSet("step-down", FlagSetDefault) | ||
flags.Usage = func() { c.Ui.Error(c.Help()) } | ||
if err := flags.Parse(args); err != nil { | ||
return 1 | ||
} | ||
|
||
client, err := c.Client() | ||
if err != nil { | ||
c.Ui.Error(fmt.Sprintf( | ||
"Error initializing client: %s", err)) | ||
return 2 | ||
} | ||
|
||
if err := client.Sys().StepDown(); err != nil { | ||
c.Ui.Error(fmt.Sprintf("Error stepping down: %s", err)) | ||
return 1 | ||
} | ||
|
||
return 0 | ||
} | ||
|
||
func (c *StepDownCommand) Synopsis() string { | ||
return "Force the Vault node to give up active duty" | ||
} | ||
|
||
func (c *StepDownCommand) Help() string { | ||
helpText := ` | ||
Usage: vault step-down [options] | ||
|
||
Force the Vault node to step down from active duty. | ||
|
||
This causes the indicated node to give up active status. Note that while the | ||
affected node will have a short delay before attempting to grab the lock | ||
again, if no other node grabs the lock beforehand, it is possible for the | ||
same node to re-grab the lock and become active again. | ||
|
||
General Options: | ||
|
||
` + generalOptionsUsage() | ||
return strings.TrimSpace(helpText) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1157,22 +1157,45 @@ func (c *Core) Unseal(key []byte) (bool, error) { | |
return true, nil | ||
} | ||
|
||
// Seal is used to re-seal the Vault. This requires the Vault to | ||
// be unsealed again to perform any further operations. | ||
func (c *Core) Seal(token string) (retErr error) { | ||
defer metrics.MeasureSince([]string{"core", "seal"}, time.Now()) | ||
// Seal is used to seal the vault | ||
func (c *Core) Seal(token string) error { | ||
return c.stepDownAndSeal(token, true) | ||
} | ||
|
||
// StepDown is used to step down from leadership | ||
func (c *Core) StepDown(token string) error { | ||
return c.stepDownAndSeal(token, false) | ||
} | ||
|
||
// stepDownAndSeal is used to step down from leadership and, optionally, | ||
// re-seal the Vault. If sealed, this requires the Vault to be unsealed again | ||
// to perform any further operations. | ||
func (c *Core) stepDownAndSeal(token string, seal bool) (retErr error) { | ||
if seal { | ||
defer metrics.MeasureSince([]string{"core", "seal"}, time.Now()) | ||
} else { | ||
defer metrics.MeasureSince([]string{"core", "step_down"}, time.Now()) | ||
} | ||
|
||
c.stateLock.Lock() | ||
defer c.stateLock.Unlock() | ||
if c.sealed { | ||
return nil | ||
} | ||
if !seal && (c.ha == nil || c.standby) { | ||
return nil | ||
} | ||
|
||
// Validate the token is a root token | ||
req := &logical.Request{ | ||
Operation: logical.UpdateOperation, | ||
Path: "sys/seal", | ||
ClientToken: token, | ||
} | ||
if seal { | ||
req.Path = "sys/seal" | ||
} else { | ||
req.Path = "sys/step-down" | ||
} | ||
acl, te, err := c.fetchACLandTokenEntry(req) | ||
|
||
// Attempt to use the token (decrement num_uses) | ||
|
@@ -1189,8 +1212,8 @@ func (c *Core) Seal(token string) (retErr error) { | |
// just returning with an error and recommending a vault restart, which | ||
// essentially does the same thing. | ||
if c.standby { | ||
c.logger.Printf("[ERR] core: vault cannot be sealed when in standby mode; please restart instead") | ||
return errors.New("vault cannot be sealed when in standby mode; please restart instead") | ||
c.logger.Printf("[ERR] core: vault cannot step down or be sealed when in standby mode; please restart instead") | ||
return errors.New("vault cannot step down or be sealed when in standby mode; please restart instead") | ||
} | ||
return err | ||
} | ||
|
@@ -1207,19 +1230,22 @@ func (c *Core) Seal(token string) (retErr error) { | |
} | ||
|
||
// Seal the Vault | ||
err = c.sealInternal() | ||
if err == nil && retErr == ErrInternalError { | ||
c.logger.Printf("[ERR] core: core is successfully sealed but another error occurred during the operation") | ||
if seal { | ||
err = c.sealInternal() | ||
if err == nil && retErr == ErrInternalError { | ||
c.logger.Printf("[ERR] core: core is successfully sealed but another error occurred during the operation") | ||
} else { | ||
retErr = err | ||
} | ||
} else { | ||
retErr = err | ||
c.stepDownInternal() | ||
} | ||
|
||
return | ||
} | ||
|
||
// sealInternal is an internal method used to seal the vault. | ||
// It does not do any authorization checking. The stateLock must | ||
// be held prior to calling. | ||
// sealInternal is an internal method used to seal the vault. It does not do | ||
// any authorization checking. The stateLock must be held prior to calling. | ||
func (c *Core) sealInternal() error { | ||
// Enable that we are sealed to prevent furthur transactions | ||
c.sealed = true | ||
|
@@ -1244,9 +1270,20 @@ func (c *Core) sealInternal() error { | |
return err | ||
} | ||
c.logger.Printf("[INFO] core: vault is sealed") | ||
|
||
return nil | ||
} | ||
|
||
// stepDownInternal is an internal method used to step down from active duty. | ||
// It does not do any authorization checking. | ||
func (c *Core) stepDownInternal() { | ||
// Merely trigger the loop to re-run. This value will cause the | ||
// loop to run through giving up leadership, but without triggering | ||
// the return at the end of the next loop run, since it's not | ||
// closed | ||
c.standbyStopCh <- struct{}{} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The problem is There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thought about this too; it'd have to be pretty unlucky, as in, you'd have to be losing leadership just as you're manually requesting a step down, but it's certainly true that it could happen. I like your approach better though. |
||
} | ||
|
||
// postUnseal is invoked after the barrier is unsealed, but before | ||
// allowing any user operations. This allows us to setup any state that | ||
// requires the Vault to be unsealed such as mount tables, logical backends, | ||
|
@@ -1443,6 +1480,10 @@ func (c *Core) runStandby(doneCh, stopCh chan struct{}) { | |
if preSealErr != nil { | ||
c.logger.Printf("[ERR] core: pre-seal teardown failed: %v", err) | ||
} | ||
|
||
// If we've merely stepped down, we could instantly grab the lock | ||
// again. Give the other nodes a chance. | ||
time.Sleep(time.Second) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We can add logic to only do this on a user initiated step down using the bool channel to avoid adding a failover delay. Also lets move the time value into a constant instead of hard coding a second. (Probably at least 5-15 seconds is reasonable on user initiated). |
||
} | ||
} | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I would just move this into the appropriate methods above to make it cleaner
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I thought about that; I was attempting to avoid copypasta since much of the logic is the same. But there's enough different that it can go either way.