Skip to content

Commit

Permalink
Add autopilot automated upgrades and redundancy zones (hashicorp#15521)
Browse files Browse the repository at this point in the history
  • Loading branch information
@raskchanky @Gabrielopesantos
raskchanky authored and Gabrielopesantos committed Jun 6, 2022
1 parent 8f77c5a commit 1a87e35
Show file tree
Hide file tree
Showing 23 changed files with 689 additions and 306 deletions.
12 changes: 11 additions & 1 deletion api/client.go
View file
Expand Up @@ -608,7 +608,7 @@ func (c *Client) CloneConfig() *Config {
return newConfig
}

// Sets the address of Vault in the client. The format of address should be
// SetAddress sets the address of Vault in the client. The format of address should be
// "<Scheme>://<Host>:<Port>". Setting this on a client will override the
// value of VAULT_ADDR environment variable.
func (c *Client) SetAddress(addr string) error {
Expand All @@ -635,6 +635,16 @@ func (c *Client) Address() string {
return c.addr.String()
}

func (c *Client) SetCheckRedirect(f func(*http.Request, []*http.Request) error) {
c.modifyLock.Lock()
defer c.modifyLock.Unlock()

c.config.modifyLock.Lock()
defer c.config.modifyLock.Unlock()

c.config.HttpClient.CheckRedirect = f
}

// SetLimiter will set the rate limiter for this client.
// This method is thread-safe.
// rateLimit and burst are specified according to https://godoc.org/golang.org/x/time/rate#NewLimiter
Expand Down
3 changes: 3 additions & 0 deletions api/sys_hastatus.go
View file
Expand Up @@ -37,4 +37,7 @@ type HANode struct {
ClusterAddress string `json:"cluster_address"`
ActiveNode bool `json:"active_node"`
LastEcho *time.Time `json:"last_echo"`
Version string `json:"version"`
UpgradeVersion string `json:"upgrade_version,omitempty"`
RedundancyZone string `json:"redundancy_zone,omitempty"`
}
67 changes: 50 additions & 17 deletions api/sys_raft.go
View file
Expand Up @@ -44,6 +44,7 @@ type AutopilotConfig struct {
MaxTrailingLogs uint64 `json:"max_trailing_logs" mapstructure:"max_trailing_logs"`
MinQuorum uint `json:"min_quorum" mapstructure:"min_quorum"`
ServerStabilizationTime time.Duration `json:"server_stabilization_time" mapstructure:"-"`
DisableUpgradeMigration bool `json:"disable_upgrade_migration" mapstructure:"disable_upgrade_migration"`
}

// MarshalJSON makes the autopilot config fields JSON compatible
Expand All @@ -55,6 +56,7 @@ func (ac *AutopilotConfig) MarshalJSON() ([]byte, error) {
"max_trailing_logs": ac.MaxTrailingLogs,
"min_quorum": ac.MinQuorum,
"server_stabilization_time": ac.ServerStabilizationTime.String(),
"disable_upgrade_migration": ac.DisableUpgradeMigration,
})
}

Expand Down Expand Up @@ -84,28 +86,59 @@ func (ac *AutopilotConfig) UnmarshalJSON(b []byte) error {

// AutopilotState represents the response of the raft autopilot state API
type AutopilotState struct {
Healthy bool `mapstructure:"healthy"`
FailureTolerance int `mapstructure:"failure_tolerance"`
Servers map[string]*AutopilotServer `mapstructure:"servers"`
Leader string `mapstructure:"leader"`
Voters []string `mapstructure:"voters"`
NonVoters []string `mapstructure:"non_voters"`
Healthy bool `mapstructure:"healthy"`
FailureTolerance int `mapstructure:"failure_tolerance"`
Servers map[string]*AutopilotServer `mapstructure:"servers"`
Leader string `mapstructure:"leader"`
Voters []string `mapstructure:"voters"`
NonVoters []string `mapstructure:"non_voters"`
RedundancyZones map[string]AutopilotZone `mapstructure:"redundancy_zones,omitempty"`
Upgrade *AutopilotUpgrade `mapstructure:"upgrade_info,omitempty"`
OptimisticFailureTolerance int `mapstructure:"optimistic_failure_tolerance,omitempty"`
}

// AutopilotServer represents the server blocks in the response of the raft
// autopilot state API.
type AutopilotServer struct {
ID string `mapstructure:"id"`
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
NodeStatus string `mapstructure:"node_status"`
LastContact string `mapstructure:"last_contact"`
LastTerm uint64 `mapstructure:"last_term"`
LastIndex uint64 `mapstructure:"last_index"`
Healthy bool `mapstructure:"healthy"`
StableSince string `mapstructure:"stable_since"`
Status string `mapstructure:"status"`
Meta map[string]string `mapstructure:"meta"`
ID string `mapstructure:"id"`
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
NodeStatus string `mapstructure:"node_status"`
LastContact string `mapstructure:"last_contact"`
LastTerm uint64 `mapstructure:"last_term"`
LastIndex uint64 `mapstructure:"last_index"`
Healthy bool `mapstructure:"healthy"`
StableSince string `mapstructure:"stable_since"`
Status string `mapstructure:"status"`
Version string `mapstructure:"version"`
UpgradeVersion string `mapstructure:"upgrade_version,omitempty"`
RedundancyZone string `mapstructure:"redundancy_zone,omitempty"`
NodeType string `mapstructure:"node_type,omitempty"`
}

type AutopilotZone struct {
Servers []string `mapstructure:"servers,omitempty"`
Voters []string `mapstructure:"voters,omitempty"`
FailureTolerance int `mapstructure:"failure_tolerance,omitempty"`
}

type AutopilotUpgrade struct {
Status string `mapstructure:"status"`
TargetVersion string `mapstructure:"target_version,omitempty"`
TargetVersionVoters []string `mapstructure:"target_version_voters,omitempty"`
TargetVersionNonVoters []string `mapstructure:"target_version_non_voters,omitempty"`
TargetVersionReadReplicas []string `mapstructure:"target_version_read_replicas,omitempty"`
OtherVersionVoters []string `mapstructure:"other_version_voters,omitempty"`
OtherVersionNonVoters []string `mapstructure:"other_version_non_voters,omitempty"`
OtherVersionReadReplicas []string `mapstructure:"other_version_read_replicas,omitempty"`
RedundancyZones map[string]AutopilotZoneUpgradeVersions `mapstructure:"redundancy_zones,omitempty"`
}

type AutopilotZoneUpgradeVersions struct {
TargetVersionVoters []string `mapstructure:"target_version_voters,omitempty"`
TargetVersionNonVoters []string `mapstructure:"target_version_non_voters,omitempty"`
OtherVersionVoters []string `mapstructure:"other_version_voters,omitempty"`
OtherVersionNonVoters []string `mapstructure:"other_version_non_voters,omitempty"`
}

// RaftJoin wraps RaftJoinWithContext using context.Background.
Expand Down
95 changes: 68 additions & 27 deletions command/format.go
View file
Expand Up @@ -171,29 +171,24 @@ func formatServer(srv *api.AutopilotServer) string {
var buffer bytes.Buffer

buffer.WriteString(fmt.Sprintf(" %s\n", srv.ID))
buffer.WriteString(fmt.Sprintf(" Name: %s\n", srv.Name))
buffer.WriteString(fmt.Sprintf(" Address: %s\n", srv.Address))
buffer.WriteString(fmt.Sprintf(" Status: %s\n", srv.Status))
buffer.WriteString(fmt.Sprintf(" Node Status: %s\n", srv.NodeStatus))
buffer.WriteString(fmt.Sprintf(" Healthy: %t\n", srv.Healthy))
buffer.WriteString(fmt.Sprintf(" Last Contact: %s\n", srv.LastContact))
buffer.WriteString(fmt.Sprintf(" Last Term: %d\n", srv.LastTerm))
buffer.WriteString(fmt.Sprintf(" Last Index: %d\n", srv.LastIndex))

if len(srv.Meta) > 0 {
buffer.WriteString(" Meta\n")
var outputs []mapOutput
for k, v := range srv.Meta {
outputs = append(outputs, mapOutput{key: k, value: fmt.Sprintf(" %q: %q\n", k, v)})
}

sort.Slice(outputs, func(i, j int) bool {
return outputs[i].key < outputs[j].key
})
buffer.WriteString(fmt.Sprintf(" Name: %s\n", srv.Name))
buffer.WriteString(fmt.Sprintf(" Address: %s\n", srv.Address))
buffer.WriteString(fmt.Sprintf(" Status: %s\n", srv.Status))
buffer.WriteString(fmt.Sprintf(" Node Status: %s\n", srv.NodeStatus))
buffer.WriteString(fmt.Sprintf(" Healthy: %t\n", srv.Healthy))
buffer.WriteString(fmt.Sprintf(" Last Contact: %s\n", srv.LastContact))
buffer.WriteString(fmt.Sprintf(" Last Term: %d\n", srv.LastTerm))
buffer.WriteString(fmt.Sprintf(" Last Index: %d\n", srv.LastIndex))
buffer.WriteString(fmt.Sprintf(" Version: %s\n", srv.Version))

for _, output := range outputs {
buffer.WriteString(output.value)
}
if srv.UpgradeVersion != "" {
buffer.WriteString(fmt.Sprintf(" Upgrade Version: %s\n", srv.UpgradeVersion))
}
if srv.RedundancyZone != "" {
buffer.WriteString(fmt.Sprintf(" Redundancy Zone: %s\n", srv.RedundancyZone))
}
if srv.NodeType != "" {
buffer.WriteString(fmt.Sprintf(" Node Type: %s\n", srv.NodeType))
}

return buffer.String()
Expand All @@ -203,9 +198,9 @@ func (p PrettyFormatter) OutputAutopilotState(ui cli.Ui, data interface{}) {
state := data.(*api.AutopilotState)

var buffer bytes.Buffer
buffer.WriteString(fmt.Sprintf("Healthy: %t\n", state.Healthy))
buffer.WriteString(fmt.Sprintf("Failure Tolerance: %d\n", state.FailureTolerance))
buffer.WriteString(fmt.Sprintf("Leader: %s\n", state.Leader))
buffer.WriteString(fmt.Sprintf("Healthy: %t\n", state.Healthy))
buffer.WriteString(fmt.Sprintf("Failure Tolerance: %d\n", state.FailureTolerance))
buffer.WriteString(fmt.Sprintf("Leader: %s\n", state.Leader))
buffer.WriteString("Voters:\n")
outputStringSlice(&buffer, " ", state.Voters)

Expand All @@ -214,20 +209,66 @@ func (p PrettyFormatter) OutputAutopilotState(ui cli.Ui, data interface{}) {
outputStringSlice(&buffer, " ", state.NonVoters)
}

if state.OptimisticFailureTolerance > 0 {
buffer.WriteString(fmt.Sprintf("Optimistic Failure Tolerance: %d\n", state.OptimisticFailureTolerance))
}

// Servers
buffer.WriteString("Servers:\n")
var outputs []mapOutput
for id, srv := range state.Servers {
outputs = append(outputs, mapOutput{key: id, value: formatServer(srv)})
}

sort.Slice(outputs, func(i, j int) bool {
return outputs[i].key < outputs[j].key
})

for _, output := range outputs {
buffer.WriteString(output.value)
}

// Redundancy Zones
if len(state.RedundancyZones) > 0 {
buffer.WriteString("Redundancy Zones:\n")
zoneList := make([]string, 0, len(state.RedundancyZones))
for z := range state.RedundancyZones {
zoneList = append(zoneList, z)
}
sort.Strings(zoneList)
for _, zoneName := range zoneList {
zone := state.RedundancyZones[zoneName]
servers := zone.Servers
voters := zone.Voters
sort.Strings(servers)
sort.Strings(voters)
buffer.WriteString(fmt.Sprintf(" %s\n", zoneName))
buffer.WriteString(fmt.Sprintf(" Servers: %s\n", strings.Join(servers, ", ")))
buffer.WriteString(fmt.Sprintf(" Voters: %s\n", strings.Join(voters, ", ")))
buffer.WriteString(fmt.Sprintf(" Failure Tolerance: %d\n", zone.FailureTolerance))
}
}

// Upgrade Info
if state.Upgrade != nil {
buffer.WriteString("Upgrade Info:\n")
buffer.WriteString(fmt.Sprintf(" Status: %s\n", state.Upgrade.Status))
buffer.WriteString(fmt.Sprintf(" Target Version: %s\n", state.Upgrade.TargetVersion))
buffer.WriteString(fmt.Sprintf(" Target Version Voters: %s\n", strings.Join(state.Upgrade.TargetVersionVoters, ", ")))
buffer.WriteString(fmt.Sprintf(" Target Version Non-Voters: %s\n", strings.Join(state.Upgrade.TargetVersionNonVoters, ", ")))
buffer.WriteString(fmt.Sprintf(" Other Version Voters: %s\n", strings.Join(state.Upgrade.OtherVersionVoters, ", ")))
buffer.WriteString(fmt.Sprintf(" Other Version Non-Voters: %s\n", strings.Join(state.Upgrade.OtherVersionNonVoters, ", ")))

if len(state.Upgrade.RedundancyZones) > 0 {
buffer.WriteString(" Redundancy Zones:\n")
for zoneName, zoneVersion := range state.Upgrade.RedundancyZones {
buffer.WriteString(fmt.Sprintf(" %s\n", zoneName))
buffer.WriteString(fmt.Sprintf(" Target Version Voters: %s\n", strings.Join(zoneVersion.TargetVersionVoters, ", ")))
buffer.WriteString(fmt.Sprintf(" Target Version Non-Voters: %s\n", strings.Join(zoneVersion.TargetVersionNonVoters, ", ")))
buffer.WriteString(fmt.Sprintf(" Other Version Voters: %s\n", strings.Join(zoneVersion.OtherVersionVoters, ", ")))
buffer.WriteString(fmt.Sprintf(" Other Version Non-Voters: %s\n", strings.Join(zoneVersion.OtherVersionNonVoters, ", ")))
}
}
}

ui.Output(buffer.String())
}

Expand Down
13 changes: 11 additions & 2 deletions command/operator_members.go
View file
Expand Up @@ -3,6 +3,7 @@ package command
import (
"fmt"
"strings"
"time"

"github.com/mitchellh/cli"
"github.com/posener/complete"
Expand Down Expand Up @@ -70,9 +71,17 @@ func (c *OperatorMembersCommand) Run(args []string) int {

switch Format(c.UI) {
case "table":
out := []string{"Host Name | API Address | Cluster Address | ActiveNode | Last Echo"}
out := make([]string, 0)
cols := []string{"Host Name", "API Address", "Cluster Address", "Active Node", "Version", "Upgrade Version", "Redundancy Zone", "Last Echo"}
out = append(out, strings.Join(cols, " | "))
for _, node := range resp.Nodes {
out = append(out, fmt.Sprintf("%s | %s | %s | %t | %s", node.Hostname, node.APIAddress, node.ClusterAddress, node.ActiveNode, node.LastEcho))
cols := []string{node.Hostname, node.APIAddress, node.ClusterAddress, fmt.Sprintf("%t", node.ActiveNode), node.Version, node.UpgradeVersion, node.RedundancyZone}
if node.LastEcho != nil {
cols = append(cols, node.LastEcho.Format(time.RFC3339))
} else {
cols = append(cols, "")
}
out = append(out, strings.Join(cols, " | "))
}
c.UI.Output(tableOutput(out, nil))
return 0
Expand Down
1 change: 1 addition & 0 deletions command/operator_raft_autopilot_get_config.go
View file
Expand Up @@ -88,6 +88,7 @@ func (c *OperatorRaftAutopilotGetConfigCommand) Run(args []string) int {
entries = append(entries, fmt.Sprintf("%s | %s", "Server Stabilization Time", config.ServerStabilizationTime.String()))
entries = append(entries, fmt.Sprintf("%s | %d", "Min Quorum", config.MinQuorum))
entries = append(entries, fmt.Sprintf("%s | %d", "Max Trailing Logs", config.MaxTrailingLogs))
entries = append(entries, fmt.Sprintf("%s | %t", "Disable Upgrade Migration", config.DisableUpgradeMigration))

return OutputData(c.UI, entries)
}
9 changes: 9 additions & 0 deletions command/operator_raft_autopilot_set_config.go
View file
Expand Up @@ -22,6 +22,7 @@ type OperatorRaftAutopilotSetConfigCommand struct {
flagMaxTrailingLogs uint64
flagMinQuorum uint
flagServerStabilizationTime time.Duration
flagDisableUpgradeMigration BoolPtr
}

func (c *OperatorRaftAutopilotSetConfigCommand) Synopsis() string {
Expand Down Expand Up @@ -73,6 +74,11 @@ func (c *OperatorRaftAutopilotSetConfigCommand) Flags() *FlagSets {
Target: &c.flagServerStabilizationTime,
})

f.BoolPtrVar(&BoolPtrVar{
Name: "disable-upgrade-migration",
Target: &c.flagDisableUpgradeMigration,
})

return set
}

Expand Down Expand Up @@ -125,6 +131,9 @@ func (c *OperatorRaftAutopilotSetConfigCommand) Run(args []string) int {
if c.flagServerStabilizationTime > 0 {
data["server_stabilization_time"] = c.flagServerStabilizationTime.String()
}
if c.flagDisableUpgradeMigration.IsSet() {
data["disable_upgrade_migration"] = c.flagDisableUpgradeMigration.Get()
}

secret, err := client.Logical().Write("sys/storage/raft/autopilot/configuration", data)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion physical/raft/fsm.go
View file
Expand Up @@ -14,7 +14,7 @@ import (
"sync/atomic"
"time"

metrics "github.com/armon/go-metrics"
"github.com/armon/go-metrics"
"github.com/golang/protobuf/proto"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
Expand Down

0 comments on commit 1a87e35

Please sign in to comment.