From 4266ae38d081b6562e86249f933178fabd52033b Mon Sep 17 00:00:00 2001 From: Nick Cabatoff Date: Mon, 26 Apr 2021 13:54:19 -0400 Subject: [PATCH] Add metrics for perf standby and replication node type. (#11472) --- helper/testhelpers/testhelpers.go | 36 ++++++++++++++++- vault/core_metrics.go | 32 ++++++++++++++- .../metrics/core_metrics_int_test.go | 40 +++---------------- 3 files changed, 71 insertions(+), 37 deletions(-) diff --git a/helper/testhelpers/testhelpers.go b/helper/testhelpers/testhelpers.go index 392f9408fc8b3..4d2741512ed87 100644 --- a/helper/testhelpers/testhelpers.go +++ b/helper/testhelpers/testhelpers.go @@ -3,15 +3,18 @@ package testhelpers import ( "context" "encoding/base64" + "encoding/json" "errors" "fmt" + "io/ioutil" "math/rand" "net/url" "sync/atomic" "time" - raftlib "github.com/hashicorp/raft" "github.com/hashicorp/vault/api" + + raftlib "github.com/hashicorp/raft" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/helper/xor" "github.com/hashicorp/vault/physical/raft" @@ -654,3 +657,34 @@ func VerifyRaftPeers(t testing.T, client *api.Client, expected map[string]bool) t.Fatalf("failed to read configuration successfully, expected peers no found in configuration list: %v", expected) } } + +func SysMetricsReq(client *api.Client, cluster *vault.TestCluster, unauth bool) (*SysMetricsJSON, error) { + r := client.NewRequest("GET", "/v1/sys/metrics") + if !unauth { + r.Headers.Set("X-Vault-Token", cluster.RootToken) + } + var data SysMetricsJSON + resp, err := client.RawRequestWithContext(context.Background(), r) + if err != nil { + return nil, err + } + bodyBytes, err := ioutil.ReadAll(resp.Response.Body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if err := json.Unmarshal(bodyBytes, &data); err != nil { + return nil, errors.New("failed to unmarshal:" + err.Error()) + } + return &data, nil +} + +type SysMetricsJSON struct { + Gauges []GaugeJSON `json:"Gauges"` +} + +type GaugeJSON struct { + Name string `json:"Name"` + Value int `json:"Value"` + Labels map[string]interface{} `json:"Labels"` +} diff --git a/vault/core_metrics.go b/vault/core_metrics.go index 04007a23087af..bb9460c9791a7 100644 --- a/vault/core_metrics.go +++ b/vault/core_metrics.go @@ -18,7 +18,7 @@ func (c *Core) metricsLoop(stopCh chan struct{}) { emitTimer := time.Tick(time.Second) identityCountTimer := time.Tick(time.Minute * 10) - // Only emit on active node of cluster that is not a DR cecondary. + // Only emit on active node of cluster that is not a DR secondary. if standby, _ := c.Standby(); standby || c.IsDRSecondary() { identityCountTimer = nil } @@ -60,6 +60,36 @@ func (c *Core) metricsLoop(stopCh chan struct{}) { c.metricSink.SetGaugeWithLabels([]string{"core", "active"}, 1, nil) } + if perfStandby := c.PerfStandby(); perfStandby { + c.metricSink.SetGaugeWithLabels([]string{"core", "performance_standby"}, 1, nil) + } else { + c.metricSink.SetGaugeWithLabels([]string{"core", "performance_standby"}, 0, nil) + } + + if c.ReplicationState().HasState(consts.ReplicationPerformancePrimary) { + c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "performance", "primary"}, 1, nil) + } else { + c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "performance", "primary"}, 0, nil) + } + + if c.IsPerfSecondary() { + c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "performance", "secondary"}, 1, nil) + } else { + c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "performance", "secondary"}, 0, nil) + } + + if c.ReplicationState().HasState(consts.ReplicationDRPrimary) { + c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "dr", "primary"}, 1, nil) + } else { + c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "dr", "primary"}, 0, nil) + } + + if c.IsDRSecondary() { + c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "dr", "secondary"}, 1, nil) + } else { + c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "dr", "secondary"}, 0, nil) + } + // Refresh gauge metrics that are looped c.cachedGaugeMetricsEmitter() diff --git a/vault/external_tests/metrics/core_metrics_int_test.go b/vault/external_tests/metrics/core_metrics_int_test.go index 7f4bd49e8eca9..40c9c8413f8a6 100644 --- a/vault/external_tests/metrics/core_metrics_int_test.go +++ b/vault/external_tests/metrics/core_metrics_int_test.go @@ -11,6 +11,7 @@ import ( "github.com/armon/go-metrics" "github.com/hashicorp/vault/api" "github.com/hashicorp/vault/helper/metricsutil" + "github.com/hashicorp/vault/helper/testhelpers" vaulthttp "github.com/hashicorp/vault/http" "github.com/hashicorp/vault/vault" ) @@ -39,7 +40,7 @@ func TestMountTableMetrics(t *testing.T) { // Verify that the nonlocal logical mount table has 3 entries -- cubbyhole, identity, and kv - data, err := sysMetricsReq(client, cluster) + data, err := testhelpers.SysMetricsReq(client, cluster, false) if err != nil { t.Fatal(err) } @@ -59,7 +60,7 @@ func TestMountTableMetrics(t *testing.T) { t.Fatal(err) } - data, err = sysMetricsReq(client, cluster) + data, err = testhelpers.SysMetricsReq(client, cluster, false) if err != nil { t.Fatal(err) } @@ -74,28 +75,7 @@ func TestMountTableMetrics(t *testing.T) { } } -func sysMetricsReq(client *api.Client, cluster *vault.TestCluster) (*SysMetricsJSON, error) { - r := client.NewRequest("GET", "/v1/sys/metrics") - r.Headers.Set("X-Vault-Token", cluster.RootToken) - var data SysMetricsJSON - mountAddResp, err := client.RawRequestWithContext(context.Background(), r) - if err != nil { - return nil, err - } - bodyBytes, err := ioutil.ReadAll(mountAddResp.Response.Body) - if err != nil { - return nil, err - } - if mountAddResp != nil { - defer mountAddResp.Body.Close() - } - if err := json.Unmarshal(bodyBytes, &data); err != nil { - return nil, errors.New("failed to unmarshal:" + err.Error()) - } - return &data, nil -} - -func gaugeSearchHelper(data *SysMetricsJSON, expectedValue int) (int, error) { +func gaugeSearchHelper(data *testhelpers.SysMetricsJSON, expectedValue int) (int, error) { foundFlag := false tablesize := int(^uint(0) >> 1) for _, gauge := range data.Gauges { @@ -173,7 +153,7 @@ func TestLeaderReElectionMetrics(t *testing.T) { if respo != nil { defer respo.Body.Close() } - var data SysMetricsJSON + var data testhelpers.SysMetricsJSON var coreLeaderMetric bool = false var coreUnsealMetric bool = false if err := json.Unmarshal(bodyBytes, &data); err != nil { @@ -241,13 +221,3 @@ func TestLeaderReElectionMetrics(t *testing.T) { defer respo.Body.Close() } } - -type SysMetricsJSON struct { - Gauges []GaugeJSON `json:"Gauges"` -} - -type GaugeJSON struct { - Name string `json:"Name"` - Value int `json:"Value"` - Labels map[string]interface{} `json:"Labels"` -}