Add metrics for perf standby and replication node type. (#11472)

helper/testhelpers/testhelpers.go

@@ -3,15 +3,18 @@ package testhelpers
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net/url"
 	"sync/atomic"
 	"time"
 
-	raftlib "github.com/hashicorp/raft"
 	"github.com/hashicorp/vault/api"
+
+	raftlib "github.com/hashicorp/raft"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/xor"
 	"github.com/hashicorp/vault/physical/raft"
@@ -654,3 +657,34 @@ func VerifyRaftPeers(t testing.T, client *api.Client, expected map[string]bool)
 		t.Fatalf("failed to read configuration successfully, expected peers no found in configuration list: %v", expected)
 	}
 }
+
+func SysMetricsReq(client *api.Client, cluster *vault.TestCluster, unauth bool) (*SysMetricsJSON, error) {
+	r := client.NewRequest("GET", "/v1/sys/metrics")
+	if !unauth {
+		r.Headers.Set("X-Vault-Token", cluster.RootToken)
+	}
+	var data SysMetricsJSON
+	resp, err := client.RawRequestWithContext(context.Background(), r)
+	if err != nil {
+		return nil, err
+	}
+	bodyBytes, err := ioutil.ReadAll(resp.Response.Body)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if err := json.Unmarshal(bodyBytes, &data); err != nil {
+		return nil, errors.New("failed to unmarshal:" + err.Error())
+	}
+	return &data, nil
+}
+
+type SysMetricsJSON struct {
+	Gauges []GaugeJSON `json:"Gauges"`
+}
+
+type GaugeJSON struct {
+	Name   string                 `json:"Name"`
+	Value  int                    `json:"Value"`
+	Labels map[string]interface{} `json:"Labels"`
+}

vault/core_metrics.go

@@ -18,7 +18,7 @@ func (c *Core) metricsLoop(stopCh chan struct{}) {
 	emitTimer := time.Tick(time.Second)
 
 	identityCountTimer := time.Tick(time.Minute * 10)
-	// Only emit on active node of cluster that is not a DR cecondary.
+	// Only emit on active node of cluster that is not a DR secondary.
 	if standby, _ := c.Standby(); standby || c.IsDRSecondary() {
 		identityCountTimer = nil
 	}
@@ -60,6 +60,36 @@ func (c *Core) metricsLoop(stopCh chan struct{}) {
 				c.metricSink.SetGaugeWithLabels([]string{"core", "active"}, 1, nil)
 			}
 
+			if perfStandby := c.PerfStandby(); perfStandby {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "performance_standby"}, 1, nil)
+			} else {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "performance_standby"}, 0, nil)
+			}
+
+			if c.ReplicationState().HasState(consts.ReplicationPerformancePrimary) {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "performance", "primary"}, 1, nil)
+			} else {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "performance", "primary"}, 0, nil)
+			}
+
+			if c.IsPerfSecondary() {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "performance", "secondary"}, 1, nil)
+			} else {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "performance", "secondary"}, 0, nil)
+			}
+
+			if c.ReplicationState().HasState(consts.ReplicationDRPrimary) {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "dr", "primary"}, 1, nil)
+			} else {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "dr", "primary"}, 0, nil)
+			}
+
+			if c.IsDRSecondary() {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "dr", "secondary"}, 1, nil)
+			} else {
+				c.metricSink.SetGaugeWithLabels([]string{"core", "replication", "dr", "secondary"}, 0, nil)
+			}
+
 			// Refresh gauge metrics that are looped
 			c.cachedGaugeMetricsEmitter()
 

vault/external_tests/metrics/core_metrics_int_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/armon/go-metrics"
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/helper/metricsutil"
+	"github.com/hashicorp/vault/helper/testhelpers"
 	vaulthttp "github.com/hashicorp/vault/http"
 	"github.com/hashicorp/vault/vault"
 )
@@ -39,7 +40,7 @@ func TestMountTableMetrics(t *testing.T) {
 
 	// Verify that the nonlocal logical mount table has 3 entries -- cubbyhole, identity, and kv
 
-	data, err := sysMetricsReq(client, cluster)
+	data, err := testhelpers.SysMetricsReq(client, cluster, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -59,7 +60,7 @@ func TestMountTableMetrics(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	data, err = sysMetricsReq(client, cluster)
+	data, err = testhelpers.SysMetricsReq(client, cluster, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -74,28 +75,7 @@ func TestMountTableMetrics(t *testing.T) {
 	}
 }
 
-func sysMetricsReq(client *api.Client, cluster *vault.TestCluster) (*SysMetricsJSON, error) {
-	r := client.NewRequest("GET", "/v1/sys/metrics")
-	r.Headers.Set("X-Vault-Token", cluster.RootToken)
-	var data SysMetricsJSON
-	mountAddResp, err := client.RawRequestWithContext(context.Background(), r)
-	if err != nil {
-		return nil, err
-	}
-	bodyBytes, err := ioutil.ReadAll(mountAddResp.Response.Body)
-	if err != nil {
-		return nil, err
-	}
-	if mountAddResp != nil {
-		defer mountAddResp.Body.Close()
-	}
-	if err := json.Unmarshal(bodyBytes, &data); err != nil {
-		return nil, errors.New("failed to unmarshal:" + err.Error())
-	}
-	return &data, nil
-}
-
-func gaugeSearchHelper(data *SysMetricsJSON, expectedValue int) (int, error) {
+func gaugeSearchHelper(data *testhelpers.SysMetricsJSON, expectedValue int) (int, error) {
 	foundFlag := false
 	tablesize := int(^uint(0) >> 1)
 	for _, gauge := range data.Gauges {
@@ -173,7 +153,7 @@ func TestLeaderReElectionMetrics(t *testing.T) {
 	if respo != nil {
 		defer respo.Body.Close()
 	}
-	var data SysMetricsJSON
+	var data testhelpers.SysMetricsJSON
 	var coreLeaderMetric bool = false
 	var coreUnsealMetric bool = false
 	if err := json.Unmarshal(bodyBytes, &data); err != nil {
@@ -241,13 +221,3 @@ func TestLeaderReElectionMetrics(t *testing.T) {
 		defer respo.Body.Close()
 	}
 }
-
-type SysMetricsJSON struct {
-	Gauges []GaugeJSON `json:"Gauges"`
-}
-
-type GaugeJSON struct {
-	Name   string                 `json:"Name"`
-	Value  int                    `json:"Value"`
-	Labels map[string]interface{} `json:"Labels"`
-}