Merge pull request #107171 from ltagliamonte-dd/add_failopen_metrics

add failopen metric
kubernetes · Jan 7, 2022 · 9d0d2e8 · 9d0d2e8
2 parents 2af53e9 + 0844891
commit 9d0d2e8
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 3 deletions.
diff --git a/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go
@@ -116,6 +116,7 @@ type AdmissionMetrics struct {
 	controller       *metricSet
 	webhook          *metricSet
 	webhookRejection *metrics.CounterVec
+	webhookFailOpen  *metrics.CounterVec
 	webhookRequest   *metrics.CounterVec
 }
 
@@ -196,6 +197,16 @@ func newAdmissionMetrics() *AdmissionMetrics {
 		},
 		[]string{"name", "type", "operation", "error_type", "rejection_code"})
 
+	webhookFailOpen := metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "webhook_fail_open_count",
+			Help:           "Admission webhook fail open count, identified by name and broken out for each admission type (validating or mutating).",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"name", "type"})
+
 	webhookRequest := metrics.NewCounterVec(
 		&metrics.CounterOpts{
 			Namespace:      namespace,
@@ -210,8 +221,9 @@ func newAdmissionMetrics() *AdmissionMetrics {
 	controller.mustRegister()
 	webhook.mustRegister()
 	legacyregistry.MustRegister(webhookRejection)
+	legacyregistry.MustRegister(webhookFailOpen)
 	legacyregistry.MustRegister(webhookRequest)
-	return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookRequest: webhookRequest}
+	return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest}
 }
 
 func (m *AdmissionMetrics) reset() {
@@ -250,6 +262,11 @@ func (m *AdmissionMetrics) ObserveWebhookRejection(ctx context.Context, name, st
 	m.webhookRejection.WithContext(ctx).WithLabelValues(name, stepType, operation, string(errorType), strconv.Itoa(rejectionCode)).Inc()
 }
 
+// ObserveWebhookFailOpen records validating or mutating webhook that fail open.
+func (m *AdmissionMetrics) ObserveWebhookFailOpen(ctx context.Context, name, stepType string) {
+	m.webhookFailOpen.WithContext(ctx).WithLabelValues(name, stepType).Inc()
+}
+
 type metricSet struct {
 	latencies        *metrics.HistogramVec
 	latenciesSummary *metrics.SummaryVec

diff --git a/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics_test.go
@@ -159,6 +159,23 @@ func TestObserveWebhookRejection(t *testing.T) {
 	expectCounterValue(t, "apiserver_admission_webhook_rejection_count", wantLabelsAPIServerInternalError, 1)
 }
 
+func TestObserveWebhookFailOpen(t *testing.T) {
+	defer Metrics.reset()
+	defer legacyregistry.Reset()
+	Metrics.ObserveWebhookFailOpen(context.TODO(), "x", stepAdmit)
+	Metrics.ObserveWebhookFailOpen(context.TODO(), "x", stepValidate)
+	wantLabelsCounterAdmit := map[string]string{
+		"name": "x",
+		"type": "admit",
+	}
+	wantLabelsCounterValidate := map[string]string{
+		"name": "x",
+		"type": "validate",
+	}
+	expectCounterValue(t, "apiserver_admission_webhook_fail_open_count", wantLabelsCounterAdmit, 1)
+	expectCounterValue(t, "apiserver_admission_webhook_fail_open_count", wantLabelsCounterValidate, 1)
+}
+
 func TestWithMetrics(t *testing.T) {
 	defer Metrics.reset()
 	defer legacyregistry.Reset()

diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go
@@ -178,7 +178,7 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
 		if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
 			if ignoreClientCallFailures {
 				klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
-
+				admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit")
 				annotator.addFailedOpenAnnotation()
 
 				utilruntime.HandleError(callErr)

diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go
@@ -140,7 +140,7 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
 			if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
 				if ignoreClientCallFailures {
 					klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
-
+					admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "validating")
 					key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
 					value := hook.Name
 					if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {