diff --git a/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go index 757e26882d5e..8482aea880b2 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go @@ -116,6 +116,7 @@ type AdmissionMetrics struct { controller *metricSet webhook *metricSet webhookRejection *metrics.CounterVec + webhookFailOpen *metrics.CounterVec webhookRequest *metrics.CounterVec } @@ -196,6 +197,16 @@ func newAdmissionMetrics() *AdmissionMetrics { }, []string{"name", "type", "operation", "error_type", "rejection_code"}) + webhookFailOpen := metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "webhook_fail_open_count", + Help: "Admission webhook fail open count, identified by name and broken out for each admission type (validating or mutating).", + StabilityLevel: metrics.ALPHA, + }, + []string{"name", "type"}) + webhookRequest := metrics.NewCounterVec( &metrics.CounterOpts{ Namespace: namespace, @@ -210,8 +221,9 @@ func newAdmissionMetrics() *AdmissionMetrics { controller.mustRegister() webhook.mustRegister() legacyregistry.MustRegister(webhookRejection) + legacyregistry.MustRegister(webhookFailOpen) legacyregistry.MustRegister(webhookRequest) - return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookRequest: webhookRequest} + return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest} } func (m *AdmissionMetrics) reset() { @@ -250,6 +262,11 @@ func (m *AdmissionMetrics) ObserveWebhookRejection(ctx context.Context, name, st m.webhookRejection.WithContext(ctx).WithLabelValues(name, stepType, operation, string(errorType), strconv.Itoa(rejectionCode)).Inc() } +// ObserveWebhookFailOpen records validating or mutating webhook that fail open. +func (m *AdmissionMetrics) ObserveWebhookFailOpen(ctx context.Context, name, stepType string) { + m.webhookFailOpen.WithContext(ctx).WithLabelValues(name, stepType).Inc() +} + type metricSet struct { latencies *metrics.HistogramVec latenciesSummary *metrics.SummaryVec diff --git a/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics_test.go index 6eb5a77430d9..1026b04b39b3 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics_test.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics_test.go @@ -159,6 +159,23 @@ func TestObserveWebhookRejection(t *testing.T) { expectCounterValue(t, "apiserver_admission_webhook_rejection_count", wantLabelsAPIServerInternalError, 1) } +func TestObserveWebhookFailOpen(t *testing.T) { + defer Metrics.reset() + defer legacyregistry.Reset() + Metrics.ObserveWebhookFailOpen(context.TODO(), "x", stepAdmit) + Metrics.ObserveWebhookFailOpen(context.TODO(), "x", stepValidate) + wantLabelsCounterAdmit := map[string]string{ + "name": "x", + "type": "admit", + } + wantLabelsCounterValidate := map[string]string{ + "name": "x", + "type": "validate", + } + expectCounterValue(t, "apiserver_admission_webhook_fail_open_count", wantLabelsCounterAdmit, 1) + expectCounterValue(t, "apiserver_admission_webhook_fail_open_count", wantLabelsCounterValidate, 1) +} + func TestWithMetrics(t *testing.T) { defer Metrics.reset() defer legacyregistry.Reset() diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go index 16642c183e22..914fe6e0b28b 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go @@ -178,7 +178,7 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok { if ignoreClientCallFailures { klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr) - + admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit") annotator.addFailedOpenAnnotation() utilruntime.HandleError(callErr) diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go index 56099695a6cb..dfa79e8b28d8 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go @@ -140,7 +140,7 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok { if ignoreClientCallFailures { klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr) - + admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "validating") key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx) value := hook.Name if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {