From 4b6013a6054376413951fb238b652806f6d1196f Mon Sep 17 00:00:00 2001 From: Kevin Delgado Date: Sat, 20 Mar 2021 01:00:26 +0000 Subject: [PATCH] StandaloneWebhook can run on any arbitrary mux --- pkg/webhook/admission/webhook.go | 64 +++++++++++++++++++++++++ pkg/webhook/server.go | 27 +---------- pkg/webhook/webhook_integration_test.go | 57 +++++++++++++++++++++- 3 files changed, 122 insertions(+), 26 deletions(-) diff --git a/pkg/webhook/admission/webhook.go b/pkg/webhook/admission/webhook.go index 4430c3132c..41de16b9a7 100644 --- a/pkg/webhook/admission/webhook.go +++ b/pkg/webhook/admission/webhook.go @@ -22,13 +22,18 @@ import ( "net/http" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" jsonpatch "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/kubernetes/scheme" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics" ) var ( @@ -203,3 +208,62 @@ func (w *Webhook) InjectFunc(f inject.Func) error { return setFields(w.Handler) } + +// InstrumentedHook adds some instrumentation on top of the given webhook. +func InstrumentedHook(path string, hookRaw http.Handler) http.Handler { + lbl := prometheus.Labels{"webhook": path} + + lat := metrics.RequestLatency.MustCurryWith(lbl) + cnt := metrics.RequestTotal.MustCurryWith(lbl) + gge := metrics.RequestInFlight.With(lbl) + + // Initialize the most likely HTTP status codes. + cnt.WithLabelValues("200") + cnt.WithLabelValues("500") + + return promhttp.InstrumentHandlerDuration( + lat, + promhttp.InstrumentHandlerCounter( + cnt, + promhttp.InstrumentHandlerInFlight(gge, hookRaw), + ), + ) +} + +// StandaloneOptions let you configure a StandaloneWebhook. +type StandaloneOptions struct { + // Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources + // Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better + // idea to pass your own scheme in. See the documentation in pkg/scheme for more information. + Scheme *runtime.Scheme + // Logger to be used by the webhook. + // If none is set, it defaults to log.Log global logger. + Logger logr.Logger + // Path the webhook will be served at. + // Used for labelling prometheus metrics. + Path string +} + +// StandaloneWebhook transforms a Webhook that needs to be registered +// on a webhook.Server into one that can be ran on any arbitrary mux. +func StandaloneWebhook(hook *Webhook, opts StandaloneOptions) (http.Handler, error) { + if opts.Scheme == nil { + opts.Scheme = scheme.Scheme + } + + var err error + hook.decoder, err = NewDecoder(opts.Scheme) + if err != nil { + return nil, err + } + + if opts.Logger == nil { + opts.Logger = logf.RuntimeLog.WithName("webhook") + } + hook.log = opts.Logger + + if opts.Path == "" { + return hook, nil + } + return InstrumentedHook(opts.Path, hook), nil +} diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index bc7e19a9a3..608d469d86 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -29,13 +29,11 @@ import ( "strconv" "sync" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "sigs.k8s.io/controller-runtime/pkg/webhook/internal/certwatcher" - "sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics" ) // DefaultPort is the default port that the webhook server serves. @@ -187,7 +185,7 @@ func (s *Server) Register(path string, hook http.Handler) { } // TODO(directxman12): call setfields if we've already started the server s.webhooks[path] = hook - s.WebhookMux.Handle(path, instrumentedHook(path, hook)) + s.WebhookMux.Handle(path, admission.InstrumentedHook(path, hook)) regLog := log.WithValues("path", path) regLog.Info("registering webhook") @@ -212,27 +210,6 @@ func (s *Server) Register(path string, hook http.Handler) { } } -// instrumentedHook adds some instrumentation on top of the given webhook. -func instrumentedHook(path string, hookRaw http.Handler) http.Handler { - lbl := prometheus.Labels{"webhook": path} - - lat := metrics.RequestLatency.MustCurryWith(lbl) - cnt := metrics.RequestTotal.MustCurryWith(lbl) - gge := metrics.RequestInFlight.With(lbl) - - // Initialize the most likely HTTP status codes. - cnt.WithLabelValues("200") - cnt.WithLabelValues("500") - - return promhttp.InstrumentHandlerDuration( - lat, - promhttp.InstrumentHandlerCounter( - cnt, - promhttp.InstrumentHandlerInFlight(gge, hookRaw), - ), - ) -} - // Start runs the server. // It will install the webhook related resources depend on the server configuration. func (s *Server) Start(ctx context.Context) error { diff --git a/pkg/webhook/webhook_integration_test.go b/pkg/webhook/webhook_integration_test.go index 0aa5207eb1..cbb6ebc037 100644 --- a/pkg/webhook/webhook_integration_test.go +++ b/pkg/webhook/webhook_integration_test.go @@ -2,7 +2,12 @@ package webhook_test import ( "context" + "crypto/tls" "fmt" + "net" + "net/http" + "path/filepath" + "strconv" "time" . "github.com/onsi/ginkgo" @@ -15,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "sigs.k8s.io/controller-runtime/pkg/webhook/internal/certwatcher" ) var _ = Describe("Webhook", func() { @@ -78,7 +84,7 @@ var _ = Describe("Webhook", func() { close(done) }) }) - Context("when running a webhook server without a manager ", func() { + Context("when running a webhook server without a manager", func() { It("should reject create request for webhook that rejects all requests", func(done Done) { opts := webhook.Options{ Port: testenv.WebhookInstallOptions.LocalServingPort, @@ -99,6 +105,55 @@ var _ = Describe("Webhook", func() { return errors.ReasonForError(err) == metav1.StatusReason("Always denied") }, 1*time.Second).Should(BeTrue()) + cancel() + close(done) + }) + }) + Context("when running a standalone webhook", func() { + It("should reject create request for webhook that rejects all requests", func(done Done) { + ctx, cancel := context.WithCancel(context.Background()) + // generate tls cfg + certPath := filepath.Join(testenv.WebhookInstallOptions.LocalServingCertDir, "tls.crt") + keyPath := filepath.Join(testenv.WebhookInstallOptions.LocalServingCertDir, "tls.key") + + certWatcher, err := certwatcher.New(certPath, keyPath) + Expect(err).NotTo(HaveOccurred()) + go func() { + Expect(certWatcher.Start(ctx)).NotTo(HaveOccurred()) + }() + + cfg := &tls.Config{ + NextProtos: []string{"h2"}, + GetCertificate: certWatcher.GetCertificate, + } + + // generate listener + listener, err := tls.Listen("tcp", net.JoinHostPort(testenv.WebhookInstallOptions.LocalServingHost, strconv.Itoa(int(testenv.WebhookInstallOptions.LocalServingPort))), cfg) + Expect(err).NotTo(HaveOccurred()) + + // create and register the standalone webhook + hook, err := admission.StandaloneWebhook(&webhook.Admission{Handler: &rejectingValidator{}}, admission.StandaloneOptions{}) + Expect(err).NotTo(HaveOccurred()) + http.Handle("/failing", hook) + + // run the http server + srv := &http.Server{} + go func() { + idleConnsClosed := make(chan struct{}) + go func() { + <-ctx.Done() + Expect(srv.Shutdown(context.Background())).NotTo(HaveOccurred()) + close(idleConnsClosed) + }() + srv.Serve(listener) + <-idleConnsClosed + }() + + Eventually(func() bool { + err = c.Create(context.TODO(), obj) + return errors.ReasonForError(err) == metav1.StatusReason("Always denied") + }, 1*time.Second).Should(BeTrue()) + cancel() close(done) })