diff --git a/pkg/webhook/admission/webhook.go b/pkg/webhook/admission/webhook.go index 4430c3132c..f0db50be81 100644 --- a/pkg/webhook/admission/webhook.go +++ b/pkg/webhook/admission/webhook.go @@ -27,8 +27,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/kubernetes/scheme" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics" ) var ( @@ -110,6 +113,9 @@ func (f HandlerFunc) Handle(ctx context.Context, req Request) Response { } // Webhook represents each individual webhook. +// +// It must be registered with a webhook.Server or +// populated by StandaloneWebhook to be ran on an arbitrary HTTP server. type Webhook struct { // Handler actually processes an admission request returning whether it was allowed or denied, // and potentially patches to apply to the handler. @@ -203,3 +209,45 @@ func (w *Webhook) InjectFunc(f inject.Func) error { return setFields(w.Handler) } + +// StandaloneOptions let you configure a StandaloneWebhook. +type StandaloneOptions struct { + // Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources + // Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better + // idea to pass your own scheme in. See the documentation in pkg/scheme for more information. + Scheme *runtime.Scheme + // Logger to be used by the webhook. + // If none is set, it defaults to log.Log global logger. + Logger logr.Logger + // MetricsPath is used for labelling prometheus metrics + // by the path is served on. + // If none is set, prometheus metrics will not be generated. + MetricsPath string +} + +// StandaloneWebhook prepares a webhook for use without a webhook.Server, +// passing in the information normally populated by webhook.Server +// and instrumenting the webhook with metrics. +// +// Use this to attach your webhook to an arbitrary HTTP server or mux. +func StandaloneWebhook(hook *Webhook, opts StandaloneOptions) (http.Handler, error) { + if opts.Scheme == nil { + opts.Scheme = scheme.Scheme + } + + var err error + hook.decoder, err = NewDecoder(opts.Scheme) + if err != nil { + return nil, err + } + + if opts.Logger == nil { + opts.Logger = logf.RuntimeLog.WithName("webhook") + } + hook.log = opts.Logger + + if opts.MetricsPath == "" { + return hook, nil + } + return metrics.InstrumentedHook(opts.MetricsPath, hook), nil +} diff --git a/pkg/webhook/example_test.go b/pkg/webhook/example_test.go index b225fea89b..1c15a4455a 100644 --- a/pkg/webhook/example_test.go +++ b/pkg/webhook/example_test.go @@ -18,18 +18,24 @@ package webhook_test import ( "context" + "net/http" + "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" . "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -func Example() { - // Build webhooks +var ( + // Build webhooks used for the various server + // configuration options + // // These handlers could be also be implementations // of the AdmissionHandler interface for more complex // implementations. - mutatingHook := &Admission{ + mutatingHook = &Admission{ Handler: admission.HandlerFunc(func(ctx context.Context, req AdmissionRequest) AdmissionResponse { return Patched("some changes", JSONPatchOp{Operation: "add", Path: "/metadata/annotations/access", Value: "granted"}, @@ -38,12 +44,16 @@ func Example() { }), } - validatingHook := &Admission{ + validatingHook = &Admission{ Handler: admission.HandlerFunc(func(ctx context.Context, req AdmissionRequest) AdmissionResponse { return Denied("none shall pass!") }), } +) +// This example registers a webhooks to a webhook server +// that gets ran by a controller manager. +func Example() { // Create a manager // Note: GetConfigOrDie will os.Exit(1) w/o any message if no kube-config can be found mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) @@ -70,3 +80,72 @@ func Example() { panic(err) } } + +// This example creates a webhook server that can be +// ran without a controller manager. +func ExampleServer_StartStandalone() { + // Create a webhook server + hookServer := &Server{ + Port: 8443, + } + + // Register the webhooks in the server. + hookServer.Register("/mutating", mutatingHook) + hookServer.Register("/validating", validatingHook) + + // Start the server without a manger + err := hookServer.StartStandalone(signals.SetupSignalHandler(), scheme.Scheme) + if err != nil { + // handle error + panic(err) + } +} + +// This example creates a standalone webhook handler +// and runs it on a vanilla go HTTP server to demonstrate +// how you could run a webhook on an existing server +// without a controller manager. +func ExampleStandaloneWebhook() { + // Assume you have an existing HTTP server at your disposal + // configured as desired (e.g. with TLS). + // For this example just create a basic http.ServeMux + mux := http.NewServeMux() + port := ":8000" + + // Create the standalone HTTP handlers from our webhooks + mutatingHookHandler, err := admission.StandaloneWebhook(mutatingHook, admission.StandaloneOptions{ + Scheme: scheme.Scheme, + // Logger let's you optionally pass + // a custom logger (defaults to log.Log global Logger) + Logger: logf.RuntimeLog.WithName("mutating-webhook"), + // MetricsPath let's you optionally + // provide the path it will be served on + // to be used for labelling prometheus metrics + // If none is set, prometheus metrics will not be generated. + MetricsPath: "/mutating", + }) + if err != nil { + // handle error + panic(err) + } + + validatingHookHandler, err := admission.StandaloneWebhook(validatingHook, admission.StandaloneOptions{ + Scheme: scheme.Scheme, + Logger: logf.RuntimeLog.WithName("validating-webhook"), + MetricsPath: "/validating", + }) + if err != nil { + // handle error + panic(err) + } + + // Register the webhook handlers to your server + mux.Handle("/mutating", mutatingHookHandler) + mux.Handle("/validating", validatingHookHandler) + + // Run your handler + if err := http.ListenAndServe(port, mux); err != nil { + panic(err) + } + +} diff --git a/pkg/webhook/internal/metrics/metrics.go b/pkg/webhook/internal/metrics/metrics.go index a29643b244..557004908b 100644 --- a/pkg/webhook/internal/metrics/metrics.go +++ b/pkg/webhook/internal/metrics/metrics.go @@ -17,7 +17,10 @@ limitations under the License. package metrics import ( + "net/http" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -59,3 +62,24 @@ var ( func init() { metrics.Registry.MustRegister(RequestLatency, RequestTotal, RequestInFlight) } + +// InstrumentedHook adds some instrumentation on top of the given webhook. +func InstrumentedHook(path string, hookRaw http.Handler) http.Handler { + lbl := prometheus.Labels{"webhook": path} + + lat := RequestLatency.MustCurryWith(lbl) + cnt := RequestTotal.MustCurryWith(lbl) + gge := RequestInFlight.With(lbl) + + // Initialize the most likely HTTP status codes. + cnt.WithLabelValues("200") + cnt.WithLabelValues("500") + + return promhttp.InstrumentHandlerDuration( + lat, + promhttp.InstrumentHandlerCounter( + cnt, + promhttp.InstrumentHandlerInFlight(gge, hookRaw), + ), + ) +} diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index 9fefc9a697..94a8fda54f 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -29,8 +29,8 @@ import ( "strconv" "sync" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/apimachinery/pkg/runtime" + kscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics" @@ -124,7 +124,7 @@ func (s *Server) Register(path string, hook http.Handler) { } // TODO(directxman12): call setfields if we've already started the server s.webhooks[path] = hook - s.WebhookMux.Handle(path, instrumentedHook(path, hook)) + s.WebhookMux.Handle(path, metrics.InstrumentedHook(path, hook)) regLog := log.WithValues("path", path) regLog.Info("registering webhook") @@ -149,25 +149,24 @@ func (s *Server) Register(path string, hook http.Handler) { } } -// instrumentedHook adds some instrumentation on top of the given webhook. -func instrumentedHook(path string, hookRaw http.Handler) http.Handler { - lbl := prometheus.Labels{"webhook": path} - - lat := metrics.RequestLatency.MustCurryWith(lbl) - cnt := metrics.RequestTotal.MustCurryWith(lbl) - gge := metrics.RequestInFlight.With(lbl) - - // Initialize the most likely HTTP status codes. - cnt.WithLabelValues("200") - cnt.WithLabelValues("500") - - return promhttp.InstrumentHandlerDuration( - lat, - promhttp.InstrumentHandlerCounter( - cnt, - promhttp.InstrumentHandlerInFlight(gge, hookRaw), - ), - ) +// StartStandalone runs a webhook server without +// a controller manager. +func (s *Server) StartStandalone(ctx context.Context, scheme *runtime.Scheme) error { + // Use the Kubernetes client-go scheme if none is specified + if scheme == nil { + scheme = kscheme.Scheme + } + + if err := s.InjectFunc(func(i interface{}) error { + if _, err := inject.SchemeInto(scheme, i); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + return s.Start(ctx) } // Start runs the server. diff --git a/pkg/webhook/server_test.go b/pkg/webhook/server_test.go index eccc438ef4..1426b4dec9 100644 --- a/pkg/webhook/server_test.go +++ b/pkg/webhook/server_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -68,12 +69,12 @@ var _ = Describe("Webhook Server", func() { Expect(servingOpts.Cleanup()).To(Succeed()) }) - startServer := func() (done <-chan struct{}) { + genericStartServer := func(f func(ctx context.Context)) (done <-chan struct{}) { doneCh := make(chan struct{}) go func() { defer GinkgoRecover() defer close(doneCh) - Expect(server.Start(ctx)).To(Succeed()) + f(ctx) }() // wait till we can ping the server to start the test Eventually(func() error { @@ -93,6 +94,12 @@ var _ = Describe("Webhook Server", func() { return doneCh } + startServer := func() (done <-chan struct{}) { + return genericStartServer(func(ctx context.Context) { + Expect(server.Start(ctx)).To(Succeed()) + }) + } + // TODO(directxman12): figure out a good way to test all the serving setup // with httptest.Server to get all the niceness from that. @@ -174,6 +181,28 @@ var _ = Describe("Webhook Server", func() { Expect(handler.injectedField).To(BeTrue()) }) }) + + It("should serve be able to serve in unmanaged mode", func() { + server = &webhook.Server{ + Host: servingOpts.LocalServingHost, + Port: servingOpts.LocalServingPort, + CertDir: servingOpts.LocalServingCertDir, + } + server.Register("/somepath", &testHandler{}) + doneCh := genericStartServer(func(ctx context.Context) { + Expect(server.StartStandalone(ctx, scheme.Scheme)) + }) + + Eventually(func() ([]byte, error) { + resp, err := client.Get(fmt.Sprintf("https://%s/somepath", testHostPort)) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + return ioutil.ReadAll(resp.Body) + }).Should(Equal([]byte("gadzooks!"))) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) + }) }) type testHandler struct { diff --git a/pkg/webhook/webhook_integration_test.go b/pkg/webhook/webhook_integration_test.go new file mode 100644 index 0000000000..2ab7b4e24f --- /dev/null +++ b/pkg/webhook/webhook_integration_test.go @@ -0,0 +1,169 @@ +package webhook_test + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "path/filepath" + "strconv" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +var _ = Describe("Webhook", func() { + var c client.Client + var obj *appsv1.Deployment + BeforeEach(func() { + Expect(cfg).NotTo(BeNil()) + var err error + c, err = client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + + obj = &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + Namespace: "default", + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"foo": "bar"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"foo": "bar"}}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + } + }) + Context("when running a webhook server with a manager", func() { + It("should reject create request for webhook that rejects all requests", func(done Done) { + m, err := manager.New(cfg, manager.Options{ + Port: testenv.WebhookInstallOptions.LocalServingPort, + Host: testenv.WebhookInstallOptions.LocalServingHost, + CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + }) // we need manager here just to leverage manager.SetFields + Expect(err).NotTo(HaveOccurred()) + server := m.GetWebhookServer() + server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{}}) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + _ = server.Start(ctx) + }() + + Eventually(func() bool { + err = c.Create(context.TODO(), obj) + return errors.ReasonForError(err) == metav1.StatusReason("Always denied") + }, 1*time.Second).Should(BeTrue()) + + cancel() + close(done) + }) + }) + Context("when running a webhook server without a manager", func() { + It("should reject create request for webhook that rejects all requests", func(done Done) { + server := webhook.Server{ + Port: testenv.WebhookInstallOptions.LocalServingPort, + Host: testenv.WebhookInstallOptions.LocalServingHost, + CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + } + server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{}}) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + _ = server.StartStandalone(ctx, scheme.Scheme) + }() + + Eventually(func() bool { + err := c.Create(context.TODO(), obj) + return errors.ReasonForError(err) == metav1.StatusReason("Always denied") + }, 1*time.Second).Should(BeTrue()) + + cancel() + close(done) + }) + }) + Context("when running a standalone webhook", func() { + It("should reject create request for webhook that rejects all requests", func(done Done) { + ctx, cancel := context.WithCancel(context.Background()) + // generate tls cfg + certPath := filepath.Join(testenv.WebhookInstallOptions.LocalServingCertDir, "tls.crt") + keyPath := filepath.Join(testenv.WebhookInstallOptions.LocalServingCertDir, "tls.key") + + certWatcher, err := certwatcher.New(certPath, keyPath) + Expect(err).NotTo(HaveOccurred()) + go func() { + Expect(certWatcher.Start(ctx)).NotTo(HaveOccurred()) + }() + + cfg := &tls.Config{ + NextProtos: []string{"h2"}, + GetCertificate: certWatcher.GetCertificate, + } + + // generate listener + listener, err := tls.Listen("tcp", + net.JoinHostPort(testenv.WebhookInstallOptions.LocalServingHost, + strconv.Itoa(int(testenv.WebhookInstallOptions.LocalServingPort))), cfg) + Expect(err).NotTo(HaveOccurred()) + + // create and register the standalone webhook + hook, err := admission.StandaloneWebhook(&webhook.Admission{Handler: &rejectingValidator{}}, admission.StandaloneOptions{}) + Expect(err).NotTo(HaveOccurred()) + http.Handle("/failing", hook) + + // run the http server + srv := &http.Server{} + go func() { + idleConnsClosed := make(chan struct{}) + go func() { + <-ctx.Done() + Expect(srv.Shutdown(context.Background())).NotTo(HaveOccurred()) + close(idleConnsClosed) + }() + _ = srv.Serve(listener) + <-idleConnsClosed + }() + + Eventually(func() bool { + err = c.Create(context.TODO(), obj) + return errors.ReasonForError(err) == metav1.StatusReason("Always denied") + }, 1*time.Second).Should(BeTrue()) + + cancel() + close(done) + }) + }) +}) + +type rejectingValidator struct { +} + +func (v *rejectingValidator) Handle(ctx context.Context, req admission.Request) admission.Response { + return admission.Denied(fmt.Sprint("Always denied")) +} diff --git a/pkg/webhook/webhook_suite_test.go b/pkg/webhook/webhook_suite_test.go index d921493f0e..fb2b02f195 100644 --- a/pkg/webhook/webhook_suite_test.go +++ b/pkg/webhook/webhook_suite_test.go @@ -17,11 +17,17 @@ limitations under the License. package webhook_test import ( + "fmt" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -33,8 +39,70 @@ func TestSource(t *testing.T) { RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) } +var testenv *envtest.Environment +var cfg *rest.Config + var _ = BeforeSuite(func(done Done) { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + testenv = &envtest.Environment{} + // we're initializing webhook here and not in webhook.go to also test the envtest install code via WebhookOptions + initializeWebhookInEnvironment() + var err error + cfg, err = testenv.Start() + Expect(err).NotTo(HaveOccurred()) close(done) }, 60) + +var _ = AfterSuite(func() { + fmt.Println("stopping?") + Expect(testenv.Stop()).To(Succeed()) +}, 60) + +func initializeWebhookInEnvironment() { + namespacedScopeV1 := admissionv1.NamespacedScope + failedTypeV1 := admissionv1.Fail + equivalentTypeV1 := admissionv1.Equivalent + noSideEffectsV1 := admissionv1.SideEffectClassNone + webhookPathV1 := "/failing" + + testenv.WebhookInstallOptions = envtest.WebhookInstallOptions{ + ValidatingWebhooks: []client.Object{ + &admissionv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deployment-validation-webhook-config", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "ValidatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1beta1", + }, + Webhooks: []admissionv1.ValidatingWebhook{ + { + Name: "deployment-validation.kubebuilder.io", + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{"CREATE", "UPDATE"}, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"deployments"}, + Scope: &namespacedScopeV1, + }, + }, + }, + FailurePolicy: &failedTypeV1, + MatchPolicy: &equivalentTypeV1, + SideEffects: &noSideEffectsV1, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Name: "deployment-validation-service", + Namespace: "default", + Path: &webhookPathV1, + }, + }, + }, + }, + }, + }, + } +}