Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Helper for unmanaged webhook server #1429

Merged
merged 6 commits into from Apr 21, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
48 changes: 48 additions & 0 deletions pkg/webhook/admission/webhook.go
Expand Up @@ -27,8 +27,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/client-go/kubernetes/scheme"

logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
)

var (
Expand Down Expand Up @@ -110,6 +113,9 @@ func (f HandlerFunc) Handle(ctx context.Context, req Request) Response {
}

// Webhook represents each individual webhook.
//
// It must be registered with a webhook.Server or
// populated by StandaloneWebhook to be ran on an arbitrary HTTP server.
type Webhook struct {
// Handler actually processes an admission request returning whether it was allowed or denied,
// and potentially patches to apply to the handler.
Expand Down Expand Up @@ -203,3 +209,45 @@ func (w *Webhook) InjectFunc(f inject.Func) error {

return setFields(w.Handler)
}

// StandaloneOptions let you configure a StandaloneWebhook.
type StandaloneOptions struct {
// Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources
// Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better
// idea to pass your own scheme in. See the documentation in pkg/scheme for more information.
Scheme *runtime.Scheme
// Logger to be used by the webhook.
// If none is set, it defaults to log.Log global logger.
Logger logr.Logger
// MetricsPath is used for labelling prometheus metrics
// by the path is served on.
// If none is set, prometheus metrics will not be generated.
MetricsPath string
}

// StandaloneWebhook prepares a webhook for use without a webhook.Server,
// passing in the information normally populated by webhook.Server
// and instrumenting the webhook with metrics.
//
// Use this to attach your webhook to an arbitrary HTTP server or mux.
func StandaloneWebhook(hook *Webhook, opts StandaloneOptions) (http.Handler, error) {
if opts.Scheme == nil {
opts.Scheme = scheme.Scheme
}

var err error
hook.decoder, err = NewDecoder(opts.Scheme)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kevindelgado what actually injects this into the hook.Handler?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤦 It doesn't... because I'm dumb

See #1490

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fyi, fixed now @stevekuznetsov

if err != nil {
return nil, err
}

if opts.Logger == nil {
opts.Logger = logf.RuntimeLog.WithName("webhook")
}
hook.log = opts.Logger

if opts.MetricsPath == "" {
return hook, nil
}
return metrics.InstrumentedHook(opts.MetricsPath, hook), nil
}
87 changes: 83 additions & 4 deletions pkg/webhook/example_test.go
Expand Up @@ -18,18 +18,24 @@ package webhook_test

import (
"context"
"net/http"

"k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
. "sigs.k8s.io/controller-runtime/pkg/webhook"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

func Example() {
// Build webhooks
var (
// Build webhooks used for the various server
// configuration options
//
// These handlers could be also be implementations
// of the AdmissionHandler interface for more complex
// implementations.
mutatingHook := &Admission{
mutatingHook = &Admission{
Handler: admission.HandlerFunc(func(ctx context.Context, req AdmissionRequest) AdmissionResponse {
return Patched("some changes",
JSONPatchOp{Operation: "add", Path: "/metadata/annotations/access", Value: "granted"},
Expand All @@ -38,12 +44,16 @@ func Example() {
}),
}

validatingHook := &Admission{
validatingHook = &Admission{
Handler: admission.HandlerFunc(func(ctx context.Context, req AdmissionRequest) AdmissionResponse {
return Denied("none shall pass!")
}),
}
)

// This example registers a webhooks to a webhook server
// that gets ran by a controller manager.
func Example() {
// Create a manager
// Note: GetConfigOrDie will os.Exit(1) w/o any message if no kube-config can be found
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{})
Expand All @@ -70,3 +80,72 @@ func Example() {
panic(err)
}
}

// This example creates a webhook server that can be
// ran without a controller manager.
func ExampleServer_StartStandalone() {
// Create a webhook server
hookServer := &Server{
Port: 8443,
}

// Register the webhooks in the server.
hookServer.Register("/mutating", mutatingHook)
hookServer.Register("/validating", validatingHook)

// Start the server without a manger
err := hookServer.StartStandalone(signals.SetupSignalHandler(), scheme.Scheme)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kevindelgado one more thing - maybe we could write an e2e for examples to help catch these, but it looks like the cert and key are not optional on this server? When using code similar to this, I get:

time="2021-04-23T20:34:54Z" level=error msg="Failed to serve admission webhooks." error="open /tmp/k8s-webhook-server/serving-certs/tls.crt: no such file or directory"

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that's intentional -- same with the non-standalone server. More broadly, the main point of the webhook server is to help with managing TLS setup properly. If you don't want TLS, use StandaloneWebhook.

We could certainly document that better, though.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if err != nil {
// handle error
panic(err)
}
}

// This example creates a standalone webhook handler
// and runs it on a vanilla go HTTP server to demonstrate
// how you could run a webhook on an existing server
// without a controller manager.
func ExampleStandaloneWebhook() {
// Assume you have an existing HTTP server at your disposal
// configured as desired (e.g. with TLS).
// For this example just create a basic http.ServeMux
mux := http.NewServeMux()
port := ":8000"

// Create the standalone HTTP handlers from our webhooks
mutatingHookHandler, err := admission.StandaloneWebhook(mutatingHook, admission.StandaloneOptions{
Scheme: scheme.Scheme,
// Logger let's you optionally pass
// a custom logger (defaults to log.Log global Logger)
Logger: logf.RuntimeLog.WithName("mutating-webhook"),
// MetricsPath let's you optionally
// provide the path it will be served on
// to be used for labelling prometheus metrics
// If none is set, prometheus metrics will not be generated.
MetricsPath: "/mutating",
})
if err != nil {
// handle error
panic(err)
}

validatingHookHandler, err := admission.StandaloneWebhook(validatingHook, admission.StandaloneOptions{
Scheme: scheme.Scheme,
Logger: logf.RuntimeLog.WithName("validating-webhook"),
MetricsPath: "/validating",
})
if err != nil {
// handle error
panic(err)
}

// Register the webhook handlers to your server
mux.Handle("/mutating", mutatingHookHandler)
mux.Handle("/validating", validatingHookHandler)

// Run your handler
if err := http.ListenAndServe(port, mux); err != nil {
panic(err)
}

}
24 changes: 24 additions & 0 deletions pkg/webhook/internal/metrics/metrics.go
Expand Up @@ -17,7 +17,10 @@ limitations under the License.
package metrics

import (
"net/http"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"

"sigs.k8s.io/controller-runtime/pkg/metrics"
)
Expand Down Expand Up @@ -59,3 +62,24 @@ var (
func init() {
metrics.Registry.MustRegister(RequestLatency, RequestTotal, RequestInFlight)
}

// InstrumentedHook adds some instrumentation on top of the given webhook.
func InstrumentedHook(path string, hookRaw http.Handler) http.Handler {
lbl := prometheus.Labels{"webhook": path}

lat := RequestLatency.MustCurryWith(lbl)
cnt := RequestTotal.MustCurryWith(lbl)
gge := RequestInFlight.With(lbl)

// Initialize the most likely HTTP status codes.
cnt.WithLabelValues("200")
cnt.WithLabelValues("500")

return promhttp.InstrumentHandlerDuration(
lat,
promhttp.InstrumentHandlerCounter(
cnt,
promhttp.InstrumentHandlerInFlight(gge, hookRaw),
),
)
}
43 changes: 21 additions & 22 deletions pkg/webhook/server.go
Expand Up @@ -29,8 +29,8 @@ import (
"strconv"
"sync"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"k8s.io/apimachinery/pkg/runtime"
kscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
Expand Down Expand Up @@ -124,7 +124,7 @@ func (s *Server) Register(path string, hook http.Handler) {
}
// TODO(directxman12): call setfields if we've already started the server
s.webhooks[path] = hook
s.WebhookMux.Handle(path, instrumentedHook(path, hook))
s.WebhookMux.Handle(path, metrics.InstrumentedHook(path, hook))

regLog := log.WithValues("path", path)
regLog.Info("registering webhook")
Expand All @@ -149,25 +149,24 @@ func (s *Server) Register(path string, hook http.Handler) {
}
}

// instrumentedHook adds some instrumentation on top of the given webhook.
func instrumentedHook(path string, hookRaw http.Handler) http.Handler {
lbl := prometheus.Labels{"webhook": path}

lat := metrics.RequestLatency.MustCurryWith(lbl)
cnt := metrics.RequestTotal.MustCurryWith(lbl)
gge := metrics.RequestInFlight.With(lbl)

// Initialize the most likely HTTP status codes.
cnt.WithLabelValues("200")
cnt.WithLabelValues("500")

return promhttp.InstrumentHandlerDuration(
lat,
promhttp.InstrumentHandlerCounter(
cnt,
promhttp.InstrumentHandlerInFlight(gge, hookRaw),
),
)
// StartStandalone runs a webhook server without
// a controller manager.
func (s *Server) StartStandalone(ctx context.Context, scheme *runtime.Scheme) error {
// Use the Kubernetes client-go scheme if none is specified
if scheme == nil {
scheme = kscheme.Scheme
}

if err := s.InjectFunc(func(i interface{}) error {
if _, err := inject.SchemeInto(scheme, i); err != nil {
return err
}
return nil
}); err != nil {
return err
}

return s.Start(ctx)
}

// Start runs the server.
Expand Down
33 changes: 31 additions & 2 deletions pkg/webhook/server_test.go
Expand Up @@ -25,6 +25,7 @@ import (

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/webhook"
Expand Down Expand Up @@ -68,12 +69,12 @@ var _ = Describe("Webhook Server", func() {
Expect(servingOpts.Cleanup()).To(Succeed())
})

startServer := func() (done <-chan struct{}) {
genericStartServer := func(f func(ctx context.Context)) (done <-chan struct{}) {
doneCh := make(chan struct{})
go func() {
defer GinkgoRecover()
defer close(doneCh)
Expect(server.Start(ctx)).To(Succeed())
f(ctx)
}()
// wait till we can ping the server to start the test
Eventually(func() error {
Expand All @@ -93,6 +94,12 @@ var _ = Describe("Webhook Server", func() {
return doneCh
}

startServer := func() (done <-chan struct{}) {
return genericStartServer(func(ctx context.Context) {
Expect(server.Start(ctx)).To(Succeed())
})
}

// TODO(directxman12): figure out a good way to test all the serving setup
// with httptest.Server to get all the niceness from that.

Expand Down Expand Up @@ -174,6 +181,28 @@ var _ = Describe("Webhook Server", func() {
Expect(handler.injectedField).To(BeTrue())
})
})

It("should serve be able to serve in unmanaged mode", func() {
server = &webhook.Server{
Host: servingOpts.LocalServingHost,
Port: servingOpts.LocalServingPort,
CertDir: servingOpts.LocalServingCertDir,
}
server.Register("/somepath", &testHandler{})
doneCh := genericStartServer(func(ctx context.Context) {
Expect(server.StartStandalone(ctx, scheme.Scheme))
})

Eventually(func() ([]byte, error) {
resp, err := client.Get(fmt.Sprintf("https://%s/somepath", testHostPort))
Expect(err).NotTo(HaveOccurred())
defer resp.Body.Close()
return ioutil.ReadAll(resp.Body)
}).Should(Equal([]byte("gadzooks!")))

ctxCancel()
Eventually(doneCh, "4s").Should(BeClosed())
})
})

type testHandler struct {
Expand Down