Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add --profiling flag to kubelet #84397

Closed
wants to merge 1 commit into from
Closed

Add --profiling flag to kubelet #84397

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/kubelet/app/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1155,8 +1155,8 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubele

// start the kubelet server
if enableServer {
go k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth,
enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling, kubeCfg.EnableSystemLogHandler)
go k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth, enableCAdvisorJSONEndpoints,
kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableProfiling, kubeCfg.EnableContentionProfiling, kubeCfg.EnableSystemLogHandler)

}
if kubeCfg.ReadOnlyPort > 0 {
Expand Down
1 change: 1 addition & 0 deletions pkg/kubelet/apis/config/helpers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ var (
"ContainerLogMaxFiles",
"ContainerLogMaxSize",
"ContentType",
"EnableProfiling",
"EnableContentionProfiling",
"EnableControllerAttachDetach",
"EnableDebuggingHandlers",
Expand Down
1 change: 1 addition & 0 deletions pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enableProfiling: true
enableServer: true
enableSystemLogHandler: true
enforceNodeAllocatable:
Expand Down
1 change: 1 addition & 0 deletions pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enableProfiling: true
enableServer: true
enableSystemLogHandler: true
enforceNodeAllocatable:
Expand Down
2 changes: 2 additions & 0 deletions pkg/kubelet/apis/config/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,8 @@ type KubeletConfiguration struct {
// enableDebuggingHandlers enables server endpoints for log collection
// and local running of containers and commands
EnableDebuggingHandlers bool
// enableProfiling enables profiling via web interface host:port/debug/pprof/
EnableProfiling bool
// enableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true.
EnableContentionProfiling bool
// healthzPort is the port of the localhost healthz endpoint (set to 0 to disable)
Expand Down
4 changes: 4 additions & 0 deletions pkg/kubelet/apis/config/v1beta1/defaults.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,4 +239,8 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura
if obj.EnableSystemLogHandler == nil {
obj.EnableSystemLogHandler = utilpointer.BoolPtr(true)
}
// Enable profiling by default in the scheduler
if obj.EnableProfiling == nil {
obj.EnableProfiling = utilpointer.BoolPtr(true)
}
}
6 changes: 6 additions & 0 deletions pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions pkg/kubelet/kubelet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ type Bootstrap interface {
GetConfiguration() kubeletconfiginternal.KubeletConfiguration
BirthCry()
StartGarbageCollection()
ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool)
ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableProfiling, enableContentionProfiling, enableSystemLogHandler bool)
ListenAndServeReadOnly(address net.IP, port uint, enableCAdvisorJSONEndpoints bool)
ListenAndServePodResources()
Run(<-chan kubetypes.PodUpdate)
Expand Down Expand Up @@ -2135,8 +2135,8 @@ func (kl *Kubelet) ResyncInterval() time.Duration {
}

// ListenAndServe runs the kubelet HTTP server.
func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool) {
server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, kl.redirectContainerStreaming, enableSystemLogHandler, kl.criHandler)
func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableProfiling, enableContentionProfiling, enableSystemLogHandler bool) {
server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableProfiling, enableContentionProfiling, kl.redirectContainerStreaming, enableSystemLogHandler, kl.criHandler)
}

// ListenAndServeReadOnly runs the kubelet HTTP server in read-only mode.
Expand Down
80 changes: 45 additions & 35 deletions pkg/kubelet/server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,8 @@ import (
cadvisorv2 "github.com/google/cadvisor/info/v2"
"github.com/google/cadvisor/metrics"
"google.golang.org/grpc"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
"k8s.io/utils/clock"

"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
Expand All @@ -61,6 +58,7 @@ import (
"k8s.io/component-base/logs"
compbasemetrics "k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/api/legacyscheme"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/core/v1/validation"
Expand All @@ -72,11 +70,13 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cri/streaming"
"k8s.io/kubernetes/pkg/kubelet/cri/streaming/portforward"
remotecommandserver "k8s.io/kubernetes/pkg/kubelet/cri/streaming/remotecommand"
"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
"k8s.io/kubernetes/pkg/kubelet/prober"
servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
"k8s.io/kubernetes/pkg/kubelet/server/stats"
kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/kubelet/util"
"k8s.io/utils/clock"
)

const (
Expand Down Expand Up @@ -145,12 +145,13 @@ func ListenAndServeKubeletServer(
auth AuthInterface,
enableCAdvisorJSONEndpoints,
enableDebuggingHandlers,
enableProfiling,
enableContentionProfiling,
redirectContainerStreaming,
enableSystemLogHandler bool,
criHandler http.Handler) {
klog.Infof("Starting to listen on %s:%d", address, port)
handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, redirectContainerStreaming, enableSystemLogHandler, criHandler)
handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableProfiling, enableContentionProfiling, redirectContainerStreaming, enableSystemLogHandler, criHandler)
s := &http.Server{
Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
Handler: &handler,
Expand All @@ -172,7 +173,7 @@ func ListenAndServeKubeletServer(
// ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet.
func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, address net.IP, port uint, enableCAdvisorJSONEndpoints bool) {
klog.V(1).Infof("Starting to listen read-only on %s:%d", address, port)
s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, false, false, false, false, nil)
s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, false, false, false, false, false, nil)

server := &http.Server{
Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
Expand Down Expand Up @@ -225,6 +226,7 @@ func NewServer(
auth AuthInterface,
enableCAdvisorJSONEndpoints,
enableDebuggingHandlers,
enableProfiling,
enableContentionProfiling,
redirectContainerStreaming,
enableSystemLogHandler bool,
Expand All @@ -247,10 +249,16 @@ func NewServer(
// To maintain backward compatibility serve logs only when enableDebuggingHandlers is also enabled
// see https://github.com/kubernetes/kubernetes/pull/87273
server.InstallSystemLogHandler(enableSystemLogHandler)
if enableContentionProfiling {
goruntime.SetBlockProfileRate(1)
if enableProfiling {
server.InstallProfilingHandlers()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Would be nice to say "profiling is disabled" when enableProfiling is false and debuggingHandler is enabled. Can you also add a unit test for this change.

if enableContentionProfiling {
goruntime.SetBlockProfileRate(1)
}
} else {
klog.Info("profiling is disabled")
}
} else {
klog.Info("debugging handlers and profiling are disabled")
server.InstallDebuggingDisabledHandlers()
}
return server
Expand Down Expand Up @@ -501,33 +509,6 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
s.addMetricsBucketMatcher("configz")
configz.InstallHandler(s.restfulCont)

s.addMetricsBucketMatcher("debug")
handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
switch name {
case "profile":
pprof.Profile(resp, req.Request)
case "symbol":
pprof.Symbol(resp, req.Request)
case "cmdline":
pprof.Cmdline(resp, req.Request)
case "trace":
pprof.Trace(resp, req.Request)
default:
pprof.Index(resp, req.Request)
}
}
// Setup pprof handlers.
ws = new(restful.WebService).Path(pprofBasePath)
ws.Route(ws.GET("/{subpath:*}").To(func(req *restful.Request, resp *restful.Response) {
handlePprofEndpoint(req, resp)
})).Doc("pprof endpoint")
s.restfulCont.Add(ws)

// Setup flags handlers.
// so far, only logging related endpoints are considered valid to add for these debug flags.
s.restfulCont.Handle("/debug/flags/v", routes.StringFlagPutHandler(logs.GlogSetter))

// The /runningpods endpoint is used for testing only.
s.addMetricsBucketMatcher("runningpods")
ws = new(restful.WebService)
Expand Down Expand Up @@ -589,6 +570,35 @@ func (s *Server) InstallSystemLogHandler(enableSystemLogHandler bool) {
}
}

// InstallProfilingHandlers registers the HTTP request patterns that provide profiling webservice
func (s *Server) InstallProfilingHandlers() {
s.addMetricsBucketMatcher("debug")
handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
switch name {
case "profile":
pprof.Profile(resp, req.Request)
case "symbol":
pprof.Symbol(resp, req.Request)
case "cmdline":
pprof.Cmdline(resp, req.Request)
case "trace":
pprof.Trace(resp, req.Request)
default:
pprof.Index(resp, req.Request)
}
}

// Setup pprof handlers.
ws := new(restful.WebService).Path(pprofBasePath)
ws.Route(ws.GET("/{subpath:*}").To(handlePprofEndpoint)).Doc("pprof endpoint")
s.restfulCont.Add(ws)

// Setup flags handlers.
// so far, only logging related endpoints are considered valid to add for these debug flags.
s.restfulCont.Handle("/debug/flags/v", routes.StringFlagPutHandler(logs.GlogSetter))
}

// Checks if kubelet's sync loop that updates containers is working.
func (s *Server) syncLoopHealthCheck(req *http.Request) error {
duration := s.host.ResyncInterval() * 2
Expand Down
30 changes: 19 additions & 11 deletions pkg/kubelet/server/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,14 +302,14 @@ type serverTestFramework struct {
}

func newServerTest() *serverTestFramework {
return newServerTestWithDebug(true, false, nil)
return newServerTestWithDebug(true, true, false, nil)
}

func newServerTestWithDebug(enableDebugging, redirectContainerStreaming bool, streamingServer streaming.Server) *serverTestFramework {
return newServerTestWithDebuggingHandlers(enableDebugging, enableDebugging, redirectContainerStreaming, streamingServer)
func newServerTestWithDebug(enableDebugging, enableProfiling, redirectContainerStreaming bool, streamingServer streaming.Server) *serverTestFramework {
return newServerTestWithDebuggingHandlers(enableDebugging, enableProfiling, enableDebugging, redirectContainerStreaming, streamingServer)
}

func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler, redirectContainerStreaming bool,
func newServerTestWithDebuggingHandlers(enableDebugging, enableProfiling, enableSystemLogHandler, redirectContainerStreaming bool,
streamingServer streaming.Server) *serverTestFramework {
fw := &serverTestFramework{}
fw.fakeKubelet = &fakeKubelet{
Expand Down Expand Up @@ -348,6 +348,7 @@ func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler,
fw.fakeAuth,
true,
enableDebugging,
enableProfiling,
false,
redirectContainerStreaming,
enableSystemLogHandler,
Expand Down Expand Up @@ -1036,7 +1037,7 @@ func TestServeExecInContainerIdleTimeout(t *testing.T) {
ss, err := newTestStreamingServer(100 * time.Millisecond)
require.NoError(t, err)
defer ss.testHTTPServer.Close()
fw := newServerTestWithDebug(true, false, ss)
fw := newServerTestWithDebug(true, true, false, ss)
defer fw.testHTTPServer.Close()

podNamespace := "other"
Expand Down Expand Up @@ -1095,7 +1096,7 @@ func testExecAttach(t *testing.T, verb string) {
ss, err := newTestStreamingServer(0)
require.NoError(t, err)
defer ss.testHTTPServer.Close()
fw := newServerTestWithDebug(true, test.redirect, ss)
fw := newServerTestWithDebug(true, true, test.redirect, ss)
defer fw.testHTTPServer.Close()
fmt.Println(desc)

Expand Down Expand Up @@ -1299,7 +1300,7 @@ func TestServePortForwardIdleTimeout(t *testing.T) {
ss, err := newTestStreamingServer(100 * time.Millisecond)
require.NoError(t, err)
defer ss.testHTTPServer.Close()
fw := newServerTestWithDebug(true, false, ss)
fw := newServerTestWithDebug(true, true, false, ss)
defer fw.testHTTPServer.Close()

podNamespace := "other"
Expand Down Expand Up @@ -1360,7 +1361,7 @@ func TestServePortForward(t *testing.T) {
ss, err := newTestStreamingServer(0)
require.NoError(t, err)
defer ss.testHTTPServer.Close()
fw := newServerTestWithDebug(true, test.redirect, ss)
fw := newServerTestWithDebug(true, true, test.redirect, ss)
defer fw.testHTTPServer.Close()

portForwardFuncDone := make(chan struct{})
Expand Down Expand Up @@ -1557,7 +1558,7 @@ func TestMetricMethodBuckets(t *testing.T) {
func TestDebuggingDisabledHandlers(t *testing.T) {
// for backward compatibility even if enablesystemLogHandler is set but not enableDebuggingHandler then /logs
//shouldn't be served.
fw := newServerTestWithDebuggingHandlers(false, true, false, nil)
fw := newServerTestWithDebuggingHandlers(false, false, true, false, nil)
defer fw.testHTTPServer.Close()

paths := []string{
Expand Down Expand Up @@ -1597,17 +1598,24 @@ func TestDebuggingDisabledHandlers(t *testing.T) {
resp, err = http.Get(fw.testHTTPServer.URL + "/spec")
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)

}

func TestDisablingSystemLogHandler(t *testing.T) {
fw := newServerTestWithDebuggingHandlers(true, false, false, nil)
fw := newServerTestWithDebuggingHandlers(true, false, false, false, nil)
defer fw.testHTTPServer.Close()

// verify logs endpoint is disabled
verifyEndpointResponse(t, fw, "/logs/kubelet.log", "logs endpoint is disabled.\n")
}

func TestDisableProfiling(t *testing.T) {
fw := newServerTestWithDebuggingHandlers(false, false, false, false, nil)
defer fw.testHTTPServer.Close()

// verify debug endpoint is disabled
verifyEndpointResponse(t, fw, "/debug/pprof/profile?seconds=2", "Debug endpoints are disabled.\n")
}

func TestFailedParseParamsSummaryHandler(t *testing.T) {
fw := newServerTest()
defer fw.testHTTPServer.Close()
Expand Down
4 changes: 2 additions & 2 deletions pkg/kubelet/server/server_websocket_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func TestServeWSPortForward(t *testing.T) {
ss, err := newTestStreamingServer(0)
require.NoError(t, err)
defer ss.testHTTPServer.Close()
fw := newServerTestWithDebug(true, false, ss)
fw := newServerTestWithDebug(true, true, false, ss)
defer fw.testHTTPServer.Close()

portForwardFuncDone := make(chan struct{})
Expand Down Expand Up @@ -158,7 +158,7 @@ func TestServeWSMultiplePortForward(t *testing.T) {
ss, err := newTestStreamingServer(0)
require.NoError(t, err)
defer ss.testHTTPServer.Close()
fw := newServerTestWithDebug(true, false, ss)
fw := newServerTestWithDebug(true, true, false, ss)
defer fw.testHTTPServer.Close()

portForwardWG := sync.WaitGroup{}
Expand Down
6 changes: 6 additions & 0 deletions staging/src/k8s.io/kubelet/config/v1beta1/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,12 @@ type KubeletConfiguration struct {
// Default: true
// +optional
EnableDebuggingHandlers *bool `json:"enableDebuggingHandlers,omitempty"`
// enableProfiling enables profiling via web interface host:port/debug/pprof/
// Dynamic Kubelet Config (beta): If dynamically updating this field, consider that
// enabling it may carry a performance impact.
// Default: true
// +optional
EnableProfiling *bool `json:"enableProfiling,omitempty"`
// enableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true.
// Dynamic Kubelet Config (beta): If dynamically updating this field, consider that
// enabling it may carry a performance impact.
Expand Down
5 changes: 5 additions & 0 deletions staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.