From e0b9d79758b1fe7b2453c275a783cacd94ba7a15 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 27 Jan 2022 11:20:19 +0100 Subject: [PATCH 1/5] scheduler: support contextual logging This replaces all log calls through the global klog logger with a logger that gets passed in by the caller, either through an explicit parameter or the context. This makes it possible to produce more informative log output by adding WithName and/or WithValues instrumentation (not done yet!). It also makes "go test" failures more useful because the log output gets associated with the test which produced it. Usually, context is the more future-proof option. It has happened repeatedly throughout the life of this commit that a function that originally had no context later got changed in the master branch to accept for reasons unrelated to contextual logging. In several places, passing a context can replace passing a stop channel and then serves two purposes (access to logger and cancellation). But klog.FromContext has some overhead, so for simple functions a logger is passed. --- .../app/testing/testserver.go | 22 +- cmd/kube-scheduler/app/options/configfile.go | 12 +- cmd/kube-scheduler/app/server.go | 18 +- cmd/kube-scheduler/app/testing/testserver.go | 25 +- hack/logcheck.conf | 5 +- .../apis/config/v1beta2/default_plugins.go | 34 +- .../config/v1beta2/default_plugins_test.go | 7 +- pkg/scheduler/apis/config/v1beta2/defaults.go | 8 +- .../apis/config/v1beta2/defaults_test.go | 10 +- .../apis/config/v1beta3/default_plugins.go | 32 +- .../config/v1beta3/default_plugins_test.go | 10 +- pkg/scheduler/apis/config/v1beta3/defaults.go | 8 +- .../apis/config/v1beta3/defaults_test.go | 12 +- pkg/scheduler/eventhandlers.go | 100 +++--- pkg/scheduler/eventhandlers_test.go | 12 +- pkg/scheduler/extender.go | 2 + pkg/scheduler/extender_test.go | 13 +- pkg/scheduler/framework/extender.go | 2 + pkg/scheduler/framework/interface.go | 13 +- .../plugins/defaultbinder/default_binder.go | 3 +- .../defaultpreemption/default_preemption.go | 5 +- .../default_preemption_test.go | 24 +- .../plugins/examples/stateful/stateful.go | 5 +- .../plugins/interpodaffinity/filtering.go | 7 +- .../interpodaffinity/filtering_test.go | 6 +- .../plugins/interpodaffinity/plugin.go | 4 +- .../plugins/interpodaffinity/scoring.go | 6 +- .../noderesources/balanced_allocation.go | 4 +- .../framework/plugins/noderesources/fit.go | 4 +- .../noderesources/resource_allocation.go | 9 +- .../framework/plugins/nodevolumelimits/csi.go | 52 +-- .../plugins/nodevolumelimits/non_csi.go | 41 +-- .../plugins/nodevolumelimits/non_csi_test.go | 16 +- .../plugins/podtopologyspread/filtering.go | 9 +- .../plugins/volumebinding/assume_cache.go | 53 +-- .../volumebinding/assume_cache_test.go | 25 +- .../framework/plugins/volumebinding/binder.go | 128 ++++---- .../plugins/volumebinding/binder_test.go | 108 +++--- .../plugins/volumebinding/fake_binder.go | 15 +- .../plugins/volumebinding/volume_binding.go | 23 +- .../plugins/volumezone/volume_zone.go | 5 +- .../framework/preemption/preemption.go | 50 +-- .../framework/preemption/preemption_test.go | 4 + pkg/scheduler/framework/runtime/framework.go | 69 ++-- .../framework/runtime/framework_test.go | 3 + pkg/scheduler/framework/types.go | 12 +- pkg/scheduler/framework/types_test.go | 4 +- pkg/scheduler/internal/cache/cache.go | 147 ++++----- pkg/scheduler/internal/cache/cache_test.go | 224 ++++++++----- .../internal/cache/debugger/comparer.go | 12 +- .../internal/cache/debugger/debugger.go | 10 +- .../internal/cache/debugger/dumper.go | 22 +- .../internal/cache/fake/fake_cache.go | 23 +- pkg/scheduler/internal/cache/interface.go | 21 +- pkg/scheduler/internal/cache/node_tree.go | 22 +- .../internal/cache/node_tree_test.go | 29 +- .../internal/queue/scheduling_queue.go | 126 +++---- .../internal/queue/scheduling_queue_test.go | 309 ++++++++++-------- pkg/scheduler/schedule_one.go | 115 ++++--- pkg/scheduler/schedule_one_test.go | 122 ++++--- pkg/scheduler/scheduler.go | 64 ++-- pkg/scheduler/scheduler_test.go | 43 +-- pkg/scheduler/testing/fake_extender.go | 8 +- pkg/scheduler/util/utils.go | 2 +- .../cloud-provider/app/testing/testserver.go | 26 +- test/integration/daemonset/daemonset_test.go | 2 +- test/integration/scheduler/queue_test.go | 8 +- test/integration/scheduler_perf/main_test.go | 5 + .../scheduler_perf/scheduler_perf_test.go | 2 +- test/integration/scheduler_perf/util.go | 5 +- test/integration/serving/serving_test.go | 22 +- test/integration/util/util.go | 10 +- 72 files changed, 1375 insertions(+), 1043 deletions(-) diff --git a/cmd/kube-controller-manager/app/testing/testserver.go b/cmd/kube-controller-manager/app/testing/testserver.go index f2ce36dcf08c..3f5513178456 100644 --- a/cmd/kube-controller-manager/app/testing/testserver.go +++ b/cmd/kube-controller-manager/app/testing/testserver.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" + "k8s.io/klog/v2" "k8s.io/kubernetes/cmd/kube-controller-manager/app" kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" @@ -45,20 +46,14 @@ type TestServer struct { TmpDir string // Temp Dir used, by the apiserver } -// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie -type Logger interface { - Errorf(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) - Logf(format string, args ...interface{}) -} - // StartTestServer starts a kube-controller-manager. A rest client config and a tear-down func, // and location of the tmpdir are returned. // // Note: we return a tear-down func instead of a stop channel because the later will leak temporary // files that because Golang testing's call to os.Exit will not give a stop channel go routine // enough time to remove temporary files. -func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { +func StartTestServer(ctx context.Context, customFlags []string) (result TestServer, err error) { + logger := klog.FromContext(ctx) stopCh := make(chan struct{}) tearDown := func() { close(stopCh) @@ -97,7 +92,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } s.SecureServing.ServerCert.CertDirectory = result.TmpDir - t.Logf("kube-controller-manager will listen securely on port %d...", s.SecureServing.BindPort) + logger.Info("kube-controller-manager will listen securely", "port", s.SecureServing.BindPort) } config, err := s.Config(all, disabled) @@ -112,7 +107,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } }(stopCh) - t.Logf("Waiting for /healthz to be ok...") + logger.Info("Waiting for /healthz to be ok...") client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) @@ -146,14 +141,13 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t Logger, flags []string) *TestServer { - result, err := StartTestServer(t, flags) +func StartTestServerOrDie(ctx context.Context, flags []string) *TestServer { + result, err := StartTestServer(ctx, flags) if err == nil { return &result } - t.Fatalf("failed to launch server: %v", err) - return nil + panic(fmt.Errorf("failed to launch server: %v", err)) } func createListenerOnFreePort() (net.Listener, int, error) { diff --git a/cmd/kube-scheduler/app/options/configfile.go b/cmd/kube-scheduler/app/options/configfile.go index 84802da767fa..507b00cfb2fa 100644 --- a/cmd/kube-scheduler/app/options/configfile.go +++ b/cmd/kube-scheduler/app/options/configfile.go @@ -80,9 +80,9 @@ func encodeConfig(cfg *config.KubeSchedulerConfiguration) (*bytes.Buffer, error) } // LogOrWriteConfig logs the completed component config and writes it into the given file name as YAML, if either is enabled -func LogOrWriteConfig(fileName string, cfg *config.KubeSchedulerConfiguration, completedProfiles []config.KubeSchedulerProfile) error { - klogV := klog.V(2) - if !klogV.Enabled() && len(fileName) == 0 { +func LogOrWriteConfig(logger klog.Logger, fileName string, cfg *config.KubeSchedulerConfiguration, completedProfiles []config.KubeSchedulerProfile) error { + loggerV := logger.V(2) + if !loggerV.Enabled() && len(fileName) == 0 { return nil } cfg.Profiles = completedProfiles @@ -92,8 +92,8 @@ func LogOrWriteConfig(fileName string, cfg *config.KubeSchedulerConfiguration, c return err } - if klogV.Enabled() { - klogV.InfoS("Using component config", "config", buf.String()) + if loggerV.Enabled() { + loggerV.Info("Using component config", "config", buf.String()) } if len(fileName) > 0 { @@ -105,7 +105,7 @@ func LogOrWriteConfig(fileName string, cfg *config.KubeSchedulerConfiguration, c if _, err := io.Copy(configFile, buf); err != nil { return err } - klog.InfoS("Wrote configuration", "file", fileName) + logger.Info("Wrote configuration", "file", fileName) os.Exit(0) } return nil diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index c352ba664d0e..6e4c6411c245 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -108,6 +108,7 @@ for more information about scheduling and the kube-scheduler component.`, cliflag.SetUsageAndHelpFunc(cmd, *nfs, cols) if err := cmd.MarkFlagFilename("config", "yaml", "yml", "json"); err != nil { + // nolint:logcheck // TODO (?): return error instead of logging it here klog.ErrorS(err, "Failed to mark flag filename") } @@ -144,10 +145,12 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op // Run executes the scheduler based on the given configuration. It only returns on error or when context is done. func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched *scheduler.Scheduler) error { + logger := klog.FromContext(ctx) + // To help debugging, immediately log version - klog.InfoS("Starting Kubernetes Scheduler", "version", version.Get()) + logger.Info("Starting Kubernetes Scheduler", "version", version.Get()) - klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) + logger.Info("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) // Configz registration. if cz, err := configz.New("componentconfig"); err == nil { @@ -213,11 +216,11 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched * select { case <-ctx.Done(): // We were asked to terminate. Exit 0. - klog.InfoS("Requested to terminate, exiting") + logger.Info("Requested to terminate, exiting") os.Exit(0) default: // We lost the lock. - klog.ErrorS(nil, "Leaderelection lost") + logger.Error(nil, "Leaderelection lost") klog.FlushAndExit(klog.ExitFlushTimeout, 1) } }, @@ -326,11 +329,12 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions recorderFactory := getRecorderFactory(&cc) completedProfiles := make([]kubeschedulerconfig.KubeSchedulerProfile, 0) // Create the scheduler. - sched, err := scheduler.New(cc.Client, + sched, err := scheduler.New( + ctx, + cc.Client, cc.InformerFactory, cc.DynInformerFactory, recorderFactory, - ctx.Done(), scheduler.WithComponentConfigVersion(cc.ComponentConfig.TypeMeta.APIVersion), scheduler.WithKubeConfig(cc.KubeConfig), scheduler.WithProfiles(cc.ComponentConfig.Profiles...), @@ -349,7 +353,7 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions if err != nil { return nil, nil, err } - if err := options.LogOrWriteConfig(opts.WriteConfigTo, &cc.ComponentConfig, completedProfiles); err != nil { + if err := options.LogOrWriteConfig(klog.FromContext(ctx), opts.WriteConfigTo, &cc.ComponentConfig, completedProfiles); err != nil { return nil, nil, err } diff --git a/cmd/kube-scheduler/app/testing/testserver.go b/cmd/kube-scheduler/app/testing/testserver.go index e76b2204b23b..ed88a921926c 100644 --- a/cmd/kube-scheduler/app/testing/testserver.go +++ b/cmd/kube-scheduler/app/testing/testserver.go @@ -48,21 +48,15 @@ type TestServer struct { TmpDir string // Temp Dir used, by the apiserver } -// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie -type Logger interface { - Errorf(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) - Logf(format string, args ...interface{}) -} - // StartTestServer starts a kube-scheduler. A rest client config and a tear-down func, // and location of the tmpdir are returned. // // Note: we return a tear-down func instead of a stop channel because the later will leak temporary // files that because Golang testing's call to os.Exit will not give a stop channel go routine // enough time to remove temporary files. -func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { - ctx, cancel := context.WithCancel(context.Background()) +func StartTestServer(ctx context.Context, customFlags []string) (result TestServer, err error) { + logger := klog.FromContext(ctx) + ctx, cancel := context.WithCancel(ctx) var errCh chan error tearDown := func() { @@ -73,7 +67,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err if errCh != nil { err, ok := <-errCh if ok && err != nil { - klog.ErrorS(err, "Failed to shutdown test server clearly") + logger.Error(err, "Failed to shutdown test server clearly") } } if len(result.TmpDir) != 0 { @@ -108,7 +102,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } opts.SecureServing.ServerCert.CertDirectory = result.TmpDir - t.Logf("kube-scheduler will listen securely on port %d...", opts.SecureServing.BindPort) + logger.Info("kube-scheduler will listen securely", "port", opts.SecureServing.BindPort) } cc, sched, err := app.Setup(ctx, opts) @@ -124,7 +118,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } }(ctx) - t.Logf("Waiting for /healthz to be ok...") + logger.Info("Waiting for /healthz to be ok...") client, err := kubernetes.NewForConfig(cc.LoopbackClientConfig) if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) @@ -158,14 +152,13 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t Logger, flags []string) *TestServer { - result, err := StartTestServer(t, flags) +func StartTestServerOrDie(ctx context.Context, flags []string) *TestServer { + result, err := StartTestServer(ctx, flags) if err == nil { return &result } - t.Fatalf("failed to launch server: %v", err) - return nil + panic(fmt.Errorf("failed to launch server: %v", err)) } func createListenerOnFreePort() (net.Listener, int, error) { diff --git a/hack/logcheck.conf b/hack/logcheck.conf index 3ec1cf4f9a57..e792731e54e7 100644 --- a/hack/logcheck.conf +++ b/hack/logcheck.conf @@ -15,16 +15,15 @@ # Now enable it again for migrated packages. structured k8s.io/kubernetes/cmd/kube-proxy/.* -structured k8s.io/kubernetes/cmd/kube-scheduler/.* structured k8s.io/kubernetes/cmd/kubelet/.* structured k8s.io/kubernetes/pkg/kubelet/.* structured k8s.io/kubernetes/pkg/proxy/.* -structured k8s.io/kubernetes/pkg/scheduler/.* # The following packages have been migrated to contextual logging. # Packages matched here do not have to be listed above because # "contextual" implies "structured". -# TODO next: contextual k8s.io/kubernetes/pkg/scheduler/.* +contextual k8s.io/kubernetes/cmd/kube-scheduler/.* +contextual k8s.io/kubernetes/pkg/scheduler/.* # As long as contextual logging is alpha or beta, all WithName, WithValues, # NewContext calls have to go through klog. Once it is GA, we can lift diff --git a/pkg/scheduler/apis/config/v1beta2/default_plugins.go b/pkg/scheduler/apis/config/v1beta2/default_plugins.go index 0eb8af06e46d..b541b9699350 100644 --- a/pkg/scheduler/apis/config/v1beta2/default_plugins.go +++ b/pkg/scheduler/apis/config/v1beta2/default_plugins.go @@ -27,7 +27,7 @@ import ( ) // getDefaultPlugins returns the default set of plugins. -func getDefaultPlugins() *v1beta2.Plugins { +func getDefaultPlugins(logger klog.Logger) *v1beta2.Plugins { plugins := &v1beta2.Plugins{ QueueSort: v1beta2.PluginSet{ Enabled: []v1beta2.Plugin{ @@ -107,34 +107,34 @@ func getDefaultPlugins() *v1beta2.Plugins { }, }, } - applyFeatureGates(plugins) + applyFeatureGates(logger, plugins) return plugins } -func applyFeatureGates(config *v1beta2.Plugins) { +func applyFeatureGates(logger klog.Logger, config *v1beta2.Plugins) { if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) { config.Score.Enabled = append(config.Score.Enabled, v1beta2.Plugin{Name: names.VolumeBinding, Weight: pointer.Int32Ptr(1)}) } } // mergePlugins merges the custom set into the given default one, handling disabled sets. -func mergePlugins(defaultPlugins, customPlugins *v1beta2.Plugins) *v1beta2.Plugins { +func mergePlugins(logger klog.Logger, defaultPlugins, customPlugins *v1beta2.Plugins) *v1beta2.Plugins { if customPlugins == nil { return defaultPlugins } - defaultPlugins.QueueSort = mergePluginSet(defaultPlugins.QueueSort, customPlugins.QueueSort) - defaultPlugins.PreFilter = mergePluginSet(defaultPlugins.PreFilter, customPlugins.PreFilter) - defaultPlugins.Filter = mergePluginSet(defaultPlugins.Filter, customPlugins.Filter) - defaultPlugins.PostFilter = mergePluginSet(defaultPlugins.PostFilter, customPlugins.PostFilter) - defaultPlugins.PreScore = mergePluginSet(defaultPlugins.PreScore, customPlugins.PreScore) - defaultPlugins.Score = mergePluginSet(defaultPlugins.Score, customPlugins.Score) - defaultPlugins.Reserve = mergePluginSet(defaultPlugins.Reserve, customPlugins.Reserve) - defaultPlugins.Permit = mergePluginSet(defaultPlugins.Permit, customPlugins.Permit) - defaultPlugins.PreBind = mergePluginSet(defaultPlugins.PreBind, customPlugins.PreBind) - defaultPlugins.Bind = mergePluginSet(defaultPlugins.Bind, customPlugins.Bind) - defaultPlugins.PostBind = mergePluginSet(defaultPlugins.PostBind, customPlugins.PostBind) + defaultPlugins.QueueSort = mergePluginSet(logger, defaultPlugins.QueueSort, customPlugins.QueueSort) + defaultPlugins.PreFilter = mergePluginSet(logger, defaultPlugins.PreFilter, customPlugins.PreFilter) + defaultPlugins.Filter = mergePluginSet(logger, defaultPlugins.Filter, customPlugins.Filter) + defaultPlugins.PostFilter = mergePluginSet(logger, defaultPlugins.PostFilter, customPlugins.PostFilter) + defaultPlugins.PreScore = mergePluginSet(logger, defaultPlugins.PreScore, customPlugins.PreScore) + defaultPlugins.Score = mergePluginSet(logger, defaultPlugins.Score, customPlugins.Score) + defaultPlugins.Reserve = mergePluginSet(logger, defaultPlugins.Reserve, customPlugins.Reserve) + defaultPlugins.Permit = mergePluginSet(logger, defaultPlugins.Permit, customPlugins.Permit) + defaultPlugins.PreBind = mergePluginSet(logger, defaultPlugins.PreBind, customPlugins.PreBind) + defaultPlugins.Bind = mergePluginSet(logger, defaultPlugins.Bind, customPlugins.Bind) + defaultPlugins.PostBind = mergePluginSet(logger, defaultPlugins.PostBind, customPlugins.PostBind) return defaultPlugins } @@ -143,7 +143,7 @@ type pluginIndex struct { plugin v1beta2.Plugin } -func mergePluginSet(defaultPluginSet, customPluginSet v1beta2.PluginSet) v1beta2.PluginSet { +func mergePluginSet(logger klog.Logger, defaultPluginSet, customPluginSet v1beta2.PluginSet) v1beta2.PluginSet { disabledPlugins := sets.NewString() enabledCustomPlugins := make(map[string]pluginIndex) // replacedPluginIndex is a set of index of plugins, which have replaced the default plugins. @@ -162,7 +162,7 @@ func mergePluginSet(defaultPluginSet, customPluginSet v1beta2.PluginSet) v1beta2 } // The default plugin is explicitly re-configured, update the default plugin accordingly. if customPlugin, ok := enabledCustomPlugins[defaultEnabledPlugin.Name]; ok { - klog.InfoS("Default plugin is explicitly re-configured; overriding", "plugin", defaultEnabledPlugin.Name) + logger.Info("Default plugin is explicitly re-configured; overriding", "plugin", defaultEnabledPlugin.Name) // Update the default plugin in place to preserve order. defaultEnabledPlugin = customPlugin.plugin replacedPluginIndex.Insert(customPlugin.index) diff --git a/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go b/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go index 6909fc740416..f9069010acd5 100644 --- a/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go +++ b/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta2" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" "k8s.io/utils/pointer" @@ -117,11 +118,12 @@ func TestApplyFeatureGates(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) for k, v := range test.features { defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)() } - gotConfig := getDefaultPlugins() + gotConfig := getDefaultPlugins(logger) if diff := cmp.Diff(test.wantConfig, gotConfig); diff != "" { t.Errorf("unexpected config diff (-want, +got): %s", diff) } @@ -370,7 +372,8 @@ func TestMergePlugins(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.defaultPlugins = mergePlugins(test.defaultPlugins, test.customPlugins) + logger, _ := ktesting.NewTestContext(t) + test.defaultPlugins = mergePlugins(logger, test.defaultPlugins, test.customPlugins) if d := cmp.Diff(test.expectedPlugins, test.defaultPlugins); d != "" { t.Fatalf("plugins mismatch (-want +got):\n%s", d) } diff --git a/pkg/scheduler/apis/config/v1beta2/defaults.go b/pkg/scheduler/apis/config/v1beta2/defaults.go index 60ef8f065a84..f76d5e964e5b 100644 --- a/pkg/scheduler/apis/config/v1beta2/defaults.go +++ b/pkg/scheduler/apis/config/v1beta2/defaults.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/klog/v2" "k8s.io/kube-scheduler/config/v1beta2" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -63,9 +64,9 @@ func pluginsNames(p *v1beta2.Plugins) []string { return n.List() } -func setDefaults_KubeSchedulerProfile(prof *v1beta2.KubeSchedulerProfile) { +func setDefaults_KubeSchedulerProfile(logger klog.Logger, prof *v1beta2.KubeSchedulerProfile) { // Set default plugins. - prof.Plugins = mergePlugins(getDefaultPlugins(), prof.Plugins) + prof.Plugins = mergePlugins(logger, getDefaultPlugins(logger), prof.Plugins) // Set default plugin configs. scheme := GetPluginArgConversionScheme() @@ -101,6 +102,7 @@ func setDefaults_KubeSchedulerProfile(prof *v1beta2.KubeSchedulerProfile) { // SetDefaults_KubeSchedulerConfiguration sets additional defaults func SetDefaults_KubeSchedulerConfiguration(obj *v1beta2.KubeSchedulerConfiguration) { + logger := klog.TODO() // called by generated code that doesn't pass a logger if obj.Parallelism == nil { obj.Parallelism = pointer.Int32Ptr(16) } @@ -117,7 +119,7 @@ func SetDefaults_KubeSchedulerConfiguration(obj *v1beta2.KubeSchedulerConfigurat // Add the default set of plugins and apply the configuration. for i := range obj.Profiles { prof := &obj.Profiles[i] - setDefaults_KubeSchedulerProfile(prof) + setDefaults_KubeSchedulerProfile(logger, prof) } if obj.PercentageOfNodesToScore == nil { diff --git a/pkg/scheduler/apis/config/v1beta2/defaults_test.go b/pkg/scheduler/apis/config/v1beta2/defaults_test.go index d25b5309365c..0687cd248293 100644 --- a/pkg/scheduler/apis/config/v1beta2/defaults_test.go +++ b/pkg/scheduler/apis/config/v1beta2/defaults_test.go @@ -30,6 +30,7 @@ import ( componentbaseconfig "k8s.io/component-base/config/v1alpha1" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta2" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" @@ -115,6 +116,7 @@ var pluginConfigs = []v1beta2.PluginConfig{ } func TestSchedulerDefaults(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) enable := true tests := []struct { name string @@ -149,7 +151,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta2.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, @@ -187,7 +189,7 @@ func TestSchedulerDefaults(t *testing.T) { Profiles: []v1beta2.KubeSchedulerProfile{ { SchedulerName: pointer.StringPtr("default-scheduler"), - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs}, }, }, @@ -242,7 +244,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta2.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: []v1beta2.PluginConfig{ {Name: "FooPlugin"}, { @@ -435,7 +437,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta2.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, diff --git a/pkg/scheduler/apis/config/v1beta3/default_plugins.go b/pkg/scheduler/apis/config/v1beta3/default_plugins.go index ca360b78260f..fc245f77b758 100644 --- a/pkg/scheduler/apis/config/v1beta3/default_plugins.go +++ b/pkg/scheduler/apis/config/v1beta3/default_plugins.go @@ -25,7 +25,7 @@ import ( ) // getDefaultPlugins returns the default set of plugins. -func getDefaultPlugins() *v1beta3.Plugins { +func getDefaultPlugins(logger klog.Logger) *v1beta3.Plugins { plugins := &v1beta3.Plugins{ MultiPoint: v1beta3.PluginSet{ Enabled: []v1beta3.Plugin{ @@ -57,23 +57,23 @@ func getDefaultPlugins() *v1beta3.Plugins { } // mergePlugins merges the custom set into the given default one, handling disabled sets. -func mergePlugins(defaultPlugins, customPlugins *v1beta3.Plugins) *v1beta3.Plugins { +func mergePlugins(logger klog.Logger, defaultPlugins, customPlugins *v1beta3.Plugins) *v1beta3.Plugins { if customPlugins == nil { return defaultPlugins } - defaultPlugins.MultiPoint = mergePluginSet(defaultPlugins.MultiPoint, customPlugins.MultiPoint) - defaultPlugins.QueueSort = mergePluginSet(defaultPlugins.QueueSort, customPlugins.QueueSort) - defaultPlugins.PreFilter = mergePluginSet(defaultPlugins.PreFilter, customPlugins.PreFilter) - defaultPlugins.Filter = mergePluginSet(defaultPlugins.Filter, customPlugins.Filter) - defaultPlugins.PostFilter = mergePluginSet(defaultPlugins.PostFilter, customPlugins.PostFilter) - defaultPlugins.PreScore = mergePluginSet(defaultPlugins.PreScore, customPlugins.PreScore) - defaultPlugins.Score = mergePluginSet(defaultPlugins.Score, customPlugins.Score) - defaultPlugins.Reserve = mergePluginSet(defaultPlugins.Reserve, customPlugins.Reserve) - defaultPlugins.Permit = mergePluginSet(defaultPlugins.Permit, customPlugins.Permit) - defaultPlugins.PreBind = mergePluginSet(defaultPlugins.PreBind, customPlugins.PreBind) - defaultPlugins.Bind = mergePluginSet(defaultPlugins.Bind, customPlugins.Bind) - defaultPlugins.PostBind = mergePluginSet(defaultPlugins.PostBind, customPlugins.PostBind) + defaultPlugins.MultiPoint = mergePluginSet(logger, defaultPlugins.MultiPoint, customPlugins.MultiPoint) + defaultPlugins.QueueSort = mergePluginSet(logger, defaultPlugins.QueueSort, customPlugins.QueueSort) + defaultPlugins.PreFilter = mergePluginSet(logger, defaultPlugins.PreFilter, customPlugins.PreFilter) + defaultPlugins.Filter = mergePluginSet(logger, defaultPlugins.Filter, customPlugins.Filter) + defaultPlugins.PostFilter = mergePluginSet(logger, defaultPlugins.PostFilter, customPlugins.PostFilter) + defaultPlugins.PreScore = mergePluginSet(logger, defaultPlugins.PreScore, customPlugins.PreScore) + defaultPlugins.Score = mergePluginSet(logger, defaultPlugins.Score, customPlugins.Score) + defaultPlugins.Reserve = mergePluginSet(logger, defaultPlugins.Reserve, customPlugins.Reserve) + defaultPlugins.Permit = mergePluginSet(logger, defaultPlugins.Permit, customPlugins.Permit) + defaultPlugins.PreBind = mergePluginSet(logger, defaultPlugins.PreBind, customPlugins.PreBind) + defaultPlugins.Bind = mergePluginSet(logger, defaultPlugins.Bind, customPlugins.Bind) + defaultPlugins.PostBind = mergePluginSet(logger, defaultPlugins.PostBind, customPlugins.PostBind) return defaultPlugins } @@ -82,7 +82,7 @@ type pluginIndex struct { plugin v1beta3.Plugin } -func mergePluginSet(defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3.PluginSet { +func mergePluginSet(logger klog.Logger, defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3.PluginSet { disabledPlugins := sets.NewString() enabledCustomPlugins := make(map[string]pluginIndex) // replacedPluginIndex is a set of index of plugins, which have replaced the default plugins. @@ -114,7 +114,7 @@ func mergePluginSet(defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3 } // The default plugin is explicitly re-configured, update the default plugin accordingly. if customPlugin, ok := enabledCustomPlugins[defaultEnabledPlugin.Name]; ok { - klog.InfoS("Default plugin is explicitly re-configured; overriding", "plugin", defaultEnabledPlugin.Name) + logger.Info("Default plugin is explicitly re-configured; overriding", "plugin", defaultEnabledPlugin.Name) // Update the default plugin in place to preserve order. defaultEnabledPlugin = customPlugin.plugin replacedPluginIndex.Insert(customPlugin.index) diff --git a/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go b/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go index 05d0a33d5d3b..b0ebaca375d2 100644 --- a/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go +++ b/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go @@ -19,11 +19,13 @@ package v1beta3 import ( "testing" + "k8s.io/kube-scheduler/config/v1beta3" + "github.com/google/go-cmp/cmp" "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kube-scheduler/config/v1beta3" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" "k8s.io/utils/pointer" ) @@ -67,11 +69,12 @@ func TestApplyFeatureGates(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) for k, v := range test.features { defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)() } - gotConfig := getDefaultPlugins() + gotConfig := getDefaultPlugins(logger) if diff := cmp.Diff(test.wantConfig, gotConfig); diff != "" { t.Errorf("unexpected config diff (-want, +got): %s", diff) } @@ -514,7 +517,8 @@ func TestMergePlugins(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - gotPlugins := mergePlugins(test.defaultPlugins, test.customPlugins) + logger, _ := ktesting.NewTestContext(t) + gotPlugins := mergePlugins(logger, test.defaultPlugins, test.customPlugins) if d := cmp.Diff(test.expectedPlugins, gotPlugins); d != "" { t.Fatalf("plugins mismatch (-want +got):\n%s", d) } diff --git a/pkg/scheduler/apis/config/v1beta3/defaults.go b/pkg/scheduler/apis/config/v1beta3/defaults.go index 0ff3781a4211..710e57eea3ca 100644 --- a/pkg/scheduler/apis/config/v1beta3/defaults.go +++ b/pkg/scheduler/apis/config/v1beta3/defaults.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/klog/v2" "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -64,9 +65,9 @@ func pluginsNames(p *v1beta3.Plugins) []string { return n.List() } -func setDefaults_KubeSchedulerProfile(prof *v1beta3.KubeSchedulerProfile) { +func setDefaults_KubeSchedulerProfile(logger klog.Logger, prof *v1beta3.KubeSchedulerProfile) { // Set default plugins. - prof.Plugins = mergePlugins(getDefaultPlugins(), prof.Plugins) + prof.Plugins = mergePlugins(logger, getDefaultPlugins(logger), prof.Plugins) // Set default plugin configs. scheme := GetPluginArgConversionScheme() existingConfigs := sets.NewString() @@ -101,6 +102,7 @@ func setDefaults_KubeSchedulerProfile(prof *v1beta3.KubeSchedulerProfile) { // SetDefaults_KubeSchedulerConfiguration sets additional defaults func SetDefaults_KubeSchedulerConfiguration(obj *v1beta3.KubeSchedulerConfiguration) { + logger := klog.TODO() // called by generated code which doesn't pass a logger if obj.Parallelism == nil { obj.Parallelism = pointer.Int32Ptr(16) } @@ -117,7 +119,7 @@ func SetDefaults_KubeSchedulerConfiguration(obj *v1beta3.KubeSchedulerConfigurat // Add the default set of plugins and apply the configuration. for i := range obj.Profiles { prof := &obj.Profiles[i] - setDefaults_KubeSchedulerProfile(prof) + setDefaults_KubeSchedulerProfile(logger, prof) } if obj.PercentageOfNodesToScore == nil { diff --git a/pkg/scheduler/apis/config/v1beta3/defaults_test.go b/pkg/scheduler/apis/config/v1beta3/defaults_test.go index f1c2a0fa27a4..b5a8ea447297 100644 --- a/pkg/scheduler/apis/config/v1beta3/defaults_test.go +++ b/pkg/scheduler/apis/config/v1beta3/defaults_test.go @@ -30,6 +30,7 @@ import ( componentbaseconfig "k8s.io/component-base/config/v1alpha1" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" @@ -115,6 +116,7 @@ var pluginConfigs = []v1beta3.PluginConfig{ } func TestSchedulerDefaults(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) enable := true tests := []struct { name string @@ -149,7 +151,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta3.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, @@ -187,7 +189,7 @@ func TestSchedulerDefaults(t *testing.T) { Profiles: []v1beta3.KubeSchedulerProfile{ { SchedulerName: pointer.StringPtr("default-scheduler"), - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs}, }, }, @@ -242,7 +244,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta3.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: []v1beta3.PluginConfig{ {Name: "FooPlugin"}, { @@ -393,7 +395,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta3.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, @@ -430,7 +432,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta3.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, diff --git a/pkg/scheduler/eventhandlers.go b/pkg/scheduler/eventhandlers.go index 0c39dcde2775..c7fc3eae75c9 100644 --- a/pkg/scheduler/eventhandlers.go +++ b/pkg/scheduler/eventhandlers.go @@ -41,9 +41,10 @@ import ( ) func (sched *Scheduler) onStorageClassAdd(obj interface{}) { + logger := sched.logger sc, ok := obj.(*storagev1.StorageClass) if !ok { - klog.ErrorS(nil, "Cannot convert to *storagev1.StorageClass", "obj", obj) + logger.Error(nil, "Cannot convert to *storagev1.StorageClass", "obj", obj) return } @@ -54,42 +55,45 @@ func (sched *Scheduler) onStorageClassAdd(obj interface{}) { // We don't need to invalidate cached results because results will not be // cached for pod that has unbound immediate PVCs. if sc.VolumeBindingMode != nil && *sc.VolumeBindingMode == storagev1.VolumeBindingWaitForFirstConsumer { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.StorageClassAdd, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.StorageClassAdd, nil) } } func (sched *Scheduler) addNodeToCache(obj interface{}) { + logger := sched.logger node, ok := obj.(*v1.Node) if !ok { - klog.ErrorS(nil, "Cannot convert to *v1.Node", "obj", obj) + logger.Error(nil, "Cannot convert to *v1.Node", "obj", obj) return } - nodeInfo := sched.Cache.AddNode(node) - klog.V(3).InfoS("Add event for node", "node", klog.KObj(node)) - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.NodeAdd, preCheckForNode(nodeInfo)) + nodeInfo := sched.Cache.AddNode(logger, node) + logger.V(3).Info("Add event for node", "node", klog.KObj(node)) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.NodeAdd, preCheckForNode(nodeInfo)) } func (sched *Scheduler) updateNodeInCache(oldObj, newObj interface{}) { + logger := sched.logger oldNode, ok := oldObj.(*v1.Node) if !ok { - klog.ErrorS(nil, "Cannot convert oldObj to *v1.Node", "oldObj", oldObj) + logger.Error(nil, "Cannot convert oldObj to *v1.Node", "oldObj", oldObj) return } newNode, ok := newObj.(*v1.Node) if !ok { - klog.ErrorS(nil, "Cannot convert newObj to *v1.Node", "newObj", newObj) + logger.Error(nil, "Cannot convert newObj to *v1.Node", "newObj", newObj) return } - nodeInfo := sched.Cache.UpdateNode(oldNode, newNode) + nodeInfo := sched.Cache.UpdateNode(logger, oldNode, newNode) // Only requeue unschedulable pods if the node became more schedulable. if event := nodeSchedulingPropertiesChange(newNode, oldNode); event != nil { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(*event, preCheckForNode(nodeInfo)) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, *event, preCheckForNode(nodeInfo)) } } func (sched *Scheduler) deleteNodeFromCache(obj interface{}) { + logger := sched.logger var node *v1.Node switch t := obj.(type) { case *v1.Node: @@ -98,23 +102,24 @@ func (sched *Scheduler) deleteNodeFromCache(obj interface{}) { var ok bool node, ok = t.Obj.(*v1.Node) if !ok { - klog.ErrorS(nil, "Cannot convert to *v1.Node", "obj", t.Obj) + logger.Error(nil, "Cannot convert to *v1.Node", "obj", t.Obj) return } default: - klog.ErrorS(nil, "Cannot convert to *v1.Node", "obj", t) + logger.Error(nil, "Cannot convert to *v1.Node", "obj", t) return } - klog.V(3).InfoS("Delete event for node", "node", klog.KObj(node)) - if err := sched.Cache.RemoveNode(node); err != nil { - klog.ErrorS(err, "Scheduler cache RemoveNode failed") + logger.V(3).Info("Delete event for node", "node", klog.KObj(node)) + if err := sched.Cache.RemoveNode(logger, node); err != nil { + logger.Error(err, "Scheduler cache RemoveNode failed") } } func (sched *Scheduler) addPodToSchedulingQueue(obj interface{}) { + logger := sched.logger pod := obj.(*v1.Pod) - klog.V(3).InfoS("Add event for unscheduled pod", "pod", klog.KObj(pod)) - if err := sched.SchedulingQueue.Add(pod); err != nil { + logger.V(3).Info("Add event for unscheduled pod", "pod", klog.KObj(pod)) + if err := sched.SchedulingQueue.Add(logger, pod); err != nil { utilruntime.HandleError(fmt.Errorf("unable to queue %T: %v", obj, err)) } } @@ -135,7 +140,8 @@ func (sched *Scheduler) updatePodInSchedulingQueue(oldObj, newObj interface{}) { return } - if err := sched.SchedulingQueue.Update(oldPod, newPod); err != nil { + logger := sched.logger + if err := sched.SchedulingQueue.Update(logger, oldPod, newPod); err != nil { utilruntime.HandleError(fmt.Errorf("unable to update %T: %v", newObj, err)) } } @@ -156,61 +162,65 @@ func (sched *Scheduler) deletePodFromSchedulingQueue(obj interface{}) { utilruntime.HandleError(fmt.Errorf("unable to handle object in %T: %T", sched, obj)) return } - klog.V(3).InfoS("Delete event for unscheduled pod", "pod", klog.KObj(pod)) - if err := sched.SchedulingQueue.Delete(pod); err != nil { + logger := sched.logger + logger.V(3).Info("Delete event for unscheduled pod", "pod", klog.KObj(pod)) + if err := sched.SchedulingQueue.Delete(logger, pod); err != nil { utilruntime.HandleError(fmt.Errorf("unable to dequeue %T: %v", obj, err)) } fwk, err := sched.frameworkForPod(pod) if err != nil { // This shouldn't happen, because we only accept for scheduling the pods // which specify a scheduler name that matches one of the profiles. - klog.ErrorS(err, "Unable to get profile", "pod", klog.KObj(pod)) + logger.Error(err, "Unable to get profile", "pod", klog.KObj(pod)) return } // If a waiting pod is rejected, it indicates it's previously assumed and we're // removing it from the scheduler cache. In this case, signal a AssignedPodDelete // event to immediately retry some unscheduled Pods. if fwk.RejectWaitingPod(pod.UID) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.AssignedPodDelete, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.AssignedPodDelete, nil) } } func (sched *Scheduler) addPodToCache(obj interface{}) { + logger := sched.logger pod, ok := obj.(*v1.Pod) if !ok { - klog.ErrorS(nil, "Cannot convert to *v1.Pod", "obj", obj) + logger.Error(nil, "Cannot convert to *v1.Pod", "obj", obj) return } - klog.V(3).InfoS("Add event for scheduled pod", "pod", klog.KObj(pod)) + logger.V(3).Info("Add event for scheduled pod", "pod", klog.KObj(pod)) - if err := sched.Cache.AddPod(pod); err != nil { - klog.ErrorS(err, "Scheduler cache AddPod failed", "pod", klog.KObj(pod)) + if err := sched.Cache.AddPod(logger, pod); err != nil { + logger.Error(err, "Scheduler cache AddPod failed", "pod", klog.KObj(pod)) } - sched.SchedulingQueue.AssignedPodAdded(pod) + sched.SchedulingQueue.AssignedPodAdded(logger, pod) } func (sched *Scheduler) updatePodInCache(oldObj, newObj interface{}) { + logger := sched.logger oldPod, ok := oldObj.(*v1.Pod) if !ok { - klog.ErrorS(nil, "Cannot convert oldObj to *v1.Pod", "oldObj", oldObj) + logger.Error(nil, "Cannot convert oldObj to *v1.Pod", "oldObj", oldObj) return } newPod, ok := newObj.(*v1.Pod) if !ok { - klog.ErrorS(nil, "Cannot convert newObj to *v1.Pod", "newObj", newObj) + logger.Error(nil, "Cannot convert newObj to *v1.Pod", "newObj", newObj) return } - klog.V(4).InfoS("Update event for scheduled pod", "pod", klog.KObj(oldPod)) + logger.V(4).Info("Update event for scheduled pod", "pod", klog.KObj(oldPod)) - if err := sched.Cache.UpdatePod(oldPod, newPod); err != nil { - klog.ErrorS(err, "Scheduler cache UpdatePod failed", "pod", klog.KObj(oldPod)) + if err := sched.Cache.UpdatePod(logger, oldPod, newPod); err != nil { + logger.Error(err, "Scheduler cache UpdatePod failed", "pod", klog.KObj(oldPod)) } - sched.SchedulingQueue.AssignedPodUpdated(newPod) + sched.SchedulingQueue.AssignedPodUpdated(logger, newPod) } func (sched *Scheduler) deletePodFromCache(obj interface{}) { + logger := sched.logger var pod *v1.Pod switch t := obj.(type) { case *v1.Pod: @@ -219,19 +229,19 @@ func (sched *Scheduler) deletePodFromCache(obj interface{}) { var ok bool pod, ok = t.Obj.(*v1.Pod) if !ok { - klog.ErrorS(nil, "Cannot convert to *v1.Pod", "obj", t.Obj) + logger.Error(nil, "Cannot convert to *v1.Pod", "obj", t.Obj) return } default: - klog.ErrorS(nil, "Cannot convert to *v1.Pod", "obj", t) + logger.Error(nil, "Cannot convert to *v1.Pod", "obj", t) return } - klog.V(3).InfoS("Delete event for scheduled pod", "pod", klog.KObj(pod)) - if err := sched.Cache.RemovePod(pod); err != nil { - klog.ErrorS(err, "Scheduler cache RemovePod failed", "pod", klog.KObj(pod)) + logger.V(3).Info("Delete event for scheduled pod", "pod", klog.KObj(pod)) + if err := sched.Cache.RemovePod(logger, pod); err != nil { + logger.Error(err, "Scheduler cache RemovePod failed", "pod", klog.KObj(pod)) } - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.AssignedPodDelete, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.AssignedPodDelete, nil) } // assignedPod selects pods that are assigned (scheduled and running). @@ -252,6 +262,8 @@ func addAllEventHandlers( dynInformerFactory dynamicinformer.DynamicSharedInformerFactory, gvkMap map[framework.GVK]framework.ActionType, ) { + logger := sched.logger + // scheduled pod cache informerFactory.Core().V1().Pods().Informer().AddEventHandler( cache.FilteringResourceEventHandler{ @@ -320,19 +332,19 @@ func addAllEventHandlers( if at&framework.Add != 0 { evt := framework.ClusterEvent{Resource: gvk, ActionType: framework.Add, Label: fmt.Sprintf("%vAdd", shortGVK)} funcs.AddFunc = func(_ interface{}) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(evt, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, evt, nil) } } if at&framework.Update != 0 { evt := framework.ClusterEvent{Resource: gvk, ActionType: framework.Update, Label: fmt.Sprintf("%vUpdate", shortGVK)} funcs.UpdateFunc = func(_, _ interface{}) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(evt, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, evt, nil) } } if at&framework.Delete != 0 { evt := framework.ClusterEvent{Resource: gvk, ActionType: framework.Delete, Label: fmt.Sprintf("%vDelete", shortGVK)} funcs.DeleteFunc = func(_ interface{}) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(evt, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, evt, nil) } } return funcs @@ -388,7 +400,7 @@ func addAllEventHandlers( informerFactory.Storage().V1().StorageClasses().Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(_, _ interface{}) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.StorageClassUpdate, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.StorageClassUpdate, nil) }, }, ) @@ -407,7 +419,7 @@ func addAllEventHandlers( // - foos.v1 (2 sections) // - foo.v1.example.com (the first section should be plural) if strings.Count(string(gvk), ".") < 2 { - klog.ErrorS(nil, "incorrect event registration", "gvk", gvk) + logger.Error(nil, "incorrect event registration", "gvk", gvk) continue } // Fall back to try dynamic informers. diff --git a/pkg/scheduler/eventhandlers_test.go b/pkg/scheduler/eventhandlers_test.go index 856d2302652d..841b5b6a2447 100644 --- a/pkg/scheduler/eventhandlers_test.go +++ b/pkg/scheduler/eventhandlers_test.go @@ -29,6 +29,7 @@ import ( storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2/ktesting" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -36,7 +37,6 @@ import ( dyfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" - "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename" @@ -220,11 +220,13 @@ func TestUpdatePodInCache(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() sched := &Scheduler{ - Cache: cache.New(ttl, ctx.Done()), + Cache: cache.New(ctx, ttl), SchedulingQueue: queue.NewTestQueue(ctx, nil), + logger: logger, } sched.addPodToCache(tt.oldObj) sched.updatePodInCache(tt.oldObj, tt.newObj) @@ -430,7 +432,8 @@ func TestAddAllEventHandlers(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() informerFactory := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 0) @@ -438,6 +441,7 @@ func TestAddAllEventHandlers(t *testing.T) { testSched := Scheduler{ StopEverything: ctx.Done(), SchedulingQueue: schedulingQueue, + logger: logger, } dynclient := dyfake.NewSimpleDynamicClient(scheme) diff --git a/pkg/scheduler/extender.go b/pkg/scheduler/extender.go index e29026ad1552..c6c6428f44f9 100644 --- a/pkg/scheduler/extender.go +++ b/pkg/scheduler/extender.go @@ -28,6 +28,7 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" restclient "k8s.io/client-go/rest" + "k8s.io/klog/v2" extenderv1 "k8s.io/kube-scheduler/extender/v1" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" @@ -163,6 +164,7 @@ func (h *HTTPExtender) SupportsPreemption() bool { // ProcessPreemption returns filtered candidate nodes and victims after running preemption logic in extender. func (h *HTTPExtender) ProcessPreemption( + logger klog.Logger, pod *v1.Pod, nodeNameToVictims map[string]*extenderv1.Victims, nodeInfos framework.NodeInfoLister, diff --git a/pkg/scheduler/extender_test.go b/pkg/scheduler/extender_test.go index 113f8c4906f8..63fd9e33f4ce 100644 --- a/pkg/scheduler/extender_test.go +++ b/pkg/scheduler/extender_test.go @@ -25,9 +25,9 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" clientsetfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/klog/v2/ktesting" extenderv1 "k8s.io/kube-scheduler/extender/v1" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" @@ -272,6 +272,9 @@ func TestSchedulerWithExtenders(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) @@ -279,21 +282,23 @@ func TestSchedulerWithExtenders(t *testing.T) { for ii := range test.extenders { extenders = append(extenders, &test.extenders[ii]) } - cache := internalcache.New(time.Duration(0), wait.NeverStop) + cache := internalcache.New(ctx, time.Duration(0)) for _, name := range test.nodes { - cache.AddNode(createNode(name)) + cache.AddNode(logger, createNode(name)) } fwk, err := st.NewFramework( test.registerPlugins, "", runtime.WithClientSet(client), runtime.WithInformerFactory(informerFactory), runtime.WithPodNominator(internalqueue.NewPodNominator(informerFactory.Core().V1().Pods().Lister())), + runtime.WithLogger(logger), ) if err != nil { t.Fatal(err) } scheduler := newScheduler( + logger, cache, extenders, nil, @@ -305,7 +310,7 @@ func TestSchedulerWithExtenders(t *testing.T) { emptySnapshot, schedulerapi.DefaultPercentageOfNodesToScore) podIgnored := &v1.Pod{} - result, err := scheduler.SchedulePod(context.Background(), fwk, framework.NewCycleState(), podIgnored) + result, err := scheduler.SchedulePod(ctx, fwk, framework.NewCycleState(), podIgnored) if test.expectsErr { if err == nil { t.Errorf("Unexpected non-error, result %+v", result) diff --git a/pkg/scheduler/framework/extender.go b/pkg/scheduler/framework/extender.go index d47c89bf1e40..4b1ca9297adb 100644 --- a/pkg/scheduler/framework/extender.go +++ b/pkg/scheduler/framework/extender.go @@ -18,6 +18,7 @@ package framework import ( v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" extenderv1 "k8s.io/kube-scheduler/extender/v1" ) @@ -58,6 +59,7 @@ type Extender interface { // 1. Subset of given candidate nodes after preemption phase of extender. // 2. A different set of victim pod for every given candidate node after preemption phase of extender. ProcessPreemption( + logger klog.Logger, pod *v1.Pod, nodeNameToVictims map[string]*extenderv1.Victims, nodeInfos NodeInfoLister, diff --git a/pkg/scheduler/framework/interface.go b/pkg/scheduler/framework/interface.go index feadee9af8dc..b07fa1246f72 100644 --- a/pkg/scheduler/framework/interface.go +++ b/pkg/scheduler/framework/interface.go @@ -35,6 +35,7 @@ import ( clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/events" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework/parallelize" ) @@ -621,6 +622,10 @@ type Handle interface { // Parallelizer returns a parallelizer holding parallelism for scheduler. Parallelizer() parallelize.Parallelizer + + // Logger returns a logger that can be used for background activities when + // the caller does not provide a context-specific logger. + Logger() klog.Logger } // PreFilterResult wraps needed info for scheduler framework to act upon PreFilter phase. @@ -690,13 +695,13 @@ func (ni *NominatingInfo) Mode() NominatingMode { type PodNominator interface { // AddNominatedPod adds the given pod to the nominator or // updates it if it already exists. - AddNominatedPod(pod *PodInfo, nominatingInfo *NominatingInfo) + AddNominatedPod(logger klog.Logger, pod *PodInfo, nominatingInfo *NominatingInfo) // DeleteNominatedPodIfExists deletes nominatedPod from internal cache. It's a no-op if it doesn't exist. - DeleteNominatedPodIfExists(pod *v1.Pod) + DeleteNominatedPodIfExists(logger klog.Logger, pod *v1.Pod) // UpdateNominatedPod updates the with . - UpdateNominatedPod(oldPod *v1.Pod, newPodInfo *PodInfo) + UpdateNominatedPod(logger klog.Logger, oldPod *v1.Pod, newPodInfo *PodInfo) // NominatedPodsForNode returns nominatedPods on the given node. - NominatedPodsForNode(nodeName string) []*PodInfo + NominatedPodsForNode(logger klog.Logger, nodeName string) []*PodInfo } // PluginsRunner abstracts operations to run some plugins. diff --git a/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go b/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go index e695e320dfaf..ead881874819 100644 --- a/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go +++ b/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go @@ -49,7 +49,8 @@ func (b DefaultBinder) Name() string { // Bind binds pods to nodes using the k8s client. func (b DefaultBinder) Bind(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) *framework.Status { - klog.V(3).InfoS("Attempting to bind pod to node", "pod", klog.KObj(p), "node", klog.KRef("", nodeName)) + logger := klog.FromContext(ctx) + logger.V(3).Info("Attempting to bind pod to node", "pod", klog.KObj(p), "node", klog.KRef("", nodeName)) binding := &v1.Binding{ ObjectMeta: metav1.ObjectMeta{Namespace: p.Namespace, Name: p.Name, UID: p.UID}, Target: v1.ObjectReference{Kind: "Node", Name: nodeName}, diff --git a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption.go b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption.go index 91fc9ef9ed00..acf6887242ce 100644 --- a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption.go +++ b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption.go @@ -141,8 +141,9 @@ func (pl *DefaultPreemption) SelectVictimsOnNode( nodeInfo *framework.NodeInfo, pdbs []*policy.PodDisruptionBudget) ([]*v1.Pod, int, *framework.Status) { var potentialVictims []*framework.PodInfo + logger := klog.FromContext(ctx) removePod := func(rpi *framework.PodInfo) error { - if err := nodeInfo.RemovePod(rpi.Pod); err != nil { + if err := nodeInfo.RemovePod(logger, rpi.Pod); err != nil { return err } status := pl.fh.RunPreFilterExtensionRemovePod(ctx, state, pod, rpi, nodeInfo) @@ -205,7 +206,7 @@ func (pl *DefaultPreemption) SelectVictimsOnNode( } rpi := pi.Pod victims = append(victims, rpi) - klog.V(5).InfoS("Pod is a potential preemption victim on node", "pod", klog.KObj(rpi), "node", klog.KObj(nodeInfo.Node())) + logger.V(5).Info("Pod is a potential preemption victim on node", "pod", klog.KObj(rpi), "node", klog.KObj(nodeInfo.Node())) } return fits, nil } diff --git a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go index ebbbcaf7291e..52118b6809d4 100644 --- a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go +++ b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go @@ -36,6 +36,7 @@ import ( clientsetfake "k8s.io/client-go/kubernetes/fake" clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/events" + "k8s.io/klog/v2/ktesting" kubeschedulerconfigv1beta2 "k8s.io/kube-scheduler/config/v1beta2" extenderv1 "k8s.io/kube-scheduler/extender/v1" "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -328,6 +329,7 @@ func TestPostFilter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) cs := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(cs, 0) podInformer := informerFactory.Core().V1().Pods().Informer() @@ -358,6 +360,7 @@ func TestPostFilter(t *testing.T) { frameworkruntime.WithPodNominator(internalqueue.NewPodNominator(informerFactory.Core().V1().Pods().Lister())), frameworkruntime.WithExtenders(extenders), frameworkruntime.WithSnapshotSharedLister(internalcache.NewSnapshot(tt.pods, tt.nodes)), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) @@ -1037,6 +1040,7 @@ func TestDryRunPreemption(t *testing.T) { labelKeys := []string{"hostname", "zone", "region"} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) nodes := make([]*v1.Node, len(tt.nodeNames)) fakeFilterRCMap := make(map[string]framework.Code, len(tt.nodeNames)) for i, nodeName := range tt.nodeNames { @@ -1089,6 +1093,7 @@ func TestDryRunPreemption(t *testing.T) { frameworkruntime.WithSnapshotSharedLister(snapshot), frameworkruntime.WithInformerFactory(informerFactory), frameworkruntime.WithParallelism(parallelism), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) @@ -1320,6 +1325,7 @@ func TestSelectBestCandidate(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) rand.Seed(4) nodes := make([]*v1.Node, len(tt.nodeNames)) for i, nodeName := range tt.nodeNames { @@ -1343,6 +1349,7 @@ func TestSelectBestCandidate(t *testing.T) { "", frameworkruntime.WithPodNominator(internalqueue.NewPodNominator(informerFactory.Core().V1().Pods().Lister())), frameworkruntime.WithSnapshotSharedLister(snapshot), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) @@ -1374,7 +1381,7 @@ func TestSelectBestCandidate(t *testing.T) { } offset, numCandidates := pl.GetOffsetAndNumCandidates(int32(len(nodeInfos))) candidates, _, _ := pe.DryRunPreemption(context.Background(), tt.pod, nodeInfos, nil, offset, numCandidates) - s := pe.SelectCandidate(candidates) + s := pe.SelectCandidate(logger, candidates) if s == nil || len(s.Name()) == 0 { return } @@ -1437,6 +1444,7 @@ func TestPodEligibleToPreemptOthers(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) var nodes []*v1.Node for _, n := range test.nodes { nodes = append(nodes, st.MakeNode().Name(n).Obj()) @@ -1447,6 +1455,7 @@ func TestPodEligibleToPreemptOthers(t *testing.T) { } f, err := st.NewFramework(registeredPlugins, "", frameworkruntime.WithSnapshotSharedLister(internalcache.NewSnapshot(test.pods, nodes)), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) @@ -1625,6 +1634,9 @@ func TestPreempt(t *testing.T) { labelKeys := []string{"hostname", "zone", "region"} for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) podInformer := informerFactory.Core().V1().Pods().Informer() @@ -1639,12 +1651,9 @@ func TestPreempt(t *testing.T) { return true, nil, nil }) - stop := make(chan struct{}) - defer close(stop) - - cache := internalcache.New(time.Duration(0), stop) + cache := internalcache.New(ctx, time.Duration(0)) for _, pod := range test.pods { - cache.AddPod(pod) + cache.AddPod(logger, pod) } cachedNodeInfoMap := map[string]*framework.NodeInfo{} nodes := make([]*v1.Node, len(test.nodeNames)) @@ -1657,7 +1666,7 @@ func TestPreempt(t *testing.T) { node.ObjectMeta.Labels[labelKeys[i]] = label } node.Name = node.ObjectMeta.Labels["hostname"] - cache.AddNode(node) + cache.AddNode(logger, node) nodes[i] = node // Set nodeInfo to extenders to mock extenders' cache for preemption. @@ -1684,6 +1693,7 @@ func TestPreempt(t *testing.T) { frameworkruntime.WithPodNominator(internalqueue.NewPodNominator(informerFactory.Core().V1().Pods().Lister())), frameworkruntime.WithSnapshotSharedLister(internalcache.NewSnapshot(test.pods, nodes)), frameworkruntime.WithInformerFactory(informerFactory), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) diff --git a/pkg/scheduler/framework/plugins/examples/stateful/stateful.go b/pkg/scheduler/framework/plugins/examples/stateful/stateful.go index 2eb22468da82..ecd7eb68a550 100644 --- a/pkg/scheduler/framework/plugins/examples/stateful/stateful.go +++ b/pkg/scheduler/framework/plugins/examples/stateful/stateful.go @@ -23,7 +23,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -82,9 +81,9 @@ func (mp *MultipointExample) PreBind(ctx context.Context, state *framework.Cycle } // New initializes a new plugin and returns it. -func New(config *runtime.Unknown, _ framework.Handle) (framework.Plugin, error) { +func New(config *runtime.Unknown, fh framework.Handle) (framework.Plugin, error) { if config == nil { - klog.ErrorS(nil, "MultipointExample configuration cannot be empty") + fh.Logger().Error(nil, "MultipointExample configuration cannot be empty") return nil, fmt.Errorf("MultipointExample configuration cannot be empty") } mp := MultipointExample{} diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/filtering.go b/pkg/scheduler/framework/plugins/interpodaffinity/filtering.go index 778a70f2c7bc..249bd11390ce 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/filtering.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/filtering.go @@ -159,7 +159,7 @@ func (pl *InterPodAffinity) getExistingAntiAffinityCounts(ctx context.Context, p nodeInfo := nodes[i] node := nodeInfo.Node() if node == nil { - klog.ErrorS(nil, "Node not found") + klog.FromContext(ctx).Error(nil, "Node not found") return } topoMap := make(topologyToMatchedTermCount) @@ -198,7 +198,7 @@ func (pl *InterPodAffinity) getIncomingAffinityAntiAffinityCounts(ctx context.Co nodeInfo := allNodes[i] node := nodeInfo.Node() if node == nil { - klog.ErrorS(nil, "Node not found") + klog.FromContext(ctx).Error(nil, "Node not found") return } affinity := make(topologyToMatchedTermCount) @@ -255,7 +255,8 @@ func (pl *InterPodAffinity) PreFilter(ctx context.Context, cycleState *framework return nil, framework.AsStatus(err) } } - s.namespaceLabels = GetNamespaceLabelsSnapshot(pod.Namespace, pl.nsLister) + logger := klog.FromContext(ctx) + s.namespaceLabels = GetNamespaceLabelsSnapshot(logger, pod.Namespace, pl.nsLister) s.existingAntiAffinityCounts = pl.getExistingAntiAffinityCounts(ctx, pod, s.namespaceLabels, nodesWithRequiredAntiAffinityPods) s.affinityCounts, s.antiAffinityCounts = pl.getIncomingAffinityAntiAffinityCounts(ctx, s.podInfo, allNodes) diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go b/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go index ce0aa6b6beda..8081c39b2f7b 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go @@ -26,6 +26,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" plugintesting "k8s.io/kubernetes/pkg/scheduler/framework/plugins/testing" @@ -1434,10 +1435,11 @@ func TestGetTPMapMatchingIncomingAffinityAntiAffinity(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() snapshot := cache.NewSnapshot(tt.existingPods, tt.nodes) l, _ := snapshot.NodeInfos().List() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() p := plugintesting.SetupPluginWithInformers(ctx, t, New, &config.InterPodAffinityArgs{}, snapshot, nil) gotAffinityPodsMap, gotAntiAffinityPodsMap := p.(*InterPodAffinity).getIncomingAffinityAntiAffinityCounts(ctx, framework.NewPodInfo(tt.pod), l) if !reflect.DeepEqual(gotAffinityPodsMap, tt.wantAffinityPodsMap) { diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/plugin.go b/pkg/scheduler/framework/plugins/interpodaffinity/plugin.go index f7e84ce09d96..2ca14362f7b0 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/plugin.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/plugin.go @@ -122,12 +122,12 @@ func (pl *InterPodAffinity) mergeAffinityTermNamespacesIfNotEmpty(at *framework. // GetNamespaceLabelsSnapshot returns a snapshot of the labels associated with // the namespace. -func GetNamespaceLabelsSnapshot(ns string, nsLister listersv1.NamespaceLister) (nsLabels labels.Set) { +func GetNamespaceLabelsSnapshot(logger klog.Logger, ns string, nsLister listersv1.NamespaceLister) (nsLabels labels.Set) { podNS, err := nsLister.Get(ns) if err == nil { // Create and return snapshot of the labels. return labels.Merge(podNS.Labels, nil) } - klog.V(3).InfoS("getting namespace, assuming empty set of namespace labels", "namespace", ns, "err", err) + logger.V(3).Info("getting namespace, assuming empty set of namespace labels", "namespace", ns, "err", err) return } diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/scoring.go b/pkg/scheduler/framework/plugins/interpodaffinity/scoring.go index f333bb6e3ca8..058dac274916 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/scoring.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/scoring.go @@ -22,8 +22,9 @@ import ( "math" "sync/atomic" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -178,7 +179,8 @@ func (pl *InterPodAffinity) PreScore( return framework.AsStatus(fmt.Errorf("updating PreferredAntiAffinityTerms: %w", err)) } } - state.namespaceLabels = GetNamespaceLabelsSnapshot(pod.Namespace, pl.nsLister) + logger := klog.FromContext(pCtx) + state.namespaceLabels = GetNamespaceLabelsSnapshot(logger, pod.Namespace, pl.nsLister) topoScores := make([]scoreMap, len(allNodes)) index := int32(-1) diff --git a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go index 4f91937ba542..3904ea277013 100644 --- a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go @@ -23,6 +23,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" "k8s.io/kubernetes/pkg/scheduler/framework" @@ -54,12 +55,13 @@ func (ba *BalancedAllocation) Score(ctx context.Context, state *framework.CycleS return 0, framework.AsStatus(fmt.Errorf("getting node %q from Snapshot: %w", nodeName, err)) } + logger := klog.FromContext(ctx) // ba.score favors nodes with balanced resource usage rate. // It calculates the standard deviation for those resources and prioritizes the node based on how close the usage of those resources is to each other. // Detail: score = (1 - std) * MaxNodeScore, where std is calculated by the root square of Σ((fraction(i)-mean)^2)/len(resources) // The algorithm is partly inspired by: // "Wei Huang et al. An Energy Efficient Virtual Machine Placement Algorithm with Balanced Resource Utilization" - return ba.score(pod, nodeInfo) + return ba.score(logger, pod, nodeInfo) } // ScoreExtensions of the Score plugin. diff --git a/pkg/scheduler/framework/plugins/noderesources/fit.go b/pkg/scheduler/framework/plugins/noderesources/fit.go index e4b2a5044dd5..a91d97ae5aef 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" @@ -331,5 +332,6 @@ func (f *Fit) Score(ctx context.Context, state *framework.CycleState, pod *v1.Po return 0, framework.AsStatus(fmt.Errorf("getting node %q from Snapshot: %w", nodeName, err)) } - return f.score(pod, nodeInfo) + logger := klog.FromContext(ctx) + return f.score(logger, pod, nodeInfo) } diff --git a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go index 48eadc88ff09..50617fff1bde 100644 --- a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go @@ -45,6 +45,7 @@ type resourceToValueMap map[v1.ResourceName]int64 // score will use `scorer` function to calculate the score. func (r *resourceAllocationScorer) score( + logger klog.Logger, pod *v1.Pod, nodeInfo *framework.NodeInfo) (int64, *framework.Status) { node := nodeInfo.Node() @@ -58,7 +59,7 @@ func (r *resourceAllocationScorer) score( requested := make(resourceToValueMap) allocatable := make(resourceToValueMap) for resource := range r.resourceToWeightMap { - alloc, req := r.calculateResourceAllocatableRequest(nodeInfo, pod, resource) + alloc, req := r.calculateResourceAllocatableRequest(logger, nodeInfo, pod, resource) if alloc != 0 { // Only fill the extended resource entry when it's non-zero. allocatable[resource], requested[resource] = alloc, req @@ -67,7 +68,7 @@ func (r *resourceAllocationScorer) score( score := r.scorer(requested, allocatable) - klog.V(10).InfoS("Listing internal info for allocatable resources, requested resources and score", "pod", + logger.V(10).Info("Listing internal info for allocatable resources, requested resources and score", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceAllocationScorer", r.Name, "allocatableResource", allocatable, "requestedResource", requested, "resourceScore", score, ) @@ -79,7 +80,7 @@ func (r *resourceAllocationScorer) score( // - 1st param: quantity of allocatable resource on the node. // - 2nd param: aggregated quantity of requested resource on the node. // Note: if it's an extended resource, and the pod doesn't request it, (0, 0) is returned. -func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(nodeInfo *framework.NodeInfo, pod *v1.Pod, resource v1.ResourceName) (int64, int64) { +func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(logger klog.Logger, nodeInfo *framework.NodeInfo, pod *v1.Pod, resource v1.ResourceName) (int64, int64) { requested := nodeInfo.NonZeroRequested if r.useRequested { requested = nodeInfo.Requested @@ -103,7 +104,7 @@ func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(nodeInfo return nodeInfo.Allocatable.ScalarResources[resource], (nodeInfo.Requested.ScalarResources[resource] + podRequest) } } - klog.V(10).InfoS("Requested resource is omitted for node score calculation", "resourceName", resource) + logger.V(10).Info("Requested resource is omitted for node score calculation", "resourceName", resource) return 0, 0 } diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go index 8c33826d488e..03f55f98fb89 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go @@ -92,15 +92,17 @@ func (pl *CSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod *v return framework.NewStatus(framework.Error, "node not found") } + logger := klog.FromContext(ctx) + // If CSINode doesn't exist, the predicate may read the limits from Node object csiNode, err := pl.csiNodeLister.Get(node.Name) if err != nil { // TODO: return the error once CSINode is created by default (2 releases) - klog.V(5).InfoS("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) + logger.V(5).Info("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) } newVolumes := make(map[string]string) - if err := pl.filterAttachableVolumes(pod, csiNode, true /* new pod */, newVolumes); err != nil { + if err := pl.filterAttachableVolumes(logger, pod, csiNode, true /* new pod */, newVolumes); err != nil { return framework.AsStatus(err) } @@ -117,7 +119,7 @@ func (pl *CSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod *v attachedVolumes := make(map[string]string) for _, existingPod := range nodeInfo.Pods { - if err := pl.filterAttachableVolumes(existingPod.Pod, csiNode, false /* existing pod */, attachedVolumes); err != nil { + if err := pl.filterAttachableVolumes(logger, existingPod.Pod, csiNode, false /* existing pod */, attachedVolumes); err != nil { return framework.AsStatus(err) } } @@ -138,7 +140,7 @@ func (pl *CSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod *v maxVolumeLimit, ok := nodeVolumeLimits[v1.ResourceName(volumeLimitKey)] if ok { currentVolumeCount := attachedVolumeCount[volumeLimitKey] - klog.V(5).InfoS("Found plugin volume limits", "node", node.Name, "volumeLimitKey", volumeLimitKey, + logger.V(5).Info("Found plugin volume limits", "node", node.Name, "volumeLimitKey", volumeLimitKey, "maxLimits", maxVolumeLimit, "currentVolumeCount", currentVolumeCount, "newVolumeCount", count, "pod", klog.KObj(pod)) if currentVolumeCount+count > int(maxVolumeLimit) { @@ -151,7 +153,7 @@ func (pl *CSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod *v } func (pl *CSILimits) filterAttachableVolumes( - pod *v1.Pod, csiNode *storagev1.CSINode, newPod bool, result map[string]string) error { + logger klog.Logger, pod *v1.Pod, csiNode *storagev1.CSINode, newPod bool, result map[string]string) error { for _, vol := range pod.Spec.Volumes { pvcName := "" isEphemeral := false @@ -172,7 +174,7 @@ func (pl *CSILimits) filterAttachableVolumes( // - If the volume is migratable and CSI migration is enabled, need to count it // as well. // - If the volume is not migratable, it will be count in non_csi filter. - if err := pl.checkAttachableInlineVolume(vol, csiNode, pod, result); err != nil { + if err := pl.checkAttachableInlineVolume(logger, vol, csiNode, pod, result); err != nil { return err } @@ -194,7 +196,7 @@ func (pl *CSILimits) filterAttachableVolumes( } // If the PVC is invalid, we don't count the volume because // there's no guarantee that it belongs to the running predicate. - klog.V(5).InfoS("Unable to look up PVC info", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName)) + logger.V(5).Info("Unable to look up PVC info", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName)) continue } @@ -205,9 +207,9 @@ func (pl *CSILimits) filterAttachableVolumes( } } - driverName, volumeHandle := pl.getCSIDriverInfo(csiNode, pvc) + driverName, volumeHandle := pl.getCSIDriverInfo(logger, csiNode, pvc) if driverName == "" || volumeHandle == "" { - klog.V(5).InfoS("Could not find a CSI driver name or volume handle, not counting volume") + logger.V(5).Info("Could not find a CSI driver name or volume handle, not counting volume") continue } @@ -220,7 +222,7 @@ func (pl *CSILimits) filterAttachableVolumes( // checkAttachableInlineVolume takes an inline volume and add to the result map if the // volume is migratable and CSI migration for this plugin has been enabled. -func (pl *CSILimits) checkAttachableInlineVolume(vol v1.Volume, csiNode *storagev1.CSINode, +func (pl *CSILimits) checkAttachableInlineVolume(logger klog.Logger, vol v1.Volume, csiNode *storagev1.CSINode, pod *v1.Pod, result map[string]string) error { if !pl.translator.IsInlineMigratable(&vol) { return nil @@ -231,7 +233,7 @@ func (pl *CSILimits) checkAttachableInlineVolume(vol v1.Volume, csiNode *storage return fmt.Errorf("looking up provisioner name for volume %v: %w", vol, err) } if !isCSIMigrationOn(csiNode, inTreeProvisionerName) { - klog.V(5).InfoS("CSI Migration is not enabled for provisioner", "provisioner", inTreeProvisionerName, + logger.V(5).Info("CSI Migration is not enabled for provisioner", "provisioner", inTreeProvisionerName, "pod", klog.KObj(pod), "csiNode", csiNode.Name) return nil } @@ -258,21 +260,21 @@ func (pl *CSILimits) checkAttachableInlineVolume(vol v1.Volume, csiNode *storage // getCSIDriverInfo returns the CSI driver name and volume ID of a given PVC. // If the PVC is from a migrated in-tree plugin, this function will return // the information of the CSI driver that the plugin has been migrated to. -func (pl *CSILimits) getCSIDriverInfo(csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { +func (pl *CSILimits) getCSIDriverInfo(logger klog.Logger, csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { pvName := pvc.Spec.VolumeName if pvName == "" { - klog.V(5).InfoS("Persistent volume had no name for claim", "PVC", klog.KObj(pvc)) - return pl.getCSIDriverInfoFromSC(csiNode, pvc) + logger.V(5).Info("Persistent volume had no name for claim", "PVC", klog.KObj(pvc)) + return pl.getCSIDriverInfoFromSC(logger, csiNode, pvc) } pv, err := pl.pvLister.Get(pvName) if err != nil { - klog.V(5).InfoS("Unable to look up PV info for PVC and PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvName)) + logger.V(5).Info("Unable to look up PV info for PVC and PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvName)) // If we can't fetch PV associated with PVC, may be it got deleted // or PVC was prebound to a PVC that hasn't been created yet. // fallback to using StorageClass for volume counting - return pl.getCSIDriverInfoFromSC(csiNode, pvc) + return pl.getCSIDriverInfoFromSC(logger, csiNode, pvc) } csiSource := pv.Spec.PersistentVolumeSource.CSI @@ -284,23 +286,23 @@ func (pl *CSILimits) getCSIDriverInfo(csiNode *storagev1.CSINode, pvc *v1.Persis pluginName, err := pl.translator.GetInTreePluginNameFromSpec(pv, nil) if err != nil { - klog.V(5).InfoS("Unable to look up plugin name from PV spec", "err", err) + logger.V(5).Info("Unable to look up plugin name from PV spec", "err", err) return "", "" } if !isCSIMigrationOn(csiNode, pluginName) { - klog.V(5).InfoS("CSI Migration of plugin is not enabled", "plugin", pluginName) + logger.V(5).Info("CSI Migration of plugin is not enabled", "plugin", pluginName) return "", "" } csiPV, err := pl.translator.TranslateInTreePVToCSI(pv) if err != nil { - klog.V(5).InfoS("Unable to translate in-tree volume to CSI", "err", err) + logger.V(5).Info("Unable to translate in-tree volume to CSI", "err", err) return "", "" } if csiPV.Spec.PersistentVolumeSource.CSI == nil { - klog.V(5).InfoS("Unable to get a valid volume source for translated PV", "PV", pvName) + logger.V(5).Info("Unable to get a valid volume source for translated PV", "PV", pvName) return "", "" } @@ -311,7 +313,7 @@ func (pl *CSILimits) getCSIDriverInfo(csiNode *storagev1.CSINode, pvc *v1.Persis } // getCSIDriverInfoFromSC returns the CSI driver name and a random volume ID of a given PVC's StorageClass. -func (pl *CSILimits) getCSIDriverInfoFromSC(csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { +func (pl *CSILimits) getCSIDriverInfoFromSC(logger klog.Logger, csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { namespace := pvc.Namespace pvcName := pvc.Name scName := storagehelpers.GetPersistentVolumeClaimClass(pvc) @@ -319,13 +321,13 @@ func (pl *CSILimits) getCSIDriverInfoFromSC(csiNode *storagev1.CSINode, pvc *v1. // If StorageClass is not set or not found, then PVC must be using immediate binding mode // and hence it must be bound before scheduling. So it is safe to not count it. if scName == "" { - klog.V(5).InfoS("PVC has no StorageClass", "PVC", klog.KObj(pvc)) + logger.V(5).Info("PVC has no StorageClass", "PVC", klog.KObj(pvc)) return "", "" } storageClass, err := pl.scLister.Get(scName) if err != nil { - klog.V(5).InfoS("Could not get StorageClass for PVC", "PVC", klog.KObj(pvc), "err", err) + logger.V(5).Info("Could not get StorageClass for PVC", "PVC", klog.KObj(pvc), "err", err) return "", "" } @@ -337,13 +339,13 @@ func (pl *CSILimits) getCSIDriverInfoFromSC(csiNode *storagev1.CSINode, pvc *v1. provisioner := storageClass.Provisioner if pl.translator.IsMigratableIntreePluginByName(provisioner) { if !isCSIMigrationOn(csiNode, provisioner) { - klog.V(5).InfoS("CSI Migration of provisioner is not enabled", "provisioner", provisioner) + logger.V(5).Info("CSI Migration of provisioner is not enabled", "provisioner", provisioner) return "", "" } driverName, err := pl.translator.GetCSINameFromInTreeName(provisioner) if err != nil { - klog.V(5).InfoS("Unable to look up driver name from provisioner name", "provisioner", provisioner, "err", err) + logger.V(5).Info("Unable to look up driver name from provisioner name", "provisioner", provisioner, "err", err) return "", "" } return driverName, volumeHandle diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi.go b/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi.go index 4a2535e5f26b..eb4cfbfb6ebe 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi.go @@ -72,7 +72,7 @@ const AzureDiskName = names.AzureDiskLimits // NewAzureDisk returns function that initializes a new plugin and returns it. func NewAzureDisk(_ runtime.Object, handle framework.Handle, fts feature.Features) (framework.Plugin, error) { informerFactory := handle.SharedInformerFactory() - return newNonCSILimitsWithInformerFactory(azureDiskVolumeFilterType, informerFactory, fts), nil + return newNonCSILimitsWithInformerFactory(handle.Logger(), azureDiskVolumeFilterType, informerFactory, fts), nil } // CinderName is the name of the plugin used in the plugin registry and configurations. @@ -81,7 +81,7 @@ const CinderName = names.CinderLimits // NewCinder returns function that initializes a new plugin and returns it. func NewCinder(_ runtime.Object, handle framework.Handle, fts feature.Features) (framework.Plugin, error) { informerFactory := handle.SharedInformerFactory() - return newNonCSILimitsWithInformerFactory(cinderVolumeFilterType, informerFactory, fts), nil + return newNonCSILimitsWithInformerFactory(handle.Logger(), cinderVolumeFilterType, informerFactory, fts), nil } // EBSName is the name of the plugin used in the plugin registry and configurations. @@ -90,7 +90,7 @@ const EBSName = names.EBSLimits // NewEBS returns function that initializes a new plugin and returns it. func NewEBS(_ runtime.Object, handle framework.Handle, fts feature.Features) (framework.Plugin, error) { informerFactory := handle.SharedInformerFactory() - return newNonCSILimitsWithInformerFactory(ebsVolumeFilterType, informerFactory, fts), nil + return newNonCSILimitsWithInformerFactory(handle.Logger(), ebsVolumeFilterType, informerFactory, fts), nil } // GCEPDName is the name of the plugin used in the plugin registry and configurations. @@ -99,7 +99,7 @@ const GCEPDName = names.GCEPDLimits // NewGCEPD returns function that initializes a new plugin and returns it. func NewGCEPD(_ runtime.Object, handle framework.Handle, fts feature.Features) (framework.Plugin, error) { informerFactory := handle.SharedInformerFactory() - return newNonCSILimitsWithInformerFactory(gcePDVolumeFilterType, informerFactory, fts), nil + return newNonCSILimitsWithInformerFactory(handle.Logger(), gcePDVolumeFilterType, informerFactory, fts), nil } // nonCSILimits contains information to check the max number of volumes for a plugin. @@ -124,6 +124,7 @@ var _ framework.EnqueueExtensions = &nonCSILimits{} // newNonCSILimitsWithInformerFactory returns a plugin with filter name and informer factory. func newNonCSILimitsWithInformerFactory( + logger klog.Logger, filterName string, informerFactory informers.SharedInformerFactory, fts feature.Features, @@ -133,7 +134,7 @@ func newNonCSILimitsWithInformerFactory( csiNodesLister := informerFactory.Storage().V1().CSINodes().Lister() scLister := informerFactory.Storage().V1().StorageClasses().Lister() - return newNonCSILimits(filterName, csiNodesLister, scLister, pvLister, pvcLister, fts) + return newNonCSILimits(logger, filterName, csiNodesLister, scLister, pvLister, pvcLister, fts) } // newNonCSILimits creates a plugin which evaluates whether a pod can fit based on the @@ -147,6 +148,7 @@ func newNonCSILimitsWithInformerFactory( // types, counts the number of unique volumes, and rejects the new pod if it would place the total count over // the maximum. func newNonCSILimits( + logger klog.Logger, filterName string, csiNodeLister storagelisters.CSINodeLister, scLister storagelisters.StorageClassLister, @@ -176,14 +178,14 @@ func newNonCSILimits( filter = cinderVolumeFilter volumeLimitKey = v1.ResourceName(volumeutil.CinderVolumeLimitKey) default: - klog.ErrorS(errors.New("wrong filterName"), "Cannot create nonCSILimits plugin") + logger.Error(errors.New("wrong filterName"), "Cannot create nonCSILimits plugin") return nil } pl := &nonCSILimits{ name: name, filter: filter, volumeLimitKey: volumeLimitKey, - maxVolumeFunc: getMaxVolumeFunc(filterName), + maxVolumeFunc: getMaxVolumeFunc(logger, filterName), csiNodeLister: csiNodeLister, pvLister: pvLister, pvcLister: pvcLister, @@ -216,8 +218,9 @@ func (pl *nonCSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod return nil } + logger := klog.FromContext(ctx) newVolumes := make(sets.String) - if err := pl.filterVolumes(pod, true /* new pod */, newVolumes); err != nil { + if err := pl.filterVolumes(logger, pod, true /* new pod */, newVolumes); err != nil { return framework.AsStatus(err) } @@ -238,7 +241,7 @@ func (pl *nonCSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod if err != nil { // we don't fail here because the CSINode object is only necessary // for determining whether the migration is enabled or not - klog.V(5).InfoS("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) + logger.V(5).Info("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) } } @@ -250,7 +253,7 @@ func (pl *nonCSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod // count unique volumes existingVolumes := make(sets.String) for _, existingPod := range nodeInfo.Pods { - if err := pl.filterVolumes(existingPod.Pod, false /* existing pod */, existingVolumes); err != nil { + if err := pl.filterVolumes(logger, existingPod.Pod, false /* existing pod */, existingVolumes); err != nil { return framework.AsStatus(err) } } @@ -274,7 +277,7 @@ func (pl *nonCSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod return nil } -func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes sets.String) error { +func (pl *nonCSILimits) filterVolumes(logger klog.Logger, pod *v1.Pod, newPod bool, filteredVolumes sets.String) error { volumes := pod.Spec.Volumes for i := range volumes { vol := &volumes[i] @@ -317,7 +320,7 @@ func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes } // If the PVC is invalid, we don't count the volume because // there's no guarantee that it belongs to the running predicate. - klog.V(4).InfoS("Unable to look up PVC info, assuming PVC doesn't match predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName), "err", err) + logger.V(4).Info("Unable to look up PVC info, assuming PVC doesn't match predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName), "err", err) continue } @@ -335,7 +338,7 @@ func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes // original PV where it was bound to, so we count the volume if // it belongs to the running predicate. if pl.matchProvisioner(pvc) { - klog.V(4).InfoS("PVC is not bound, assuming PVC matches predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName)) + logger.V(4).Info("PVC is not bound, assuming PVC matches predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName)) filteredVolumes.Insert(pvID) } continue @@ -346,7 +349,7 @@ func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes // If the PV is invalid and PVC belongs to the running predicate, // log the error and count the PV towards the PV limit. if pl.matchProvisioner(pvc) { - klog.V(4).InfoS("Unable to look up PV, assuming PV matches predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName), "PV", klog.KRef("", pvName), "err", err) + logger.V(4).Info("Unable to look up PV, assuming PV matches predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName), "PV", klog.KRef("", pvName), "err", err) filteredVolumes.Insert(pvID) } continue @@ -375,12 +378,12 @@ func (pl *nonCSILimits) matchProvisioner(pvc *v1.PersistentVolumeClaim) bool { } // getMaxVolLimitFromEnv checks the max PD volumes environment variable, otherwise returning a default value. -func getMaxVolLimitFromEnv() int { +func getMaxVolLimitFromEnv(logger klog.Logger) int { if rawMaxVols := os.Getenv(KubeMaxPDVols); rawMaxVols != "" { if parsedMaxVols, err := strconv.Atoi(rawMaxVols); err != nil { - klog.ErrorS(err, "Unable to parse maximum PD volumes value, using default") + logger.Error(err, "Unable to parse maximum PD volumes value, using default") } else if parsedMaxVols <= 0 { - klog.ErrorS(errors.New("maximum PD volumes is negative"), "Unable to parse maximum PD volumes value, using default") + logger.Error(errors.New("maximum PD volumes is negative"), "Unable to parse maximum PD volumes value, using default") } else { return parsedMaxVols } @@ -501,9 +504,9 @@ var cinderVolumeFilter = VolumeFilter{ }, } -func getMaxVolumeFunc(filterName string) func(node *v1.Node) int { +func getMaxVolumeFunc(logger klog.Logger, filterName string) func(node *v1.Node) int { return func(node *v1.Node) int { - maxVolumesFromEnv := getMaxVolLimitFromEnv() + maxVolumesFromEnv := getMaxVolLimitFromEnv(logger) if maxVolumesFromEnv > 0 { return maxVolumesFromEnv } diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi_test.go index 922abd071dbb..f22e401aa6a3 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi_test.go @@ -27,6 +27,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" csilibplugins "k8s.io/csi-translation-lib/plugins" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/framework" fakeframework "k8s.io/kubernetes/pkg/scheduler/framework/fake" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" @@ -154,9 +155,10 @@ func TestEphemeralLimits(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) fts := feature.Features{} node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), filterName) - p := newNonCSILimits(filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(filterName, driverName), getFakePVLister(filterName), append(getFakePVCLister(filterName), test.extraClaims...), fts).(framework.FilterPlugin) + p := newNonCSILimits(logger, filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(filterName, driverName), getFakePVLister(filterName), append(getFakePVCLister(filterName), test.extraClaims...), fts).(framework.FilterPlugin) gotStatus := p.Filter(context.Background(), nil, test.newPod, node) if !reflect.DeepEqual(gotStatus, test.wantStatus) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) @@ -319,8 +321,9 @@ func TestAzureDiskLimits(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) - p := newNonCSILimits(test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) + p := newNonCSILimits(logger, test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) gotStatus := p.Filter(context.Background(), nil, test.newPod, node) if !reflect.DeepEqual(gotStatus, test.wantStatus) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) @@ -529,8 +532,9 @@ func TestEBSLimits(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) - p := newNonCSILimits(test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) + p := newNonCSILimits(logger, test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) gotStatus := p.Filter(context.Background(), nil, test.newPod, node) if !reflect.DeepEqual(gotStatus, test.wantStatus) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) @@ -693,8 +697,9 @@ func TestGCEPDLimits(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) - p := newNonCSILimits(test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) + p := newNonCSILimits(logger, test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) gotStatus := p.Filter(context.Background(), nil, test.newPod, node) if !reflect.DeepEqual(gotStatus, test.wantStatus) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) @@ -730,8 +735,9 @@ func TestGetMaxVols(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) os.Setenv(KubeMaxPDVols, test.rawMaxVols) - result := getMaxVolLimitFromEnv() + result := getMaxVolLimitFromEnv(logger) if result != test.expected { t.Errorf("expected %v got %v", test.expected, result) } diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go index 888750d03a0e..67dfe799cd11 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go @@ -262,7 +262,7 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod) nodeInfo := allNodes[i] node := nodeInfo.Node() if node == nil { - klog.ErrorS(nil, "Node not found") + klog.FromContext(ctx).Error(nil, "Node not found") return } @@ -335,12 +335,13 @@ func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.C return nil } + logger := klog.FromContext(ctx) podLabelSet := labels.Set(pod.Labels) for _, c := range s.Constraints { tpKey := c.TopologyKey tpVal, ok := node.Labels[c.TopologyKey] if !ok { - klog.V(5).InfoS("Node doesn't have required label", "node", klog.KObj(node), "label", tpKey) + logger.V(5).Info("Node doesn't have required label", "node", klog.KObj(node), "label", tpKey) return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonNodeLabelNotMatch) } @@ -348,7 +349,7 @@ func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.C // 'existing matching num' + 'if self-match (1 or 0)' - 'global minimum' <= 'maxSkew' minMatchNum, err := s.minMatchNum(tpKey, c.MinDomains, pl.enableMinDomainsInPodTopologySpread) if err != nil { - klog.ErrorS(err, "Internal error occurred while retrieving value precalculated in PreFilter", "topologyKey", tpKey, "paths", s.TpKeyToCriticalPaths) + logger.Error(err, "Internal error occurred while retrieving value precalculated in PreFilter", "topologyKey", tpKey, "paths", s.TpKeyToCriticalPaths) continue } @@ -364,7 +365,7 @@ func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.C } skew := matchNum + selfMatchNum - minMatchNum if skew > int(c.MaxSkew) { - klog.V(5).InfoS("Node failed spreadConstraint: matchNum + selfMatchNum - minMatchNum > maxSkew", "node", klog.KObj(node), "topologyKey", tpKey, "matchNum", matchNum, "selfMatchNum", selfMatchNum, "minMatchNum", minMatchNum, "maxSkew", c.MaxSkew) + logger.V(5).Info("Node failed spreadConstraint: matchNum + selfMatchNum - minMatchNum > maxSkew", "node", klog.KObj(node), "topologyKey", tpKey, "matchNum", matchNum, "selfMatchNum", selfMatchNum, "minMatchNum", minMatchNum, "maxSkew", c.MaxSkew) return framework.NewStatus(framework.Unschedulable, ErrReasonConstraintsNotMatch) } } diff --git a/pkg/scheduler/framework/plugins/volumebinding/assume_cache.go b/pkg/scheduler/framework/plugins/volumebinding/assume_cache.go index 1ee97a3eb24b..e06194086aac 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/assume_cache.go +++ b/pkg/scheduler/framework/plugins/volumebinding/assume_cache.go @@ -87,6 +87,10 @@ func (e *errObjectName) Error() string { // Restore() sets the latest object pointer back to the informer object. // Get/List() always returns the latest object pointer. type assumeCache struct { + // The logger that was chosen when setting up the cache. + // Will be used for all operations. + logger klog.Logger + // Synchronizes updates to store rwMutex sync.RWMutex @@ -129,8 +133,9 @@ func (c *assumeCache) objInfoIndexFunc(obj interface{}) ([]string, error) { } // NewAssumeCache creates an assume cache for general objects. -func NewAssumeCache(informer cache.SharedIndexInformer, description, indexName string, indexFunc cache.IndexFunc) AssumeCache { +func NewAssumeCache(logger klog.Logger, informer cache.SharedIndexInformer, description, indexName string, indexFunc cache.IndexFunc) AssumeCache { c := &assumeCache{ + logger: logger, description: description, indexFunc: indexFunc, indexName: indexName, @@ -161,7 +166,7 @@ func (c *assumeCache) add(obj interface{}) { name, err := cache.MetaNamespaceKeyFunc(obj) if err != nil { - klog.ErrorS(&errObjectName{err}, "Add failed") + c.logger.Error(&errObjectName{err}, "Add failed") return } @@ -171,29 +176,29 @@ func (c *assumeCache) add(obj interface{}) { if objInfo, _ := c.getObjInfo(name); objInfo != nil { newVersion, err := c.getObjVersion(name, obj) if err != nil { - klog.ErrorS(err, "Add failed: couldn't get object version") + c.logger.Error(err, "Add failed: couldn't get object version") return } storedVersion, err := c.getObjVersion(name, objInfo.latestObj) if err != nil { - klog.ErrorS(err, "Add failed: couldn't get stored object version") + c.logger.Error(err, "Add failed: couldn't get stored object version") return } // Only update object if version is newer. // This is so we don't override assumed objects due to informer resync. if newVersion <= storedVersion { - klog.V(10).InfoS("Skip adding object to assume cache because version is not newer than storedVersion", "description", c.description, "cacheKey", name, "newVersion", newVersion, "storedVersion", storedVersion) + c.logger.V(10).Info("Skip adding object to assume cache because version is not newer than storedVersion", "description", c.description, "cacheKey", name, "newVersion", newVersion, "storedVersion", storedVersion) return } } objInfo := &objInfo{name: name, latestObj: obj, apiObj: obj} if err = c.store.Update(objInfo); err != nil { - klog.InfoS("Error occurred while updating stored object", "err", err) + c.logger.Info("Error occurred while updating stored object", "err", err) } else { - klog.V(10).InfoS("Adding object to assume cache", "description", c.description, "cacheKey", name, "assumeCache", obj) + c.logger.V(10).Info("Adding object to assume cache", "description", c.description, "cacheKey", name, "assumeCache", obj) } } @@ -208,7 +213,7 @@ func (c *assumeCache) delete(obj interface{}) { name, err := cache.MetaNamespaceKeyFunc(obj) if err != nil { - klog.ErrorS(&errObjectName{err}, "Failed to delete") + c.logger.Error(&errObjectName{err}, "Failed to delete") return } @@ -218,7 +223,7 @@ func (c *assumeCache) delete(obj interface{}) { objInfo := &objInfo{name: name} err = c.store.Delete(objInfo) if err != nil { - klog.ErrorS(err, "Failed to delete", "description", c.description, "cacheKey", name) + c.logger.Error(err, "Failed to delete", "description", c.description, "cacheKey", name) } } @@ -280,14 +285,14 @@ func (c *assumeCache) List(indexObj interface{}) []interface{} { allObjs := []interface{}{} objs, err := c.store.Index(c.indexName, &objInfo{latestObj: indexObj}) if err != nil { - klog.ErrorS(err, "List index error") + c.logger.Error(err, "List index error") return nil } for _, obj := range objs { objInfo, ok := obj.(*objInfo) if !ok { - klog.ErrorS(&errWrongType{"objInfo", obj}, "List error") + c.logger.Error(&errWrongType{"objInfo", obj}, "List error") continue } allObjs = append(allObjs, objInfo.latestObj) @@ -325,7 +330,7 @@ func (c *assumeCache) Assume(obj interface{}) error { // Only update the cached object objInfo.latestObj = obj - klog.V(4).InfoS("Assumed object", "description", c.description, "cacheKey", name, "version", newVersion) + c.logger.V(4).Info("Assumed object", "description", c.description, "cacheKey", name, "version", newVersion) return nil } @@ -336,10 +341,10 @@ func (c *assumeCache) Restore(objName string) { objInfo, err := c.getObjInfo(objName) if err != nil { // This could be expected if object got deleted - klog.V(5).InfoS("Restore object", "description", c.description, "cacheKey", objName, "err", err) + c.logger.V(5).Info("Restore object", "description", c.description, "cacheKey", objName, "err", err) } else { objInfo.latestObj = objInfo.apiObj - klog.V(4).InfoS("Restored object", "description", c.description, "cacheKey", objName) + c.logger.V(4).Info("Restored object", "description", c.description, "cacheKey", objName) } } @@ -354,6 +359,7 @@ type PVAssumeCache interface { type pvAssumeCache struct { AssumeCache + logger klog.Logger } func pvStorageClassIndexFunc(obj interface{}) ([]string, error) { @@ -364,8 +370,12 @@ func pvStorageClassIndexFunc(obj interface{}) ([]string, error) { } // NewPVAssumeCache creates a PV assume cache. -func NewPVAssumeCache(informer cache.SharedIndexInformer) PVAssumeCache { - return &pvAssumeCache{NewAssumeCache(informer, "v1.PersistentVolume", "storageclass", pvStorageClassIndexFunc)} +func NewPVAssumeCache(logger klog.Logger, informer cache.SharedIndexInformer) PVAssumeCache { + logger = klog.LoggerWithName(logger, "PV Cache") + return &pvAssumeCache{ + AssumeCache: NewAssumeCache(logger, informer, "v1.PersistentVolume", "storageclass", pvStorageClassIndexFunc), + logger: logger, + } } func (c *pvAssumeCache) GetPV(pvName string) (*v1.PersistentVolume, error) { @@ -403,7 +413,7 @@ func (c *pvAssumeCache) ListPVs(storageClassName string) []*v1.PersistentVolume for _, obj := range objs { pv, ok := obj.(*v1.PersistentVolume) if !ok { - klog.ErrorS(&errWrongType{"v1.PersistentVolume", obj}, "ListPVs") + c.logger.Error(&errWrongType{"v1.PersistentVolume", obj}, "ListPVs") continue } pvs = append(pvs, pv) @@ -423,11 +433,16 @@ type PVCAssumeCache interface { type pvcAssumeCache struct { AssumeCache + logger klog.Logger } // NewPVCAssumeCache creates a PVC assume cache. -func NewPVCAssumeCache(informer cache.SharedIndexInformer) PVCAssumeCache { - return &pvcAssumeCache{NewAssumeCache(informer, "v1.PersistentVolumeClaim", "", nil)} +func NewPVCAssumeCache(logger klog.Logger, informer cache.SharedIndexInformer) PVCAssumeCache { + logger = klog.LoggerWithName(logger, "PVC Cache") + return &pvcAssumeCache{ + AssumeCache: NewAssumeCache(logger, informer, "v1.PersistentVolumeClaim", "", nil), + logger: logger, + } } func (c *pvcAssumeCache) GetPVC(pvcKey string) (*v1.PersistentVolumeClaim, error) { diff --git a/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go b/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go index 49fcb1cee852..6dcb0bff74ba 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go @@ -23,6 +23,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-helpers/storage/volume" + "k8s.io/klog/v2/ktesting" ) func verifyListPVs(t *testing.T, cache PVAssumeCache, expectedPVs map[string]*v1.PersistentVolume, storageClassName string) { @@ -53,6 +54,7 @@ func verifyPV(cache PVAssumeCache, name string, expectedPV *v1.PersistentVolume) } func TestAssumePV(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) scenarios := map[string]struct { oldPV *v1.PersistentVolume newPV *v1.PersistentVolume @@ -96,7 +98,7 @@ func TestAssumePV(t *testing.T) { } for name, scenario := range scenarios { - cache := NewPVAssumeCache(nil) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -130,7 +132,8 @@ func TestAssumePV(t *testing.T) { } func TestRestorePV(t *testing.T) { - cache := NewPVAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -170,7 +173,8 @@ func TestRestorePV(t *testing.T) { } func TestBasicPVCache(t *testing.T) { - cache := NewPVAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -214,7 +218,8 @@ func TestBasicPVCache(t *testing.T) { } func TestPVCacheWithStorageClasses(t *testing.T) { - cache := NewPVAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -260,7 +265,8 @@ func TestPVCacheWithStorageClasses(t *testing.T) { } func TestAssumeUpdatePVCache(t *testing.T) { - cache := NewPVAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -353,7 +359,8 @@ func TestAssumePVC(t *testing.T) { } for name, scenario := range scenarios { - cache := NewPVCAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVCAssumeCache(logger, nil) internalCache, ok := cache.(*pvcAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -387,7 +394,8 @@ func TestAssumePVC(t *testing.T) { } func TestRestorePVC(t *testing.T) { - cache := NewPVCAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVCAssumeCache(logger, nil) internalCache, ok := cache.(*pvcAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -427,7 +435,8 @@ func TestRestorePVC(t *testing.T) { } func TestAssumeUpdatePVCCache(t *testing.T) { - cache := NewPVCAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVCAssumeCache(logger, nil) internalCache, ok := cache.(*pvcAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder.go b/pkg/scheduler/framework/plugins/volumebinding/binder.go index f715699e52dc..020a31b00c87 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder.go @@ -146,7 +146,7 @@ type InTreeToCSITranslator interface { type SchedulerVolumeBinder interface { // GetPodVolumes returns a pod's PVCs separated into bound, unbound with delayed binding (including provisioning) // and unbound with immediate binding (including prebound) - GetPodVolumes(pod *v1.Pod) (boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) + GetPodVolumes(logger klog.Logger, pod *v1.Pod) (boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) // FindPodVolumes checks if all of a Pod's PVCs can be satisfied by the // node and returns pod's volumes information. @@ -161,7 +161,7 @@ type SchedulerVolumeBinder interface { // for volumes that still need to be created. // // This function is called by the scheduler VolumeBinding plugin and can be called in parallel - FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) + FindPodVolumes(logger klog.Logger, pod *v1.Pod, boundClaims, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) // AssumePodVolumes will: // 1. Take the PV matches for unbound PVCs and update the PV cache assuming @@ -172,10 +172,10 @@ type SchedulerVolumeBinder interface { // It returns true if all volumes are fully bound // // This function is called serially. - AssumePodVolumes(assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (allFullyBound bool, err error) + AssumePodVolumes(logger klog.Logger, assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (allFullyBound bool, err error) // RevertAssumedPodVolumes will revert assumed PV and PVC cache. - RevertAssumedPodVolumes(podVolumes *PodVolumes) + RevertAssumedPodVolumes(logger klog.Logger, podVolumes *PodVolumes) // BindPodVolumes will: // 1. Initiate the volume binding by making the API call to prebind the PV @@ -185,7 +185,7 @@ type SchedulerVolumeBinder interface { // 3. Wait for PVCs to be completely bound by the PV controller // // This function can be called in parallel. - BindPodVolumes(assumedPod *v1.Pod, podVolumes *PodVolumes) error + BindPodVolumes(logger klog.Logger, assumedPod *v1.Pod, podVolumes *PodVolumes) error } type volumeBinder struct { @@ -220,6 +220,7 @@ type CapacityCheck struct { // // capacityCheck determines how storage capacity is checked (CSIStorageCapacity feature). func NewVolumeBinder( + logger klog.Logger, kubeClient clientset.Interface, podInformer coreinformers.PodInformer, nodeInformer coreinformers.NodeInformer, @@ -235,8 +236,8 @@ func NewVolumeBinder( classLister: storageClassInformer.Lister(), nodeLister: nodeInformer.Lister(), csiNodeLister: csiNodeInformer.Lister(), - pvcCache: NewPVCAssumeCache(pvcInformer.Informer()), - pvCache: NewPVAssumeCache(pvInformer.Informer()), + pvcCache: NewPVCAssumeCache(logger, pvcInformer.Informer()), + pvCache: NewPVAssumeCache(logger, pvInformer.Informer()), bindTimeout: bindTimeout, translator: csitrans.New(), } @@ -250,11 +251,11 @@ func NewVolumeBinder( // FindPodVolumes finds the matching PVs for PVCs and nodes to provision PVs // for the given pod and node. If the node does not fit, confilict reasons are // returned. -func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) { +func (b *volumeBinder) FindPodVolumes(logger klog.Logger, pod *v1.Pod, boundClaims, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) { podVolumes = &PodVolumes{} // Warning: Below log needs high verbosity as it can be printed several times (#60933). - klog.V(5).InfoS("FindPodVolumes", "pod", klog.KObj(pod), "node", klog.KObj(node)) + logger.V(5).Info("FindPodVolumes starts") // Initialize to true for pods that don't have volumes. These // booleans get translated into reason strings when the function @@ -306,7 +307,7 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []* // Check PV node affinity on bound volumes if len(boundClaims) > 0 { - boundVolumesSatisfied, boundPVsFound, err = b.checkBoundClaims(boundClaims, node, pod) + boundVolumesSatisfied, boundPVsFound, err = b.checkBoundClaims(logger, boundClaims, node) if err != nil { return } @@ -336,7 +337,7 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []* // Find matching volumes if len(claimsToFindMatching) > 0 { var unboundClaims []*v1.PersistentVolumeClaim - unboundVolumesSatisfied, staticBindings, unboundClaims, err = b.findMatchingVolumes(pod, claimsToFindMatching, node) + unboundVolumesSatisfied, staticBindings, unboundClaims, err = b.findMatchingVolumes(logger, pod, claimsToFindMatching, node) if err != nil { return } @@ -346,7 +347,7 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []* // Check for claims to provision. This is the first time where we potentially // find out that storage is not sufficient for the node. if len(claimsToProvision) > 0 { - unboundVolumesSatisfied, sufficientStorage, dynamicProvisions, err = b.checkVolumeProvisions(pod, claimsToProvision, node) + unboundVolumesSatisfied, sufficientStorage, dynamicProvisions, err = b.checkVolumeProvisions(logger, pod, claimsToProvision, node) if err != nil { return } @@ -361,16 +362,15 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []* // 1. Update the pvCache with the new prebound PV. // 2. Update the pvcCache with the new PVCs with annotations set // 3. Update PodVolumes again with cached API updates for PVs and PVCs. -func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (allFullyBound bool, err error) { - klog.V(4).InfoS("AssumePodVolumes", "pod", klog.KObj(assumedPod), "node", klog.KRef("", nodeName)) +func (b *volumeBinder) AssumePodVolumes(logger klog.Logger, assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (allFullyBound bool, err error) { defer func() { if err != nil { metrics.VolumeSchedulingStageFailed.WithLabelValues("assume").Inc() } }() - if allBound := b.arePodVolumesBound(assumedPod); allBound { - klog.V(4).InfoS("AssumePodVolumes: all PVCs bound and nothing to do", "pod", klog.KObj(assumedPod), "node", klog.KRef("", nodeName)) + if allBound := b.arePodVolumesBound(logger, assumedPod); allBound { + logger.V(4).Info("All PVCs bound and nothing to do") return true, nil } @@ -378,7 +378,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, pod newBindings := []*BindingInfo{} for _, binding := range podVolumes.StaticBindings { newPV, dirty, err := volume.GetBindVolumeToClaim(binding.pv, binding.pvc) - klog.V(5).InfoS("AssumePodVolumes: GetBindVolumeToClaim", + logger.V(5).Info("AssumePodVolumes: GetBindVolumeToClaim", "pod", klog.KObj(assumedPod), "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc), @@ -386,7 +386,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, pod "dirty", dirty, ) if err != nil { - klog.ErrorS(err, "AssumePodVolumes: fail to GetBindVolumeToClaim") + logger.Error(err, "AssumePodVolumes: fail to GetBindVolumeToClaim") b.revertAssumedPVs(newBindings) return false, err } @@ -424,7 +424,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, pod } // RevertAssumedPodVolumes will revert assumed PV and PVC cache. -func (b *volumeBinder) RevertAssumedPodVolumes(podVolumes *PodVolumes) { +func (b *volumeBinder) RevertAssumedPodVolumes(logger klog.Logger, podVolumes *PodVolumes) { b.revertAssumedPVs(podVolumes.StaticBindings) b.revertAssumedPVCs(podVolumes.DynamicProvisions) } @@ -432,9 +432,8 @@ func (b *volumeBinder) RevertAssumedPodVolumes(podVolumes *PodVolumes) { // BindPodVolumes gets the cached bindings and PVCs to provision in pod's volumes information, // makes the API update for those PVs/PVCs, and waits for the PVCs to be completely bound // by the PV controller. -func (b *volumeBinder) BindPodVolumes(assumedPod *v1.Pod, podVolumes *PodVolumes) (err error) { - klog.V(4).InfoS("BindPodVolumes", "pod", klog.KObj(assumedPod), "node", klog.KRef("", assumedPod.Spec.NodeName)) - +func (b *volumeBinder) BindPodVolumes(logger klog.Logger, assumedPod *v1.Pod, podVolumes *PodVolumes) (err error) { + podName := getPodName(assumedPod) defer func() { if err != nil { metrics.VolumeSchedulingStageFailed.WithLabelValues("bind").Inc() @@ -445,13 +444,13 @@ func (b *volumeBinder) BindPodVolumes(assumedPod *v1.Pod, podVolumes *PodVolumes claimsToProvision := podVolumes.DynamicProvisions // Start API operations - err = b.bindAPIUpdate(assumedPod, bindings, claimsToProvision) + err = b.bindAPIUpdate(logger, podName, bindings, claimsToProvision) if err != nil { return err } err = wait.Poll(time.Second, b.bindTimeout, func() (bool, error) { - b, err := b.checkBindings(assumedPod, bindings, claimsToProvision) + b, err := b.checkBindings(logger, assumedPod, bindings, claimsToProvision) return b, err }) if err != nil { @@ -469,8 +468,7 @@ func getPVCName(pvc *v1.PersistentVolumeClaim) string { } // bindAPIUpdate makes the API update for those PVs/PVCs. -func (b *volumeBinder) bindAPIUpdate(pod *v1.Pod, bindings []*BindingInfo, claimsToProvision []*v1.PersistentVolumeClaim) error { - podName := getPodName(pod) +func (b *volumeBinder) bindAPIUpdate(logger klog.Logger, podName string, bindings []*BindingInfo, claimsToProvision []*v1.PersistentVolumeClaim) error { if bindings == nil { return fmt.Errorf("failed to get cached bindings for pod %q", podName) } @@ -500,15 +498,15 @@ func (b *volumeBinder) bindAPIUpdate(pod *v1.Pod, bindings []*BindingInfo, claim // Do the actual prebinding. Let the PV controller take care of the rest // There is no API rollback if the actual binding fails for _, binding = range bindings { - klog.V(5).InfoS("bindAPIUpdate: binding PV to PVC", "pod", klog.KObj(pod), "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc)) + logger.V(5).Info("Binding", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc)) // TODO: does it hurt if we make an api call and nothing needs to be updated? - klog.V(2).InfoS("Claim bound to volume", "PVC", klog.KObj(binding.pvc), "PV", klog.KObj(binding.pv)) + logger.V(2).Info("Claim bound to volume", "PVC", klog.KObj(binding.pvc), "PV", klog.KObj(binding.pv)) newPV, err := b.kubeClient.CoreV1().PersistentVolumes().Update(context.TODO(), binding.pv, metav1.UpdateOptions{}) if err != nil { - klog.V(4).InfoS("Updating PersistentVolume: binding to claim failed", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc), "err", err) + logger.V(4).Info("Updating PersistentVolume: binding to claim failed", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc), "err", err) return err } - klog.V(4).InfoS("Updating PersistentVolume: bound to claim", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc)) + logger.V(4).Info("Updating PersistentVolume: bound to claim", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc)) // Save updated object from apiserver for later checking. binding.pv = newPV lastProcessedBinding++ @@ -517,7 +515,7 @@ func (b *volumeBinder) bindAPIUpdate(pod *v1.Pod, bindings []*BindingInfo, claim // Update claims objects to trigger volume provisioning. Let the PV controller take care of the rest // PV controller is expected to signal back by removing related annotations if actual provisioning fails for i, claim = range claimsToProvision { - klog.V(5).InfoS("Updating claims objects to trigger volume provisioning", "pod", klog.KObj(pod), "PVC", klog.KObj(claim)) + logger.V(5).Info("Updating claims objects to trigger volume provisioning", "PVC", klog.KObj(claim)) newClaim, err := b.kubeClient.CoreV1().PersistentVolumeClaims(claim.Namespace).Update(context.TODO(), claim, metav1.UpdateOptions{}) if err != nil { return err @@ -544,7 +542,7 @@ var ( // PV/PVC cache can be assumed again in main scheduler loop, we must check // latest state in API server which are shared with PV controller and // provisioners -func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claimsToProvision []*v1.PersistentVolumeClaim) (bool, error) { +func (b *volumeBinder) checkBindings(logger klog.Logger, pod *v1.Pod, bindings []*BindingInfo, claimsToProvision []*v1.PersistentVolumeClaim) (bool, error) { podName := getPodName(pod) if bindings == nil { return false, fmt.Errorf("failed to get cached bindings for pod %q", podName) @@ -561,7 +559,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim csiNode, err := b.csiNodeLister.Get(node.Name) if err != nil { // TODO: return the error once CSINode is created by default - klog.V(4).InfoS("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) + logger.V(4).Info("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) } // Check for any conditions that might require scheduling retry @@ -573,7 +571,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim if apierrors.IsNotFound(err) { return false, fmt.Errorf("pod does not exist any more: %w", err) } - klog.ErrorS(err, "Failed to get pod from the lister", "pod", klog.KObj(pod)) + logger.Error(err, "Failed to get pod from the lister") } for _, binding := range bindings { @@ -669,11 +667,11 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim } // All pvs and pvcs that we operated on are bound - klog.V(4).InfoS("All PVCs for pod are bound", "pod", klog.KObj(pod)) + logger.V(4).Info("All PVCs for pod are bound") return true, nil } -func (b *volumeBinder) isVolumeBound(pod *v1.Pod, vol *v1.Volume) (bound bool, pvc *v1.PersistentVolumeClaim, err error) { +func (b *volumeBinder) isVolumeBound(logger klog.Logger, pod *v1.Pod, vol *v1.Volume) (bound bool, pvc *v1.PersistentVolumeClaim, err error) { pvcName := "" isEphemeral := false switch { @@ -688,7 +686,7 @@ func (b *volumeBinder) isVolumeBound(pod *v1.Pod, vol *v1.Volume) (bound bool, p return true, nil, nil } - bound, pvc, err = b.isPVCBound(pod.Namespace, pvcName) + bound, pvc, err = b.isPVCBound(logger, pod.Namespace, pvcName) // ... the PVC must be owned by the pod. if isEphemeral && err == nil && pvc != nil { if err := ephemeral.VolumeIsForPod(pod, pvc); err != nil { @@ -698,7 +696,7 @@ func (b *volumeBinder) isVolumeBound(pod *v1.Pod, vol *v1.Volume) (bound bool, p return } -func (b *volumeBinder) isPVCBound(namespace, pvcName string) (bool, *v1.PersistentVolumeClaim, error) { +func (b *volumeBinder) isPVCBound(logger klog.Logger, namespace, pvcName string) (bool, *v1.PersistentVolumeClaim, error) { claim := &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, @@ -713,12 +711,12 @@ func (b *volumeBinder) isPVCBound(namespace, pvcName string) (bool, *v1.Persiste fullyBound := b.isPVCFullyBound(pvc) if fullyBound { - klog.V(5).InfoS("PVC is fully bound to PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvc.Spec.VolumeName)) + logger.V(5).Info("PVC is fully bound to PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvc.Spec.VolumeName)) } else { if pvc.Spec.VolumeName != "" { - klog.V(5).InfoS("PVC is not fully bound to PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvc.Spec.VolumeName)) + logger.V(5).Info("PVC is not fully bound to PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvc.Spec.VolumeName)) } else { - klog.V(5).InfoS("PVC is not bound", "PVC", klog.KObj(pvc)) + logger.V(5).Info("PVC is not bound", "PVC", klog.KObj(pvc)) } } return fullyBound, pvc, nil @@ -729,9 +727,9 @@ func (b *volumeBinder) isPVCFullyBound(pvc *v1.PersistentVolumeClaim) bool { } // arePodVolumesBound returns true if all volumes are fully bound -func (b *volumeBinder) arePodVolumesBound(pod *v1.Pod) bool { +func (b *volumeBinder) arePodVolumesBound(logger klog.Logger, pod *v1.Pod) bool { for _, vol := range pod.Spec.Volumes { - if isBound, _, _ := b.isVolumeBound(pod, &vol); !isBound { + if isBound, _, _ := b.isVolumeBound(logger, pod, &vol); !isBound { // Pod has at least one PVC that needs binding return false } @@ -741,13 +739,13 @@ func (b *volumeBinder) arePodVolumesBound(pod *v1.Pod) bool { // GetPodVolumes returns a pod's PVCs separated into bound, unbound with delayed binding (including provisioning) // and unbound with immediate binding (including prebound) -func (b *volumeBinder) GetPodVolumes(pod *v1.Pod) (boundClaims []*v1.PersistentVolumeClaim, unboundClaimsDelayBinding []*v1.PersistentVolumeClaim, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) { +func (b *volumeBinder) GetPodVolumes(logger klog.Logger, pod *v1.Pod) (boundClaims []*v1.PersistentVolumeClaim, unboundClaimsDelayBinding []*v1.PersistentVolumeClaim, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) { boundClaims = []*v1.PersistentVolumeClaim{} unboundClaimsImmediate = []*v1.PersistentVolumeClaim{} unboundClaimsDelayBinding = []*v1.PersistentVolumeClaim{} for _, vol := range pod.Spec.Volumes { - volumeBound, pvc, err := b.isVolumeBound(pod, &vol) + volumeBound, pvc, err := b.isVolumeBound(logger, pod, &vol) if err != nil { return nil, nil, nil, err } @@ -775,11 +773,11 @@ func (b *volumeBinder) GetPodVolumes(pod *v1.Pod) (boundClaims []*v1.PersistentV return boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate, nil } -func (b *volumeBinder) checkBoundClaims(claims []*v1.PersistentVolumeClaim, node *v1.Node, pod *v1.Pod) (bool, bool, error) { +func (b *volumeBinder) checkBoundClaims(logger klog.Logger, claims []*v1.PersistentVolumeClaim, node *v1.Node) (bool, bool, error) { csiNode, err := b.csiNodeLister.Get(node.Name) if err != nil { // TODO: return the error once CSINode is created by default - klog.V(4).InfoS("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) + logger.V(4).Info("Could not get a CSINode object for the node", "err", err) } for _, pvc := range claims { @@ -799,19 +797,19 @@ func (b *volumeBinder) checkBoundClaims(claims []*v1.PersistentVolumeClaim, node err = volume.CheckNodeAffinity(pv, node.Labels) if err != nil { - klog.V(4).InfoS("PersistentVolume and node mismatch for pod", "PV", klog.KRef("", pvName), "node", klog.KObj(node), "pod", klog.KObj(pod), "err", err) + logger.V(4).Info("PersistentVolume and node mismatch for pod", "PV", klog.KRef("", pvName), "err", err) return false, true, nil } - klog.V(5).InfoS("PersistentVolume and node matches for pod", "PV", klog.KRef("", pvName), "node", klog.KObj(node), "pod", klog.KObj(pod)) + logger.V(5).Info("PersistentVolume and node matches for pod", "PV", klog.KRef("", pvName)) } - klog.V(4).InfoS("All bound volumes for pod match with node", "pod", klog.KObj(pod), "node", klog.KObj(node)) + logger.V(4).Info("All bound volumes for pod match with node") return true, true, nil } // findMatchingVolumes tries to find matching volumes for given claims, // and return unbound claims for further provision. -func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (foundMatches bool, bindings []*BindingInfo, unboundClaims []*v1.PersistentVolumeClaim, err error) { +func (b *volumeBinder) findMatchingVolumes(logger klog.Logger, pod *v1.Pod, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (foundMatches bool, bindings []*BindingInfo, unboundClaims []*v1.PersistentVolumeClaim, err error) { // Sort all the claims by increasing size request to get the smallest fits sort.Sort(byPVCSize(claimsToBind)) @@ -830,7 +828,7 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.Persi return false, nil, nil, err } if pv == nil { - klog.V(4).InfoS("No matching volumes for pod", "pod", klog.KObj(pod), "PVC", klog.KObj(pvc), "node", klog.KObj(node)) + logger.V(4).Info("No matching volumes for pod", "PVC", klog.KObj(pvc), "node", klog.KObj(node)) unboundClaims = append(unboundClaims, pvc) foundMatches = false continue @@ -839,11 +837,11 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.Persi // matching PV needs to be excluded so we don't select it again chosenPVs[pv.Name] = pv bindings = append(bindings, &BindingInfo{pv: pv, pvc: pvc}) - klog.V(5).InfoS("Found matching PV for PVC for pod", "PV", klog.KObj(pv), "PVC", klog.KObj(pvc), "node", klog.KObj(node), "pod", klog.KObj(pod)) + logger.V(5).Info("Found matching PV for PVC for pod", "PV", klog.KObj(pv), "PVC", klog.KObj(pvc), "node", klog.KObj(node)) } if foundMatches { - klog.V(4).InfoS("Found matching volumes for pod", "pod", klog.KObj(pod), "node", klog.KObj(node)) + logger.V(4).Info("Found matching volumes for pod") } return @@ -852,7 +850,7 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.Persi // checkVolumeProvisions checks given unbound claims (the claims have gone through func // findMatchingVolumes, and do not have matching volumes for binding), and return true // if all of the claims are eligible for dynamic provision. -func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v1.PersistentVolumeClaim, node *v1.Node) (provisionSatisfied, sufficientStorage bool, dynamicProvisions []*v1.PersistentVolumeClaim, err error) { +func (b *volumeBinder) checkVolumeProvisions(logger klog.Logger, pod *v1.Pod, claimsToProvision []*v1.PersistentVolumeClaim, node *v1.Node) (provisionSatisfied, sufficientStorage bool, dynamicProvisions []*v1.PersistentVolumeClaim, err error) { dynamicProvisions = []*v1.PersistentVolumeClaim{} // We return early with provisionedClaims == nil if a check @@ -870,18 +868,18 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v } provisioner := class.Provisioner if provisioner == "" || provisioner == volume.NotSupportedProvisioner { - klog.V(4).InfoS("Storage class of claim does not support dynamic provisioning", "storageClassName", className, "PVC", klog.KObj(claim)) + logger.V(4).Info("Storage class of claim does not support dynamic provisioning", "storageClassName", className, "PVC", klog.KObj(claim)) return false, true, nil, nil } // Check if the node can satisfy the topology requirement in the class if !v1helper.MatchTopologySelectorTerms(class.AllowedTopologies, labels.Set(node.Labels)) { - klog.V(4).InfoS("Node cannot satisfy provisioning topology requirements of claim", "node", klog.KObj(node), "PVC", klog.KObj(claim)) + logger.V(4).Info("Node cannot satisfy provisioning topology requirements of claim", "PVC", klog.KObj(claim)) return false, true, nil, nil } // Check storage capacity. - sufficient, err := b.hasEnoughCapacity(provisioner, claim, class, node) + sufficient, err := b.hasEnoughCapacity(logger, provisioner, claim, class, node) if err != nil { return false, false, nil, err } @@ -893,7 +891,7 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v dynamicProvisions = append(dynamicProvisions, claim) } - klog.V(4).InfoS("Provisioning for claims of pod that has no matching volumes...", "claimCount", len(claimsToProvision), "pod", klog.KObj(pod), "node", klog.KObj(node)) + logger.V(4).Info("Provisioning for claims of pod that has no matching volumes...", "claimCount", len(claimsToProvision)) return true, true, dynamicProvisions, nil } @@ -912,7 +910,7 @@ func (b *volumeBinder) revertAssumedPVCs(claims []*v1.PersistentVolumeClaim) { // hasEnoughCapacity checks whether the provisioner has enough capacity left for a new volume of the given size // that is available from the node. -func (b *volumeBinder) hasEnoughCapacity(provisioner string, claim *v1.PersistentVolumeClaim, storageClass *storagev1.StorageClass, node *v1.Node) (bool, error) { +func (b *volumeBinder) hasEnoughCapacity(logger klog.Logger, provisioner string, claim *v1.PersistentVolumeClaim, storageClass *storagev1.StorageClass, node *v1.Node) (bool, error) { quantity, ok := claim.Spec.Resources.Requests[v1.ResourceStorage] if !ok { // No capacity to check for. @@ -945,7 +943,7 @@ func (b *volumeBinder) hasEnoughCapacity(provisioner string, claim *v1.Persisten for _, capacity := range capacities { if capacity.StorageClassName == storageClass.Name && capacitySufficient(capacity, sizeInBytes) && - b.nodeHasAccess(node, capacity) { + b.nodeHasAccess(logger, node, capacity) { // Enough capacity found. return true, nil } @@ -953,8 +951,8 @@ func (b *volumeBinder) hasEnoughCapacity(provisioner string, claim *v1.Persisten // TODO (?): this doesn't give any information about which pools where considered and why // they had to be rejected. Log that above? But that might be a lot of log output... - klog.V(4).InfoS("Node has no accessible CSIStorageCapacity with enough capacity for PVC", - "node", klog.KObj(node), "PVC", klog.KObj(claim), "size", sizeInBytes, "storageClass", klog.KObj(storageClass)) + logger.V(4).Info("Node has no accessible CSIStorageCapacity with enough capacity for PVC", + "PVC", klog.KObj(claim), "size", sizeInBytes, "storageClass", klog.KObj(storageClass)) return false, nil } @@ -967,7 +965,7 @@ func capacitySufficient(capacity *storagev1.CSIStorageCapacity, sizeInBytes int6 return limit != nil && limit.Value() >= sizeInBytes } -func (b *volumeBinder) nodeHasAccess(node *v1.Node, capacity *storagev1.CSIStorageCapacity) bool { +func (b *volumeBinder) nodeHasAccess(logger klog.Logger, node *v1.Node, capacity *storagev1.CSIStorageCapacity) bool { if capacity.NodeTopology == nil { // Unavailable return false @@ -975,7 +973,7 @@ func (b *volumeBinder) nodeHasAccess(node *v1.Node, capacity *storagev1.CSIStora // Only matching by label is supported. selector, err := metav1.LabelSelectorAsSelector(capacity.NodeTopology) if err != nil { - klog.ErrorS(err, "Unexpected error converting to a label selector", "nodeTopology", capacity.NodeTopology) + logger.Error(err, "Unexpected error converting to a label selector", "nodeTopology", capacity.NodeTopology) return false } return selector.Matches(labels.Set(node.Labels)) diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go index 094144fa6b33..56520bcd9512 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go @@ -43,6 +43,8 @@ import ( featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" + _ "k8s.io/klog/v2/ktesting/init" "k8s.io/kubernetes/pkg/controller" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" "k8s.io/kubernetes/pkg/features" @@ -122,10 +124,6 @@ var ( zone1Labels = map[string]string{v1.LabelFailureDomainBetaZone: "us-east-1", v1.LabelFailureDomainBetaRegion: "us-east-1a"} ) -func init() { - klog.InitFlags(nil) -} - type testEnv struct { client clientset.Interface reactor *pvtesting.VolumeReactor @@ -142,7 +140,8 @@ type testEnv struct { internalCSIStorageCapacityInformer storageinformers.CSIStorageCapacityInformer } -func newTestBinder(t *testing.T, stopCh <-chan struct{}) *testEnv { +func newTestBinder(t *testing.T, ctx context.Context) *testEnv { + logger := klog.FromContext(ctx) client := &fake.Clientset{} reactor := pvtesting.NewVolumeReactor(client, nil, nil, nil) // TODO refactor all tests to use real watch mechanism, see #72327 @@ -169,6 +168,7 @@ func newTestBinder(t *testing.T, stopCh <-chan struct{}) *testEnv { CSIStorageCapacityInformer: csiStorageCapacityInformer, } binder := NewVolumeBinder( + logger, client, podInformer, nodeInformer, @@ -180,10 +180,10 @@ func newTestBinder(t *testing.T, stopCh <-chan struct{}) *testEnv { 10*time.Second) // Wait for informers cache sync - informerFactory.Start(stopCh) - for v, synced := range informerFactory.WaitForCacheSync(stopCh) { + informerFactory.Start(ctx.Done()) + for v, synced := range informerFactory.WaitForCacheSync(ctx.Done()) { if !synced { - klog.ErrorS(nil, "Error syncing informer", "informer", v) + logger.Error(nil, "Error syncing informer", "informer", v) os.Exit(1) } } @@ -820,15 +820,15 @@ func checkReasons(t *testing.T, actual, expected ConflictReasons) { } // findPodVolumes gets and finds volumes for given pod and node -func findPodVolumes(binder SchedulerVolumeBinder, pod *v1.Pod, node *v1.Node) (*PodVolumes, ConflictReasons, error) { - boundClaims, claimsToBind, unboundClaimsImmediate, err := binder.GetPodVolumes(pod) +func findPodVolumes(logger klog.Logger, binder SchedulerVolumeBinder, pod *v1.Pod, node *v1.Node) (*PodVolumes, ConflictReasons, error) { + boundClaims, claimsToBind, unboundClaimsImmediate, err := binder.GetPodVolumes(logger, pod) if err != nil { return nil, nil, err } if len(unboundClaimsImmediate) > 0 { return nil, nil, fmt.Errorf("pod has unbound immediate PersistentVolumeClaims") } - return binder.FindPodVolumes(pod, boundClaims, claimsToBind, node) + return binder.FindPodVolumes(logger, pod, boundClaims, claimsToBind, node) } func TestFindPodVolumesWithoutProvisioning(t *testing.T) { @@ -980,11 +980,12 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) { } run := func(t *testing.T, scenario scenarioType, csiDriver *storagev1.CSIDriver) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initVolumes(scenario.pvs, scenario.pvs) if csiDriver != nil { testEnv.addCSIDriver(csiDriver) @@ -1005,7 +1006,7 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) { } // Execute - podVolumes, reasons, err := findPodVolumes(testEnv.binder, scenario.pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, scenario.pod, testNode) // Validate if !scenario.shouldFail && err != nil { @@ -1107,11 +1108,12 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) { } run := func(t *testing.T, scenario scenarioType, csiDriver *storagev1.CSIDriver) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initVolumes(scenario.pvs, scenario.pvs) if csiDriver != nil { testEnv.addCSIDriver(csiDriver) @@ -1132,7 +1134,7 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) { } // Execute - podVolumes, reasons, err := findPodVolumes(testEnv.binder, scenario.pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, scenario.pod, testNode) // Validate if !scenario.shouldFail && err != nil { @@ -1214,13 +1216,14 @@ func TestFindPodVolumesWithCSIMigration(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigrationGCE, true)() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initVolumes(scenario.pvs, scenario.pvs) var node *v1.Node @@ -1250,7 +1253,7 @@ func TestFindPodVolumesWithCSIMigration(t *testing.T) { } // Execute - _, reasons, err := findPodVolumes(testEnv.binder, scenario.pod, node) + _, reasons, err := findPodVolumes(logger, testEnv.binder, scenario.pod, node) // Validate if !scenario.shouldFail && err != nil { @@ -1333,11 +1336,12 @@ func TestAssumePodVolumes(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initClaims(scenario.podPVCs, scenario.podPVCs) pod := makePod("test-pod"). withNamespace("testns"). @@ -1350,7 +1354,7 @@ func TestAssumePodVolumes(t *testing.T) { testEnv.initVolumes(scenario.pvs, scenario.pvs) // Execute - allBound, err := testEnv.binder.AssumePodVolumes(pod, "node1", podVolumes) + allBound, err := testEnv.binder.AssumePodVolumes(logger, pod, "node1", podVolumes) // Validate if !scenario.shouldFail && err != nil { @@ -1382,7 +1386,8 @@ func TestAssumePodVolumes(t *testing.T) { } func TestRevertAssumedPodVolumes(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() podPVCs := []*v1.PersistentVolumeClaim{unboundPVC, provisionedPVC} @@ -1393,7 +1398,7 @@ func TestRevertAssumedPodVolumes(t *testing.T) { expectedProvisionings := []*v1.PersistentVolumeClaim{selectedNodePVC} // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initClaims(podPVCs, podPVCs) pod := makePod("test-pod"). withNamespace("testns"). @@ -1405,13 +1410,13 @@ func TestRevertAssumedPodVolumes(t *testing.T) { } testEnv.initVolumes(pvs, pvs) - allbound, err := testEnv.binder.AssumePodVolumes(pod, "node1", podVolumes) + allbound, err := testEnv.binder.AssumePodVolumes(logger, pod, "node1", podVolumes) if allbound || err != nil { t.Errorf("No volumes are assumed") } testEnv.validateAssume(t, pod, expectedBindings, expectedProvisionings) - testEnv.binder.RevertAssumedPodVolumes(podVolumes) + testEnv.binder.RevertAssumedPodVolumes(logger, podVolumes) testEnv.validateCacheRestored(t, pod, bindings, provisionedPVCs) } @@ -1510,11 +1515,12 @@ func TestBindAPIUpdate(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) pod := makePod("test-pod"). withNamespace("testns"). withNodeName("node1").Pod @@ -1529,7 +1535,7 @@ func TestBindAPIUpdate(t *testing.T) { testEnv.assumeVolumes(t, "node1", pod, scenario.bindings, scenario.provisionedPVCs) // Execute - err := testEnv.internalBinder.bindAPIUpdate(pod, scenario.bindings, scenario.provisionedPVCs) + err := testEnv.internalBinder.bindAPIUpdate(logger, pod.Name, scenario.bindings, scenario.provisionedPVCs) // Validate if !scenario.shouldFail && err != nil { @@ -1708,13 +1714,14 @@ func TestCheckBindings(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup pod := makePod("test-pod"). withNamespace("testns"). withNodeName("node1").Pod - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.internalPodInformer.Informer().GetIndexer().Add(pod) testEnv.initNodes([]*v1.Node{node1}) testEnv.initVolumes(scenario.initPVs, nil) @@ -1738,7 +1745,7 @@ func TestCheckBindings(t *testing.T) { } // Execute - allBound, err := testEnv.internalBinder.checkBindings(pod, scenario.bindings, scenario.provisionedPVCs) + allBound, err := testEnv.internalBinder.checkBindings(logger, pod, scenario.bindings, scenario.provisionedPVCs) // Validate if !scenario.shouldFail && err != nil { @@ -1839,7 +1846,8 @@ func TestCheckBindingsWithCSIMigration(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigrationGCE, scenario.migrationEnabled)() @@ -1848,7 +1856,7 @@ func TestCheckBindingsWithCSIMigration(t *testing.T) { pod := makePod("test-pod"). withNamespace("testns"). withNodeName("node1").Pod - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.internalPodInformer.Informer().GetIndexer().Add(pod) testEnv.initNodes(scenario.initNodes) testEnv.initCSINodes(scenario.initCSINodes) @@ -1865,7 +1873,7 @@ func TestCheckBindingsWithCSIMigration(t *testing.T) { } // Execute - allBound, err := testEnv.internalBinder.checkBindings(pod, scenario.bindings, scenario.provisionedPVCs) + allBound, err := testEnv.internalBinder.checkBindings(logger, pod, scenario.bindings, scenario.provisionedPVCs) // Validate if !scenario.shouldFail && err != nil { @@ -2031,13 +2039,14 @@ func TestBindPodVolumes(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup pod := makePod("test-pod"). withNamespace("testns"). withNodeName("node1").Pod - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.internalPodInformer.Informer().GetIndexer().Add(pod) if scenario.nodes == nil { scenario.nodes = []*v1.Node{node1} @@ -2075,7 +2084,7 @@ func TestBindPodVolumes(t *testing.T) { go func(scenario scenarioType) { time.Sleep(5 * time.Second) // Sleep a while to run after bindAPIUpdate in BindPodVolumes - klog.V(5).InfoS("Running delay function") + logger.V(5).Info("Running delay function") scenario.delayFunc(t, ctx, testEnv, pod, scenario.initPVs, scenario.initPVCs) }(scenario) } @@ -2085,7 +2094,7 @@ func TestBindPodVolumes(t *testing.T) { StaticBindings: bindings, DynamicProvisions: claimsToProvision, } - err := testEnv.binder.BindPodVolumes(pod, podVolumes) + err := testEnv.binder.BindPodVolumes(logger, pod, podVolumes) // Validate if !scenario.shouldFail && err != nil { @@ -2111,9 +2120,10 @@ func TestFindAssumeVolumes(t *testing.T) { pvs := []*v1.PersistentVolume{pvNode2, pvNode1a, pvNode1c} // Setup - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initVolumes(pvs, pvs) testEnv.initClaims(podPVCs, podPVCs) pod := makePod("test-pod"). @@ -2132,7 +2142,7 @@ func TestFindAssumeVolumes(t *testing.T) { // Execute // 1. Find matching PVs - podVolumes, reasons, err := findPodVolumes(testEnv.binder, pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, pod, testNode) if err != nil { t.Errorf("Test failed: FindPodVolumes returned error: %v", err) } @@ -2142,7 +2152,7 @@ func TestFindAssumeVolumes(t *testing.T) { expectedBindings := podVolumes.StaticBindings // 2. Assume matches - allBound, err := testEnv.binder.AssumePodVolumes(pod, testNode.Name, podVolumes) + allBound, err := testEnv.binder.AssumePodVolumes(logger, pod, testNode.Name, podVolumes) if err != nil { t.Errorf("Test failed: AssumePodVolumes returned error: %v", err) } @@ -2158,7 +2168,7 @@ func TestFindAssumeVolumes(t *testing.T) { // This should always return the original chosen pv // Run this many times in case sorting returns different orders for the two PVs. for i := 0; i < 50; i++ { - podVolumes, reasons, err := findPodVolumes(testEnv.binder, pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, pod, testNode) if err != nil { t.Errorf("Test failed: FindPodVolumes returned error: %v", err) } @@ -2267,11 +2277,13 @@ func TestCapacity(t *testing.T) { } run := func(t *testing.T, scenario scenarioType, optIn bool) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup: the driver has the feature enabled, but the scheduler might not. - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) + testEnv.addCSIDriver(makeCSIDriver(provisioner, optIn)) testEnv.addCSIStorageCapacities(scenario.capacities) @@ -2285,7 +2297,7 @@ func TestCapacity(t *testing.T) { withPVCSVolume(scenario.pvcs).Pod // Execute - podVolumes, reasons, err := findPodVolumes(testEnv.binder, pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, pod, testNode) // Validate shouldFail := scenario.shouldFail diff --git a/pkg/scheduler/framework/plugins/volumebinding/fake_binder.go b/pkg/scheduler/framework/plugins/volumebinding/fake_binder.go index 4a07eff9a2fd..cce54242b247 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/fake_binder.go +++ b/pkg/scheduler/framework/plugins/volumebinding/fake_binder.go @@ -16,7 +16,10 @@ limitations under the License. package volumebinding -import v1 "k8s.io/api/core/v1" +import ( + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" +) // FakeVolumeBinderConfig holds configurations for fake volume binder. type FakeVolumeBinderConfig struct { @@ -43,26 +46,26 @@ type FakeVolumeBinder struct { } // GetPodVolumes implements SchedulerVolumeBinder.GetPodVolumes. -func (b *FakeVolumeBinder) GetPodVolumes(pod *v1.Pod) (boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) { +func (b *FakeVolumeBinder) GetPodVolumes(_ klog.Logger, pod *v1.Pod) (boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) { return nil, nil, nil, nil } // FindPodVolumes implements SchedulerVolumeBinder.FindPodVolumes. -func (b *FakeVolumeBinder) FindPodVolumes(pod *v1.Pod, _, _ []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) { +func (b *FakeVolumeBinder) FindPodVolumes(_ klog.Logger, pod *v1.Pod, _, _ []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) { return nil, b.config.FindReasons, b.config.FindErr } // AssumePodVolumes implements SchedulerVolumeBinder.AssumePodVolumes. -func (b *FakeVolumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (bool, error) { +func (b *FakeVolumeBinder) AssumePodVolumes(_ klog.Logger, assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (bool, error) { b.AssumeCalled = true return b.config.AllBound, b.config.AssumeErr } // RevertAssumedPodVolumes implements SchedulerVolumeBinder.RevertAssumedPodVolumes -func (b *FakeVolumeBinder) RevertAssumedPodVolumes(_ *PodVolumes) {} +func (b *FakeVolumeBinder) RevertAssumedPodVolumes(_ klog.Logger, _ *PodVolumes) {} // BindPodVolumes implements SchedulerVolumeBinder.BindPodVolumes. -func (b *FakeVolumeBinder) BindPodVolumes(assumedPod *v1.Pod, podVolumes *PodVolumes) error { +func (b *FakeVolumeBinder) BindPodVolumes(_ klog.Logger, assumedPod *v1.Pod, podVolumes *PodVolumes) error { b.BindCalled = true return b.config.BindErr } diff --git a/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go b/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go index 48ee01dea4eb..df4b87fbd371 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go +++ b/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go @@ -173,7 +173,8 @@ func (pl *VolumeBinding) PreFilter(ctx context.Context, state *framework.CycleSt state.Write(stateKey, &stateData{skip: true}) return nil, nil } - boundClaims, claimsToBind, unboundClaimsImmediate, err := pl.Binder.GetPodVolumes(pod) + logger := klog.FromContext(ctx) + boundClaims, claimsToBind, unboundClaimsImmediate, err := pl.Binder.GetPodVolumes(logger, pod) if err != nil { return nil, framework.AsStatus(err) } @@ -222,6 +223,7 @@ func getStateData(cs *framework.CycleState) (*stateData, error) { // The predicate returns true if all bound PVCs have compatible PVs with the node, and if all unbound // PVCs can be matched with an available and node-compatible PV. func (pl *VolumeBinding) Filter(ctx context.Context, cs *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { + logger := klog.FromContext(ctx) node := nodeInfo.Node() if node == nil { return framework.NewStatus(framework.Error, "node not found") @@ -236,7 +238,7 @@ func (pl *VolumeBinding) Filter(ctx context.Context, cs *framework.CycleState, p return nil } - podVolumes, reasons, err := pl.Binder.FindPodVolumes(pod, state.boundClaims, state.claimsToBind, node) + podVolumes, reasons, err := pl.Binder.FindPodVolumes(logger, pod, state.boundClaims, state.claimsToBind, node) if err != nil { return framework.AsStatus(err) @@ -301,7 +303,8 @@ func (pl *VolumeBinding) Reserve(ctx context.Context, cs *framework.CycleState, // we don't need to hold the lock as only one node will be reserved for the given pod podVolumes, ok := state.podVolumesByNode[nodeName] if ok { - allBound, err := pl.Binder.AssumePodVolumes(pod, nodeName, podVolumes) + logger := klog.FromContext(ctx) + allBound, err := pl.Binder.AssumePodVolumes(logger, pod, nodeName, podVolumes) if err != nil { return framework.AsStatus(err) } @@ -332,13 +335,14 @@ func (pl *VolumeBinding) PreBind(ctx context.Context, cs *framework.CycleState, if !ok { return framework.AsStatus(fmt.Errorf("no pod volumes found for node %q", nodeName)) } - klog.V(5).InfoS("Trying to bind volumes for pod", "pod", klog.KObj(pod)) - err = pl.Binder.BindPodVolumes(pod, podVolumes) + logger := klog.FromContext(ctx) + logger.V(5).Info("Trying to bind volumes for pod") + err = pl.Binder.BindPodVolumes(logger, pod, podVolumes) if err != nil { - klog.V(1).InfoS("Failed to bind volumes for pod", "pod", klog.KObj(pod), "err", err) + logger.V(1).Info("Failed to bind volumes for pod", "err", err) return framework.AsStatus(err) } - klog.V(5).InfoS("Success binding volumes for pod", "pod", klog.KObj(pod)) + logger.V(5).Info("Success binding volumes for pod") return nil } @@ -354,7 +358,8 @@ func (pl *VolumeBinding) Unreserve(ctx context.Context, cs *framework.CycleState if !ok { return } - pl.Binder.RevertAssumedPodVolumes(podVolumes) + logger := klog.FromContext(ctx) + pl.Binder.RevertAssumedPodVolumes(logger, podVolumes) return } @@ -379,7 +384,7 @@ func New(plArgs runtime.Object, fh framework.Handle, fts feature.Features) (fram CSIDriverInformer: fh.SharedInformerFactory().Storage().V1().CSIDrivers(), CSIStorageCapacityInformer: fh.SharedInformerFactory().Storage().V1().CSIStorageCapacities(), } - binder := NewVolumeBinder(fh.ClientSet(), podInformer, nodeInformer, csiNodeInformer, pvcInformer, pvInformer, storageClassInformer, capacityCheck, time.Duration(args.BindTimeoutSeconds)*time.Second) + binder := NewVolumeBinder(fh.Logger(), fh.ClientSet(), podInformer, nodeInformer, csiNodeInformer, pvcInformer, pvInformer, storageClassInformer, capacityCheck, time.Duration(args.BindTimeoutSeconds)*time.Second) // build score function var scorer volumeCapacityScorer diff --git a/pkg/scheduler/framework/plugins/volumezone/volume_zone.go b/pkg/scheduler/framework/plugins/volumezone/volume_zone.go index fb548121d452..16902a87f119 100644 --- a/pkg/scheduler/framework/plugins/volumezone/volume_zone.go +++ b/pkg/scheduler/framework/plugins/volumezone/volume_zone.go @@ -104,6 +104,7 @@ func (pl *VolumeZone) Filter(ctx context.Context, _ *framework.CycleState, pod * return nil } + logger := klog.FromContext(ctx) for i := range pod.Spec.Volumes { volume := pod.Spec.Volumes[i] if volume.PersistentVolumeClaim == nil { @@ -152,12 +153,12 @@ func (pl *VolumeZone) Filter(ctx context.Context, _ *framework.CycleState, pod * nodeV := nodeConstraints[k] volumeVSet, err := volumehelpers.LabelZonesToSet(v) if err != nil { - klog.InfoS("Failed to parse label, ignoring the label", "label", fmt.Sprintf("%s:%s", k, v), "err", err) + logger.Info("Failed to parse label, ignoring the label", "label", fmt.Sprintf("%s:%s", k, v), "err", err) continue } if !volumeVSet.Has(nodeV) { - klog.V(10).InfoS("Won't schedule pod onto node due to volume (mismatch on label key)", "pod", klog.KObj(pod), "node", klog.KObj(node), "PV", klog.KRef("", pvName), "PVLabelKey", k) + logger.V(10).Info("Won't schedule pod onto node due to volume (mismatch on label key)", "pod", klog.KObj(pod), "node", klog.KObj(node), "PV", klog.KRef("", pvName), "PVLabelKey", k) return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonConflict) } } diff --git a/pkg/scheduler/framework/preemption/preemption.go b/pkg/scheduler/framework/preemption/preemption.go index af5e46d1ae2b..d8c483ff6b00 100644 --- a/pkg/scheduler/framework/preemption/preemption.go +++ b/pkg/scheduler/framework/preemption/preemption.go @@ -136,6 +136,8 @@ type Evaluator struct { // - . It's the regular happy path // and the non-empty nominatedNodeName will be applied to the preemptor pod. func (ev *Evaluator) Preempt(ctx context.Context, pod *v1.Pod, m framework.NodeToStatusMap) (*framework.PostFilterResult, *framework.Status) { + logger := klog.FromContext(ctx) + // 0) Fetch the latest version of . // It's safe to directly fetch pod here. Because the informer cache has already been // initialized when creating the Scheduler obj, i.e., factory.go#MakeDefaultErrorFunc(). @@ -143,13 +145,13 @@ func (ev *Evaluator) Preempt(ctx context.Context, pod *v1.Pod, m framework.NodeT podNamespace, podName := pod.Namespace, pod.Name pod, err := ev.PodLister.Pods(pod.Namespace).Get(pod.Name) if err != nil { - klog.ErrorS(err, "Getting the updated preemptor pod object", "pod", klog.KRef(podNamespace, podName)) + logger.Error(err, "Getting the updated preemptor pod object", "pod", klog.KRef(podNamespace, podName)) return nil, framework.AsStatus(err) } // 1) Ensure the preemptor is eligible to preempt other pods. if ok, msg := ev.PodEligibleToPreemptOthers(pod, m[pod.Status.NominatedNodeName]); !ok { - klog.V(5).InfoS("Pod is not eligible for preemption", "pod", klog.KObj(pod), "reason", msg) + logger.V(5).Info("Pod is not eligible for preemption", "pod", klog.KObj(pod), "reason", msg) return nil, framework.NewStatus(framework.Unschedulable, msg) } @@ -174,13 +176,13 @@ func (ev *Evaluator) Preempt(ctx context.Context, pod *v1.Pod, m framework.NodeT } // 3) Interact with registered Extenders to filter out some candidates if needed. - candidates, status := ev.callExtenders(pod, candidates) + candidates, status := ev.callExtenders(logger, pod, candidates) if !status.IsSuccess() { return nil, status } // 4) Find the best candidate. - bestCandidate := ev.SelectCandidate(candidates) + bestCandidate := ev.SelectCandidate(logger, candidates) if bestCandidate == nil || len(bestCandidate.Name()) == 0 { return nil, framework.NewStatus(framework.Unschedulable, "no candidate node for preemption") } @@ -203,12 +205,13 @@ func (ev *Evaluator) findCandidates(ctx context.Context, pod *v1.Pod, m framewor if len(allNodes) == 0 { return nil, nil, errors.New("no nodes available") } + logger := klog.FromContext(ctx) potentialNodes, unschedulableNodeStatus := nodesWherePreemptionMightHelp(allNodes, m) if len(potentialNodes) == 0 { - klog.V(3).InfoS("Preemption will not help schedule pod on any node", "pod", klog.KObj(pod)) + logger.V(3).Info("Preemption will not help schedule pod on any node", "pod", klog.KObj(pod)) // In this case, we should clean-up any existing nominated node name of the pod. if err := util.ClearNominatedNodeName(ctx, ev.Handler.ClientSet(), pod); err != nil { - klog.ErrorS(err, "Cannot clear 'NominatedNodeName' field of pod", "pod", klog.KObj(pod)) + logger.Error(err, "Cannot clear 'NominatedNodeName' field of pod", "pod", klog.KObj(pod)) // We do not return as this error is not critical. } return nil, unschedulableNodeStatus, nil @@ -220,12 +223,12 @@ func (ev *Evaluator) findCandidates(ctx context.Context, pod *v1.Pod, m framewor } offset, numCandidates := ev.GetOffsetAndNumCandidates(int32(len(potentialNodes))) - if klogV := klog.V(5); klogV.Enabled() { + if logger := logger.V(5); logger.Enabled() { var sample []string for i := offset; i < offset+10 && i < int32(len(potentialNodes)); i++ { sample = append(sample, potentialNodes[i].Node().Name) } - klogV.InfoS("Selecting candidates from a pool of nodes", "potentialNodesCount", len(potentialNodes), "offset", offset, "sampleLength", len(sample), "sample", sample, "candidates", numCandidates) + logger.Info("Selecting candidates from a pool of nodes", "potentialNodesCount", len(potentialNodes), "offset", offset, "sampleLength", len(sample), "sample", sample, "candidates", numCandidates) } candidates, nodeStatuses, err := ev.DryRunPreemption(ctx, pod, potentialNodes, pdbs, offset, numCandidates) for node, nodeStatus := range unschedulableNodeStatus { @@ -238,7 +241,7 @@ func (ev *Evaluator) findCandidates(ctx context.Context, pod *v1.Pod, m framewor // We will only check with extenders that support preemption. // Extenders which do not support preemption may later prevent preemptor from being scheduled on the nominated // node. In that case, scheduler will find a different host for the preemptor in subsequent scheduling cycles. -func (ev *Evaluator) callExtenders(pod *v1.Pod, candidates []Candidate) ([]Candidate, *framework.Status) { +func (ev *Evaluator) callExtenders(logger klog.Logger, pod *v1.Pod, candidates []Candidate) ([]Candidate, *framework.Status) { extenders := ev.Handler.Extenders() nodeLister := ev.Handler.SnapshotSharedLister().NodeInfos() if len(extenders) == 0 { @@ -255,10 +258,10 @@ func (ev *Evaluator) callExtenders(pod *v1.Pod, candidates []Candidate) ([]Candi if !extender.SupportsPreemption() || !extender.IsInterested(pod) { continue } - nodeNameToVictims, err := extender.ProcessPreemption(pod, victimsMap, nodeLister) + nodeNameToVictims, err := extender.ProcessPreemption(logger, pod, victimsMap, nodeLister) if err != nil { if extender.IsIgnorable() { - klog.InfoS("Skipping extender as it returned error and has ignorable flag set", + logger.Info("Skipping extender as it returned error and has ignorable flag set", "extender", extender, "err", err) continue } @@ -269,7 +272,7 @@ func (ev *Evaluator) callExtenders(pod *v1.Pod, candidates []Candidate) ([]Candi if victims == nil || len(victims.Pods) == 0 { if extender.IsIgnorable() { delete(nodeNameToVictims, nodeName) - klog.InfoS("Ignoring node without victims", "node", klog.KRef("", nodeName)) + logger.Info("Ignoring node without victims", "node", klog.KRef("", nodeName)) continue } return nil, framework.AsStatus(fmt.Errorf("expected at least one victim pod on node %q", nodeName)) @@ -298,7 +301,7 @@ func (ev *Evaluator) callExtenders(pod *v1.Pod, candidates []Candidate) ([]Candi // SelectCandidate chooses the best-fit candidate from given and return it. // NOTE: This method is exported for easier testing in default preemption. -func (ev *Evaluator) SelectCandidate(candidates []Candidate) Candidate { +func (ev *Evaluator) SelectCandidate(logger klog.Logger, candidates []Candidate) Candidate { if len(candidates) == 0 { return nil } @@ -307,7 +310,7 @@ func (ev *Evaluator) SelectCandidate(candidates []Candidate) Candidate { } victimsMap := ev.CandidatesToVictimsMap(candidates) - candidateNode := pickOneNodeForPreemption(victimsMap) + candidateNode := pickOneNodeForPreemption(logger, victimsMap) // Same as candidatesToVictimsMap, this logic is not applicable for out-of-tree // preemption plugins that exercise different candidates on the same nominated node. @@ -319,7 +322,7 @@ func (ev *Evaluator) SelectCandidate(candidates []Candidate) Candidate { } // We shouldn't reach here. - klog.ErrorS(errors.New("no candidate selected"), "Should not reach here", "candidates", candidates) + logger.Error(errors.New("no candidate selected"), "Should not reach here", "candidates", candidates) // To not break the whole flow, return the first candidate. return candidates[0] } @@ -329,6 +332,7 @@ func (ev *Evaluator) SelectCandidate(candidates []Candidate) Candidate { // - Reject the victim pods if they are in waitingPod map // - Clear the low-priority pods' nominatedNodeName status if needed func (ev *Evaluator) prepareCandidate(ctx context.Context, c Candidate, pod *v1.Pod, pluginName string) *framework.Status { + logger := klog.FromContext(ctx) fh := ev.Handler cs := ev.Handler.ClientSet() for _, victim := range c.Victims().Pods { @@ -337,7 +341,7 @@ func (ev *Evaluator) prepareCandidate(ctx context.Context, c Candidate, pod *v1. if waitingPod := fh.GetWaitingPod(victim.UID); waitingPod != nil { waitingPod.Reject(pluginName, "preempted") } else if err := util.DeletePod(ctx, cs, victim); err != nil { - klog.ErrorS(err, "Preempting pod", "pod", klog.KObj(victim), "preemptor", klog.KObj(pod)) + logger.Error(err, "Preempting pod", "pod", klog.KObj(victim), "preemptor", klog.KObj(pod)) return framework.AsStatus(err) } fh.EventRecorder().Eventf(victim, pod, v1.EventTypeNormal, "Preempted", "Preempting", "Preempted by %v/%v on node %v", @@ -349,9 +353,9 @@ func (ev *Evaluator) prepareCandidate(ctx context.Context, c Candidate, pod *v1. // this node. So, we should remove their nomination. Removing their // nomination updates these pods and moves them to the active queue. It // lets scheduler find another place for them. - nominatedPods := getLowerPriorityNominatedPods(fh, pod, c.Name()) + nominatedPods := getLowerPriorityNominatedPods(logger, fh, pod, c.Name()) if err := util.ClearNominatedNodeName(ctx, cs, nominatedPods...); err != nil { - klog.ErrorS(err, "Cannot clear 'NominatedNodeName' field") + logger.Error(err, "Cannot clear 'NominatedNodeName' field") // We do not return as this error is not critical. } @@ -394,7 +398,7 @@ func getPodDisruptionBudgets(pdbLister policylisters.PodDisruptionBudgetLister) // 6. If there are still ties, the first such node is picked (sort of randomly). // The 'minNodes1' and 'minNodes2' are being reused here to save the memory // allocation and garbage collection time. -func pickOneNodeForPreemption(nodesToVictims map[string]*extenderv1.Victims) string { +func pickOneNodeForPreemption(logger klog.Logger, nodesToVictims map[string]*extenderv1.Victims) string { if len(nodesToVictims) == 0 { return "" } @@ -493,7 +497,7 @@ func pickOneNodeForPreemption(nodesToVictims map[string]*extenderv1.Victims) str if latestStartTime == nil { // If the earliest start time of all pods on the 1st node is nil, just return it, // which is not expected to happen. - klog.ErrorS(errors.New("earliestStartTime is nil for node"), "Should not reach here", "node", klog.KRef("", minNodes2[0])) + logger.Error(errors.New("earliestStartTime is nil for node"), "Should not reach here", "node", klog.KRef("", minNodes2[0])) return minNodes2[0] } nodeToReturn := minNodes2[0] @@ -502,7 +506,7 @@ func pickOneNodeForPreemption(nodesToVictims map[string]*extenderv1.Victims) str // Get earliest start time of all pods on the current node. earliestStartTimeOnNode := util.GetEarliestPodStartTime(nodesToVictims[node]) if earliestStartTimeOnNode == nil { - klog.ErrorS(errors.New("earliestStartTime is nil for node"), "Should not reach here", "node", klog.KRef("", node)) + logger.Error(errors.New("earliestStartTime is nil for node"), "Should not reach here", "node", klog.KRef("", node)) continue } if earliestStartTimeOnNode.After(latestStartTime.Time) { @@ -521,8 +525,8 @@ func pickOneNodeForPreemption(nodesToVictims map[string]*extenderv1.Victims) str // manipulation of NodeInfo and PreFilter state per nominated pod. It may not be // worth the complexity, especially because we generally expect to have a very // small number of nominated pods per node. -func getLowerPriorityNominatedPods(pn framework.PodNominator, pod *v1.Pod, nodeName string) []*v1.Pod { - podInfos := pn.NominatedPodsForNode(nodeName) +func getLowerPriorityNominatedPods(logger klog.Logger, pn framework.PodNominator, pod *v1.Pod, nodeName string) []*v1.Pod { + podInfos := pn.NominatedPodsForNode(logger, nodeName) if len(podInfos) == 0 { return nil diff --git a/pkg/scheduler/framework/preemption/preemption_test.go b/pkg/scheduler/framework/preemption/preemption_test.go index 14ecc9d87857..9c9561c8829e 100644 --- a/pkg/scheduler/framework/preemption/preemption_test.go +++ b/pkg/scheduler/framework/preemption/preemption_test.go @@ -29,6 +29,8 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" clientsetfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/klog/v2/ktesting" + _ "k8s.io/klog/v2/ktesting/init" extenderv1 "k8s.io/kube-scheduler/extender/v1" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/parallelize" @@ -266,6 +268,7 @@ func TestDryRunPreemption(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) registeredPlugins := append([]st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New)}, st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), @@ -285,6 +288,7 @@ func TestDryRunPreemption(t *testing.T) { frameworkruntime.WithInformerFactory(informerFactory), frameworkruntime.WithParallelism(parallelism), frameworkruntime.WithSnapshotSharedLister(internalcache.NewSnapshot(tt.testPods, tt.nodes)), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index e12ed28c875b..7a73bd162ebb 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -91,6 +91,7 @@ type frameworkImpl struct { kubeConfig *restclient.Config eventRecorder events.EventRecorder informerFactory informers.SharedInformerFactory + logger klog.Logger metricsRecorder *metricsRecorder profileName string @@ -146,6 +147,7 @@ type frameworkOptions struct { captureProfile CaptureProfile clusterEventMap map[framework.ClusterEvent]sets.String parallelizer parallelize.Parallelizer + logger *klog.Logger } // Option for the frameworkImpl. @@ -227,6 +229,13 @@ func WithCaptureProfile(c CaptureProfile) Option { } } +// WithLogger overrides the default logger from k8s.io/klog. +func WithLogger(logger klog.Logger) Option { + return func(o *frameworkOptions) { + o.logger = &logger + } +} + func defaultFrameworkOptions(stopCh <-chan struct{}) frameworkOptions { return frameworkOptions{ metricsRecorder: newMetricsRecorder(1000, time.Second, stopCh), @@ -251,6 +260,11 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, stopCh <-cha opt(&options) } + logger := klog.TODO() + if options.logger != nil { + logger = *options.logger + } + f := &frameworkImpl{ registry: r, snapshotSharedLister: options.snapshotSharedLister, @@ -264,6 +278,7 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, stopCh <-cha extenders: options.extenders, PodNominator: options.podNominator, parallelizer: options.parallelizer, + logger: logger, } if profile == nil { @@ -313,7 +328,7 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, stopCh <-cha pluginsMap[name] = p // Update ClusterEventMap in place. - fillEventToPluginMap(p, options.clusterEventMap) + fillEventToPluginMap(logger, p, options.clusterEventMap) } // initialize plugins per individual extension points @@ -325,7 +340,7 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, stopCh <-cha // initialize multiPoint plugins to their expanded extension points if len(profile.Plugins.MultiPoint.Enabled) > 0 { - if err := f.expandMultiPointPlugins(profile, pluginsMap); err != nil { + if err := f.expandMultiPointPlugins(logger, profile, pluginsMap); err != nil { return nil, err } } @@ -427,7 +442,7 @@ func (os *orderedSet) delete(s string) { } } -func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerProfile, pluginsMap map[string]framework.Plugin) error { +func (f *frameworkImpl) expandMultiPointPlugins(logger klog.Logger, profile *config.KubeSchedulerProfile, pluginsMap map[string]framework.Plugin) error { // initialize MultiPoint plugins for _, e := range f.getExtensionPoints(profile.Plugins) { plugins := reflect.ValueOf(e.slicePtr).Elem() @@ -444,7 +459,7 @@ func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerPro disabledSet.Insert(disabledPlugin.Name) } if disabledSet.Has("*") { - klog.V(4).InfoS("all plugins disabled for extension point, skipping MultiPoint expansion", "extension", pluginType) + logger.V(4).Info("all plugins disabled for extension point, skipping MultiPoint expansion", "extension", pluginType) continue } @@ -465,7 +480,7 @@ func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerPro // a plugin that's enabled via MultiPoint can still be disabled for specific extension points if disabledSet.Has(ep.Name) { - klog.V(4).InfoS("plugin disabled for extension point", "plugin", ep.Name, "extension", pluginType) + logger.V(4).Info("plugin disabled for extension point", "plugin", ep.Name, "extension", pluginType) continue } @@ -475,7 +490,7 @@ func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerPro // This maintains expected behavior for overriding default plugins (see https://github.com/kubernetes/kubernetes/pull/99582) if enabledSet.has(ep.Name) { overridePlugins.insert(ep.Name) - klog.InfoS("MultiPoint plugin is explicitly re-configured; overriding", "plugin", ep.Name) + logger.Info("MultiPoint plugin is explicitly re-configured; overriding", "plugin", ep.Name) continue } @@ -514,7 +529,7 @@ func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerPro return nil } -func fillEventToPluginMap(p framework.Plugin, eventToPlugins map[framework.ClusterEvent]sets.String) { +func fillEventToPluginMap(logger klog.Logger, p framework.Plugin, eventToPlugins map[framework.ClusterEvent]sets.String) { ext, ok := p.(framework.EnqueueExtensions) if !ok { // If interface EnqueueExtensions is not implemented, register the default events @@ -528,7 +543,7 @@ func fillEventToPluginMap(p framework.Plugin, eventToPlugins map[framework.Clust // We treat it as: the plugin is not interested in any event, and hence pod failed by that plugin // cannot be moved by any regular cluster event. if len(events) == 0 { - klog.InfoS("Plugin's EventsToRegister() returned nil", "plugin", p.Name()) + logger.Info("Plugin's EventsToRegister() returned nil") return } // The most common case: a plugin implements EnqueueExtensions and returns non-nil result. @@ -643,6 +658,7 @@ func (f *frameworkImpl) RunPreFilterExtensionAddPod( podInfoToAdd *framework.PodInfo, nodeInfo *framework.NodeInfo, ) (status *framework.Status) { + logger := klog.FromContext(ctx) for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue @@ -650,7 +666,7 @@ func (f *frameworkImpl) RunPreFilterExtensionAddPod( status = f.runPreFilterExtensionAddPod(ctx, pl, state, podToSchedule, podInfoToAdd, nodeInfo) if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running AddPod on PreFilter plugin", "plugin", pl.Name(), "pod", klog.KObj(podToSchedule)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running AddPod on PreFilter plugin %q: %w", pl.Name(), err)) } } @@ -678,6 +694,7 @@ func (f *frameworkImpl) RunPreFilterExtensionRemovePod( podInfoToRemove *framework.PodInfo, nodeInfo *framework.NodeInfo, ) (status *framework.Status) { + logger := klog.FromContext(ctx) for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue @@ -685,7 +702,7 @@ func (f *frameworkImpl) RunPreFilterExtensionRemovePod( status = f.runPreFilterExtensionRemovePod(ctx, pl, state, podToSchedule, podInfoToRemove, nodeInfo) if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running RemovePod on PreFilter plugin", "plugin", pl.Name(), "pod", klog.KObj(podToSchedule)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running RemovePod on PreFilter plugin %q: %w", pl.Name(), err)) } } @@ -841,7 +858,8 @@ func addNominatedPods(ctx context.Context, fh framework.Handle, pod *v1.Pod, sta // This may happen only in tests. return false, state, nodeInfo, nil } - nominatedPodInfos := fh.NominatedPodsForNode(nodeInfo.Node().Name) + logger := klog.FromContext(ctx) + nominatedPodInfos := fh.NominatedPodsForNode(logger, nodeInfo.Node().Name) if len(nominatedPodInfos) == 0 { return false, state, nodeInfo, nil } @@ -999,11 +1017,12 @@ func (f *frameworkImpl) RunPreBindPlugins(ctx context.Context, state *framework. defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(preBind, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() + logger := klog.FromContext(ctx) for _, pl := range f.preBindPlugins { status = f.runPreBindPlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running PreBind plugin", "plugin", pl.Name(), "pod", klog.KObj(pod)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running PreBind plugin %q: %w", pl.Name(), err)) } } @@ -1029,6 +1048,7 @@ func (f *frameworkImpl) RunBindPlugins(ctx context.Context, state *framework.Cyc if len(f.bindPlugins) == 0 { return framework.NewStatus(framework.Skip, "") } + logger := klog.FromContext(ctx) for _, bp := range f.bindPlugins { status = f.runBindPlugin(ctx, bp, state, pod, nodeName) if status.IsSkip() { @@ -1036,7 +1056,7 @@ func (f *frameworkImpl) RunBindPlugins(ctx context.Context, state *framework.Cyc } if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running Bind plugin", "plugin", bp.Name(), "pod", klog.KObj(pod)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running Bind plugin %q: %w", bp.Name(), err)) } return status @@ -1085,11 +1105,12 @@ func (f *frameworkImpl) RunReservePluginsReserve(ctx context.Context, state *fra defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(reserve, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() + logger := klog.FromContext(ctx) for _, pl := range f.reservePlugins { status = f.runReservePluginReserve(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running Reserve plugin", "plugin", pl.Name(), "pod", klog.KObj(pod)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running Reserve plugin %q: %w", pl.Name(), err)) } } @@ -1143,11 +1164,12 @@ func (f *frameworkImpl) RunPermitPlugins(ctx context.Context, state *framework.C }() pluginsWaitTime := make(map[string]time.Duration) statusCode := framework.Success + logger := klog.FromContext(ctx) for _, pl := range f.permitPlugins { status, timeout := f.runPermitPlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { if status.IsUnschedulable() { - klog.V(4).InfoS("Pod rejected by permit plugin", "pod", klog.KObj(pod), "plugin", pl.Name(), "status", status.Message()) + logger.V(4).Info("Pod rejected by plugin", "status", status.Message()) status.SetFailedPlugin(pl.Name()) return status } @@ -1160,7 +1182,7 @@ func (f *frameworkImpl) RunPermitPlugins(ctx context.Context, state *framework.C statusCode = framework.Wait } else { err := status.AsError() - klog.ErrorS(err, "Failed running Permit plugin", "plugin", pl.Name(), "pod", klog.KObj(pod)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running Permit plugin %q: %w", pl.Name(), err)).WithFailedPlugin(pl.Name()) } } @@ -1169,7 +1191,7 @@ func (f *frameworkImpl) RunPermitPlugins(ctx context.Context, state *framework.C waitingPod := newWaitingPod(pod, pluginsWaitTime) f.waitingPods.add(waitingPod) msg := fmt.Sprintf("one or more plugins asked to wait and no plugin rejected pod %q", pod.Name) - klog.V(4).InfoS("One or more plugins asked to wait and no plugin rejected pod", "pod", klog.KObj(pod)) + logger.V(4).Info("One or more plugins asked to wait and no plugin rejected pod") return framework.NewStatus(framework.Wait, msg) } return nil @@ -1192,7 +1214,9 @@ func (f *frameworkImpl) WaitOnPermit(ctx context.Context, pod *v1.Pod) *framewor return nil } defer f.waitingPods.remove(pod.UID) - klog.V(4).InfoS("Pod waiting on permit", "pod", klog.KObj(pod)) + + logger := klog.FromContext(ctx) + logger.V(4).Info("Pod waiting on permit", "pod", klog.KObj(pod)) startTime := time.Now() s := <-waitingPod.s @@ -1200,12 +1224,12 @@ func (f *frameworkImpl) WaitOnPermit(ctx context.Context, pod *v1.Pod) *framewor if !s.IsSuccess() { if s.IsUnschedulable() { - klog.V(4).InfoS("Pod rejected while waiting on permit", "pod", klog.KObj(pod), "status", s.Message()) + logger.V(4).Info("Pod rejected while waiting on permit", "pod", klog.KObj(pod), "status", s.Message()) s.SetFailedPlugin(s.FailedPlugin()) return s } err := s.AsError() - klog.ErrorS(err, "Failed waiting on permit for pod", "pod", klog.KObj(pod)) + logger.Error(err, "Failed waiting on permit for pod", "pod", klog.KObj(pod)) return framework.AsStatus(fmt.Errorf("waiting on permit for pod: %w", err)).WithFailedPlugin(s.FailedPlugin()) } return nil @@ -1333,3 +1357,8 @@ func (f *frameworkImpl) ProfileName() string { func (f *frameworkImpl) Parallelizer() parallelize.Parallelizer { return f.parallelizer } + +// Logger returns a logger for background activities. +func (f *frameworkImpl) Logger() klog.Logger { + return f.logger +} diff --git a/pkg/scheduler/framework/runtime/framework_test.go b/pkg/scheduler/framework/runtime/framework_test.go index a26afe96af17..67e12d144e42 100644 --- a/pkg/scheduler/framework/runtime/framework_test.go +++ b/pkg/scheduler/framework/runtime/framework_test.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/component-base/metrics/testutil" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" @@ -1590,6 +1591,7 @@ func TestFilterPluginsWithNominatedPods(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) registry := Registry{} cfgPls := &config.Plugins{} @@ -1621,6 +1623,7 @@ func TestFilterPluginsWithNominatedPods(t *testing.T) { podNominator := internalqueue.NewPodNominator(nil) if tt.nominatedPod != nil { podNominator.AddNominatedPod( + logger, framework.NewPodInfo(tt.nominatedPod), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: nodeName}) } diff --git a/pkg/scheduler/framework/types.go b/pkg/scheduler/framework/types.go index 93970327d5aa..9c8e79b1963b 100644 --- a/pkg/scheduler/framework/types.go +++ b/pkg/scheduler/framework/types.go @@ -631,11 +631,11 @@ func podWithRequiredAntiAffinity(p *v1.Pod) bool { len(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 } -func removeFromSlice(s []*PodInfo, k string) []*PodInfo { +func removeFromSlice(logger klog.Logger, s []*PodInfo, k string) []*PodInfo { for i := range s { k2, err := GetPodKey(s[i].Pod) if err != nil { - klog.ErrorS(err, "Cannot get pod key", "pod", klog.KObj(s[i].Pod)) + logger.Error(err, "Cannot get pod key", "pod", klog.KObj(s[i].Pod)) continue } if k == k2 { @@ -649,22 +649,22 @@ func removeFromSlice(s []*PodInfo, k string) []*PodInfo { } // RemovePod subtracts pod information from this NodeInfo. -func (n *NodeInfo) RemovePod(pod *v1.Pod) error { +func (n *NodeInfo) RemovePod(logger klog.Logger, pod *v1.Pod) error { k, err := GetPodKey(pod) if err != nil { return err } if podWithAffinity(pod) { - n.PodsWithAffinity = removeFromSlice(n.PodsWithAffinity, k) + n.PodsWithAffinity = removeFromSlice(logger, n.PodsWithAffinity, k) } if podWithRequiredAntiAffinity(pod) { - n.PodsWithRequiredAntiAffinity = removeFromSlice(n.PodsWithRequiredAntiAffinity, k) + n.PodsWithRequiredAntiAffinity = removeFromSlice(logger, n.PodsWithRequiredAntiAffinity, k) } for i := range n.Pods { k2, err := GetPodKey(n.Pods[i].Pod) if err != nil { - klog.ErrorS(err, "Cannot get pod key", "pod", klog.KObj(n.Pods[i].Pod)) + logger.Error(err, "Cannot get pod key", "pod", klog.KObj(n.Pods[i].Pod)) continue } if k == k2 { diff --git a/pkg/scheduler/framework/types_test.go b/pkg/scheduler/framework/types_test.go index 7b1a0b420906..31764f8f906b 100644 --- a/pkg/scheduler/framework/types_test.go +++ b/pkg/scheduler/framework/types_test.go @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2/ktesting" ) func TestNewResource(t *testing.T) { @@ -1081,10 +1082,11 @@ func TestNodeInfoRemovePod(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) ni := fakeNodeInfo(pods...) gen := ni.Generation - err := ni.RemovePod(test.pod) + err := ni.RemovePod(logger, test.pod) if err != nil { if test.errExpected { expectedErrorMsg := fmt.Errorf("no corresponding pod %s in pods of node %s", test.pod.Name, ni.Node().Name) diff --git a/pkg/scheduler/internal/cache/cache.go b/pkg/scheduler/internal/cache/cache.go index 1cd69a9704db..9c1f6a531025 100644 --- a/pkg/scheduler/internal/cache/cache.go +++ b/pkg/scheduler/internal/cache/cache.go @@ -17,6 +17,7 @@ limitations under the License. package cache import ( + "context" "fmt" "os" "sync" @@ -37,10 +38,11 @@ var ( // New returns a Cache implementation. // It automatically starts a go routine that manages expiration of assumed pods. // "ttl" is how long the assumed pod will get expired. -// "stop" is the channel that would close the background goroutine. -func New(ttl time.Duration, stop <-chan struct{}) Cache { - cache := newCache(ttl, cleanAssumedPeriod, stop) - cache.run() +// "ctx" is the context that would close the background goroutine. +func New(ctx context.Context, ttl time.Duration) Cache { + logger := klog.FromContext(ctx) + cache := newCache(ctx, ttl, cleanAssumedPeriod) + cache.run(logger) return cache } @@ -97,14 +99,15 @@ func (cache *cacheImpl) createImageStateSummary(state *imageState) *framework.Im } } -func newCache(ttl, period time.Duration, stop <-chan struct{}) *cacheImpl { +func newCache(ctx context.Context, ttl, period time.Duration) *cacheImpl { + logger := klog.FromContext(ctx) return &cacheImpl{ ttl: ttl, period: period, - stop: stop, + stop: ctx.Done(), nodes: make(map[string]*nodeInfoListItem), - nodeTree: newNodeTree(nil), + nodeTree: newNodeTree(logger, nil), assumedPods: make(sets.String), podStates: make(map[string]*podState), imageStates: make(map[string]*imageState), @@ -121,10 +124,10 @@ func newNodeInfoListItem(ni *framework.NodeInfo) *nodeInfoListItem { // moveNodeInfoToHead moves a NodeInfo to the head of "cache.nodes" doubly // linked list. The head is the most recently updated NodeInfo. // We assume cache lock is already acquired. -func (cache *cacheImpl) moveNodeInfoToHead(name string) { +func (cache *cacheImpl) moveNodeInfoToHead(logger klog.Logger, name string) { ni, ok := cache.nodes[name] if !ok { - klog.ErrorS(nil, "No node info with given name found in the cache", "node", klog.KRef("", name)) + logger.Error(nil, "No node info with given name found in the cache", "node", klog.KRef("", name)) return } // if the node info list item is already at the head, we are done. @@ -149,10 +152,10 @@ func (cache *cacheImpl) moveNodeInfoToHead(name string) { // removeNodeInfoFromList removes a NodeInfo from the "cache.nodes" doubly // linked list. // We assume cache lock is already acquired. -func (cache *cacheImpl) removeNodeInfoFromList(name string) { +func (cache *cacheImpl) removeNodeInfoFromList(logger klog.Logger, name string) { ni, ok := cache.nodes[name] if !ok { - klog.ErrorS(nil, "No node info with given name found in the cache", "node", klog.KRef("", name)) + logger.Error(nil, "No node info with given name found in the cache", "node", klog.KRef("", name)) return } @@ -194,7 +197,7 @@ func (cache *cacheImpl) Dump() *Dump { // nodeInfo.Node() is guaranteed to be not nil for all the nodes in the snapshot. // This function tracks generation number of NodeInfo and updates only the // entries of an existing snapshot that have changed after the snapshot was taken. -func (cache *cacheImpl) UpdateSnapshot(nodeSnapshot *Snapshot) error { +func (cache *cacheImpl) UpdateSnapshot(logger klog.Logger, nodeSnapshot *Snapshot) error { cache.mu.Lock() defer cache.mu.Unlock() @@ -271,7 +274,7 @@ func (cache *cacheImpl) UpdateSnapshot(nodeSnapshot *Snapshot) error { } if updateAllLists || updateNodesHavePodsWithAffinity || updateNodesHavePodsWithRequiredAntiAffinity || updateUsedPVCSet { - cache.updateNodeInfoSnapshotList(nodeSnapshot, updateAllLists) + cache.updateNodeInfoSnapshotList(logger, nodeSnapshot, updateAllLists) } if len(nodeSnapshot.nodeInfoList) != cache.nodeTree.numNodes { @@ -280,17 +283,17 @@ func (cache *cacheImpl) UpdateSnapshot(nodeSnapshot *Snapshot) error { ", trying to recover", len(nodeSnapshot.nodeInfoList), cache.nodeTree.numNodes, len(nodeSnapshot.nodeInfoMap), len(cache.nodes)) - klog.ErrorS(nil, errMsg) + logger.Error(nil, errMsg) // We will try to recover by re-creating the lists for the next scheduling cycle, but still return an // error to surface the problem, the error will likely cause a failure to the current scheduling cycle. - cache.updateNodeInfoSnapshotList(nodeSnapshot, true) + cache.updateNodeInfoSnapshotList(logger, nodeSnapshot, true) return fmt.Errorf(errMsg) } return nil } -func (cache *cacheImpl) updateNodeInfoSnapshotList(snapshot *Snapshot, updateAll bool) { +func (cache *cacheImpl) updateNodeInfoSnapshotList(logger klog.Logger, snapshot *Snapshot, updateAll bool) { snapshot.havePodsWithAffinityNodeInfoList = make([]*framework.NodeInfo, 0, cache.nodeTree.numNodes) snapshot.havePodsWithRequiredAntiAffinityNodeInfoList = make([]*framework.NodeInfo, 0, cache.nodeTree.numNodes) snapshot.usedPVCSet = sets.NewString() @@ -299,7 +302,7 @@ func (cache *cacheImpl) updateNodeInfoSnapshotList(snapshot *Snapshot, updateAll snapshot.nodeInfoList = make([]*framework.NodeInfo, 0, cache.nodeTree.numNodes) nodesList, err := cache.nodeTree.list() if err != nil { - klog.ErrorS(err, "Error occurred while retrieving the list of names of the nodes from node tree") + logger.Error(err, "Error occurred while retrieving the list of names of the nodes from node tree") } for _, nodeName := range nodesList { if nodeInfo := snapshot.nodeInfoMap[nodeName]; nodeInfo != nil { @@ -314,7 +317,7 @@ func (cache *cacheImpl) updateNodeInfoSnapshotList(snapshot *Snapshot, updateAll snapshot.usedPVCSet.Insert(key) } } else { - klog.ErrorS(nil, "Node exists in nodeTree but not in NodeInfoMap, this should not happen", "node", klog.KRef("", nodeName)) + logger.Error(nil, "Node exists in nodeTree but not in NodeInfoMap, this should not happen", "node", klog.KRef("", nodeName)) } } } else { @@ -369,7 +372,7 @@ func (cache *cacheImpl) PodCount() (int, error) { return count, nil } -func (cache *cacheImpl) AssumePod(pod *v1.Pod) error { +func (cache *cacheImpl) AssumePod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -381,15 +384,15 @@ func (cache *cacheImpl) AssumePod(pod *v1.Pod) error { return fmt.Errorf("pod %v is in the cache, so can't be assumed", key) } - return cache.addPod(pod, true) + return cache.addPod(logger, pod, true) } -func (cache *cacheImpl) FinishBinding(pod *v1.Pod) error { - return cache.finishBinding(pod, time.Now()) +func (cache *cacheImpl) FinishBinding(logger klog.Logger, pod *v1.Pod) error { + return cache.finishBinding(logger, pod, time.Now()) } // finishBinding exists to make tests deterministic by injecting now as an argument -func (cache *cacheImpl) finishBinding(pod *v1.Pod, now time.Time) error { +func (cache *cacheImpl) finishBinding(logger klog.Logger, pod *v1.Pod, now time.Time) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -398,7 +401,7 @@ func (cache *cacheImpl) finishBinding(pod *v1.Pod, now time.Time) error { cache.mu.RLock() defer cache.mu.RUnlock() - klog.V(5).InfoS("Finished binding for pod, can be expired", "pod", klog.KObj(pod)) + logger.V(5).Info("Finished binding for pod, can be expired", "pod", klog.KObj(pod)) currState, ok := cache.podStates[key] if ok && cache.assumedPods.Has(key) { dl := now.Add(cache.ttl) @@ -408,7 +411,7 @@ func (cache *cacheImpl) finishBinding(pod *v1.Pod, now time.Time) error { return nil } -func (cache *cacheImpl) ForgetPod(pod *v1.Pod) error { +func (cache *cacheImpl) ForgetPod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -424,13 +427,13 @@ func (cache *cacheImpl) ForgetPod(pod *v1.Pod) error { // Only assumed pod can be forgotten. if ok && cache.assumedPods.Has(key) { - return cache.removePod(pod) + return cache.removePod(logger, pod) } return fmt.Errorf("pod %v wasn't assumed so cannot be forgotten", key) } // Assumes that lock is already acquired. -func (cache *cacheImpl) addPod(pod *v1.Pod, assumePod bool) error { +func (cache *cacheImpl) addPod(logger klog.Logger, pod *v1.Pod, assumePod bool) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -441,7 +444,7 @@ func (cache *cacheImpl) addPod(pod *v1.Pod, assumePod bool) error { cache.nodes[pod.Spec.NodeName] = n } n.info.AddPod(pod) - cache.moveNodeInfoToHead(pod.Spec.NodeName) + cache.moveNodeInfoToHead(logger, pod.Spec.NodeName) ps := &podState{ pod: pod, } @@ -453,18 +456,18 @@ func (cache *cacheImpl) addPod(pod *v1.Pod, assumePod bool) error { } // Assumes that lock is already acquired. -func (cache *cacheImpl) updatePod(oldPod, newPod *v1.Pod) error { - if err := cache.removePod(oldPod); err != nil { +func (cache *cacheImpl) updatePod(logger klog.Logger, oldPod, newPod *v1.Pod) error { + if err := cache.removePod(logger, oldPod); err != nil { return err } - return cache.addPod(newPod, false) + return cache.addPod(logger, newPod, false) } // Assumes that lock is already acquired. // Removes a pod from the cached node info. If the node information was already // removed and there are no more pods left in the node, cleans up the node from // the cache. -func (cache *cacheImpl) removePod(pod *v1.Pod) error { +func (cache *cacheImpl) removePod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -472,15 +475,15 @@ func (cache *cacheImpl) removePod(pod *v1.Pod) error { n, ok := cache.nodes[pod.Spec.NodeName] if !ok { - klog.ErrorS(nil, "Node not found when trying to remove pod", "node", klog.KRef("", pod.Spec.NodeName), "pod", klog.KObj(pod)) + logger.Error(nil, "Node not found when trying to remove pod", "node", klog.KRef("", pod.Spec.NodeName), "pod", klog.KObj(pod)) } else { - if err := n.info.RemovePod(pod); err != nil { + if err := n.info.RemovePod(logger, pod); err != nil { return err } if len(n.info.Pods) == 0 && n.info.Node() == nil { - cache.removeNodeInfoFromList(pod.Spec.NodeName) + cache.removeNodeInfoFromList(logger, pod.Spec.NodeName) } else { - cache.moveNodeInfoToHead(pod.Spec.NodeName) + cache.moveNodeInfoToHead(logger, pod.Spec.NodeName) } } @@ -489,7 +492,7 @@ func (cache *cacheImpl) removePod(pod *v1.Pod) error { return nil } -func (cache *cacheImpl) AddPod(pod *v1.Pod) error { +func (cache *cacheImpl) AddPod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -503,9 +506,9 @@ func (cache *cacheImpl) AddPod(pod *v1.Pod) error { case ok && cache.assumedPods.Has(key): if currState.pod.Spec.NodeName != pod.Spec.NodeName { // The pod was added to a different node than it was assumed to. - klog.InfoS("Pod was added to a different node than it was assumed", "pod", klog.KObj(pod), "assumedNode", klog.KRef("", pod.Spec.NodeName), "currentNode", klog.KRef("", currState.pod.Spec.NodeName)) - if err = cache.updatePod(currState.pod, pod); err != nil { - klog.ErrorS(err, "Error occurred while updating pod") + logger.Info("Pod was added to a different node than it was assumed", "pod", klog.KObj(pod), "assumedNode", klog.KRef("", pod.Spec.NodeName), "currentNode", klog.KRef("", currState.pod.Spec.NodeName)) + if err = cache.updatePod(logger, currState.pod, pod); err != nil { + logger.Error(err, "Error occurred while updating pod") } } else { delete(cache.assumedPods, key) @@ -514,8 +517,8 @@ func (cache *cacheImpl) AddPod(pod *v1.Pod) error { } case !ok: // Pod was expired. We should add it back. - if err = cache.addPod(pod, false); err != nil { - klog.ErrorS(err, "Error occurred while adding pod") + if err = cache.addPod(logger, pod, false); err != nil { + logger.Error(err, "Error occurred while adding pod") } default: return fmt.Errorf("pod %v was already in added state", key) @@ -523,7 +526,7 @@ func (cache *cacheImpl) AddPod(pod *v1.Pod) error { return nil } -func (cache *cacheImpl) UpdatePod(oldPod, newPod *v1.Pod) error { +func (cache *cacheImpl) UpdatePod(logger klog.Logger, oldPod, newPod *v1.Pod) error { key, err := framework.GetPodKey(oldPod) if err != nil { return err @@ -537,16 +540,16 @@ func (cache *cacheImpl) UpdatePod(oldPod, newPod *v1.Pod) error { // before Update event, in which case the state would change from Assumed to Added. if ok && !cache.assumedPods.Has(key) { if currState.pod.Spec.NodeName != newPod.Spec.NodeName { - klog.ErrorS(nil, "Pod updated on a different node than previously added to", "pod", klog.KObj(oldPod)) - klog.ErrorS(nil, "scheduler cache is corrupted and can badly affect scheduling decisions") + logger.Error(nil, "Pod updated on a different node than previously added to", "pod", klog.KObj(oldPod)) + logger.Error(nil, "scheduler cache is corrupted and can badly affect scheduling decisions") os.Exit(1) } - return cache.updatePod(oldPod, newPod) + return cache.updatePod(logger, oldPod, newPod) } return fmt.Errorf("pod %v is not added to scheduler cache, so cannot be updated", key) } -func (cache *cacheImpl) RemovePod(pod *v1.Pod) error { +func (cache *cacheImpl) RemovePod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -560,15 +563,15 @@ func (cache *cacheImpl) RemovePod(pod *v1.Pod) error { return fmt.Errorf("pod %v is not found in scheduler cache, so cannot be removed from it", key) } if currState.pod.Spec.NodeName != pod.Spec.NodeName { - klog.ErrorS(nil, "Pod was added to a different node than it was assumed", "pod", klog.KObj(pod), "assumedNode", klog.KRef("", pod.Spec.NodeName), "currentNode", klog.KRef("", currState.pod.Spec.NodeName)) + logger.Error(nil, "Pod was added to a different node than it was assumed", "pod", klog.KObj(pod), "assumedNode", klog.KRef("", pod.Spec.NodeName), "currentNode", klog.KRef("", currState.pod.Spec.NodeName)) if pod.Spec.NodeName != "" { // An empty NodeName is possible when the scheduler misses a Delete // event and it gets the last known state from the informer cache. - klog.ErrorS(nil, "scheduler cache is corrupted and can badly affect scheduling decisions") + logger.Error(nil, "scheduler cache is corrupted and can badly affect scheduling decisions") os.Exit(1) } } - return cache.removePod(currState.pod) + return cache.removePod(logger, currState.pod) } func (cache *cacheImpl) IsAssumedPod(pod *v1.Pod) (bool, error) { @@ -602,7 +605,7 @@ func (cache *cacheImpl) GetPod(pod *v1.Pod) (*v1.Pod, error) { return podState.pod, nil } -func (cache *cacheImpl) AddNode(node *v1.Node) *framework.NodeInfo { +func (cache *cacheImpl) AddNode(logger klog.Logger, node *v1.Node) *framework.NodeInfo { cache.mu.Lock() defer cache.mu.Unlock() @@ -613,15 +616,15 @@ func (cache *cacheImpl) AddNode(node *v1.Node) *framework.NodeInfo { } else { cache.removeNodeImageStates(n.info.Node()) } - cache.moveNodeInfoToHead(node.Name) + cache.moveNodeInfoToHead(logger, node.Name) - cache.nodeTree.addNode(node) + cache.nodeTree.addNode(logger, node) cache.addNodeImageStates(node, n.info) n.info.SetNode(node) return n.info.Clone() } -func (cache *cacheImpl) UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInfo { +func (cache *cacheImpl) UpdateNode(logger klog.Logger, oldNode, newNode *v1.Node) *framework.NodeInfo { cache.mu.Lock() defer cache.mu.Unlock() @@ -629,13 +632,13 @@ func (cache *cacheImpl) UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInf if !ok { n = newNodeInfoListItem(framework.NewNodeInfo()) cache.nodes[newNode.Name] = n - cache.nodeTree.addNode(newNode) + cache.nodeTree.addNode(logger, newNode) } else { cache.removeNodeImageStates(n.info.Node()) } - cache.moveNodeInfoToHead(newNode.Name) + cache.moveNodeInfoToHead(logger, newNode.Name) - cache.nodeTree.updateNode(oldNode, newNode) + cache.nodeTree.updateNode(logger, oldNode, newNode) cache.addNodeImageStates(newNode, n.info) n.info.SetNode(newNode) return n.info.Clone() @@ -647,7 +650,7 @@ func (cache *cacheImpl) UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInf // the source of truth. // However, we keep a ghost node with the list of pods until all pod deletion // events have arrived. A ghost node is skipped from snapshots. -func (cache *cacheImpl) RemoveNode(node *v1.Node) error { +func (cache *cacheImpl) RemoveNode(logger klog.Logger, node *v1.Node) error { cache.mu.Lock() defer cache.mu.Unlock() @@ -661,11 +664,11 @@ func (cache *cacheImpl) RemoveNode(node *v1.Node) error { // in a different watch, and thus can potentially be observed later, even though // they happened before node removal. if len(n.info.Pods) == 0 { - cache.removeNodeInfoFromList(node.Name) + cache.removeNodeInfoFromList(logger, node.Name) } else { - cache.moveNodeInfoToHead(node.Name) + cache.moveNodeInfoToHead(logger, node.Name) } - if err := cache.nodeTree.removeNode(node); err != nil { + if err := cache.nodeTree.removeNode(logger, node); err != nil { return err } cache.removeNodeImageStates(node) @@ -723,17 +726,15 @@ func (cache *cacheImpl) removeNodeImageStates(node *v1.Node) { } } -func (cache *cacheImpl) run() { - go wait.Until(cache.cleanupExpiredAssumedPods, cache.period, cache.stop) -} - -func (cache *cacheImpl) cleanupExpiredAssumedPods() { - cache.cleanupAssumedPods(time.Now()) +func (cache *cacheImpl) run(logger klog.Logger) { + go wait.Until(func() { + cache.cleanupAssumedPods(logger, time.Now()) + }, cache.period, cache.stop) } // cleanupAssumedPods exists for making test deterministic by taking time as input argument. // It also reports metrics on the cache size for nodes, pods, and assumed pods. -func (cache *cacheImpl) cleanupAssumedPods(now time.Time) { +func (cache *cacheImpl) cleanupAssumedPods(logger klog.Logger, now time.Time) { cache.mu.Lock() defer cache.mu.Unlock() defer cache.updateMetrics() @@ -742,18 +743,18 @@ func (cache *cacheImpl) cleanupAssumedPods(now time.Time) { for key := range cache.assumedPods { ps, ok := cache.podStates[key] if !ok { - klog.ErrorS(nil, "Key found in assumed set but not in podStates, potentially a logical error") + logger.Error(nil, "Key found in assumed set but not in podStates, potentially a logical error") os.Exit(1) } if !ps.bindingFinished { - klog.V(5).InfoS("Could not expire cache for pod as binding is still in progress", + logger.V(5).Info("Could not expire cache for pod as binding is still in progress", "pod", klog.KObj(ps.pod)) continue } if now.After(*ps.deadline) { - klog.InfoS("Pod expired", "pod", klog.KObj(ps.pod)) - if err := cache.removePod(ps.pod); err != nil { - klog.ErrorS(err, "ExpirePod failed", "pod", klog.KObj(ps.pod)) + logger.Info("Pod expired", "pod", klog.KObj(ps.pod)) + if err := cache.removePod(logger, ps.pod); err != nil { + logger.Error(err, "ExpirePod failed", "pod", klog.KObj(ps.pod)) } } } diff --git a/pkg/scheduler/internal/cache/cache_test.go b/pkg/scheduler/internal/cache/cache_test.go index daaeb1ddeeed..bb31a229b92d 100644 --- a/pkg/scheduler/internal/cache/cache_test.go +++ b/pkg/scheduler/internal/cache/cache_test.go @@ -17,6 +17,7 @@ limitations under the License. package cache import ( + "context" "errors" "fmt" "reflect" @@ -30,6 +31,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/framework" st "k8s.io/kubernetes/pkg/scheduler/testing" schedutil "k8s.io/kubernetes/pkg/scheduler/util" @@ -207,9 +210,12 @@ func TestAssumePodScheduled(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(time.Second, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, time.Second, time.Second) for _, pod := range tt.pods { - if err := cache.AssumePod(pod); err != nil { + if err := cache.AssumePod(logger, pod); err != nil { t.Fatalf("AssumePod failed: %v", err) } } @@ -219,7 +225,7 @@ func TestAssumePodScheduled(t *testing.T) { } for _, pod := range tt.pods { - if err := cache.ForgetPod(pod); err != nil { + if err := cache.ForgetPod(logger, pod); err != nil { t.Fatalf("ForgetPod failed: %v", err) } if err := isForgottenFromCache(pod, cache); err != nil { @@ -236,11 +242,11 @@ type testExpirePodStruct struct { assumedTime time.Time } -func assumeAndFinishBinding(cache *cacheImpl, pod *v1.Pod, assumedTime time.Time) error { - if err := cache.AssumePod(pod); err != nil { +func assumeAndFinishBinding(logger klog.Logger, cache *cacheImpl, pod *v1.Pod, assumedTime time.Time) error { + if err := cache.AssumePod(logger, pod); err != nil { return err } - return cache.finishBinding(pod, assumedTime) + return cache.finishBinding(logger, pod, assumedTime) } // TestExpirePod tests that assumed pods will be removed if expired. @@ -290,22 +296,25 @@ func TestExpirePod(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, pod := range tt.pods { - if err := cache.AssumePod(pod.pod); err != nil { + if err := cache.AssumePod(logger, pod.pod); err != nil { t.Fatal(err) } if !pod.finishBind { continue } - if err := cache.finishBinding(pod.pod, pod.assumedTime); err != nil { + if err := cache.finishBinding(logger, pod.pod, pod.assumedTime); err != nil { t.Fatal(err) } } // pods that got bound and have assumedTime + ttl < cleanupTime will get // expired and removed - cache.cleanupAssumedPods(tt.cleanupTime) + cache.cleanupAssumedPods(logger, tt.cleanupTime) n := cache.nodes[nodeName] if err := deepEqualWithoutGeneration(n, tt.wNodeInfo); err != nil { t.Error(err) @@ -350,18 +359,21 @@ func TestAddPodWillConfirm(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, podToAssume := range tt.podsToAssume { - if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, podToAssume, now); err != nil { t.Fatalf("assumePod failed: %v", err) } } for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Fatalf("AddPod failed: %v", err) } } - cache.cleanupAssumedPods(now.Add(2 * ttl)) + cache.cleanupAssumedPods(logger, now.Add(2*ttl)) // check after expiration. confirmed pods shouldn't be expired. n := cache.nodes[nodeName] if err := deepEqualWithoutGeneration(n, tt.wNodeInfo); err != nil { @@ -390,14 +402,17 @@ func TestDump(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, podToAssume := range tt.podsToAssume { - if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, podToAssume, now); err != nil { t.Errorf("assumePod failed: %v", err) } } for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Errorf("AddPod failed: %v", err) } } @@ -458,19 +473,22 @@ func TestAddPodWillReplaceAssumed(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, podToAssume := range tt.podsToAssume { - if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, podToAssume, now); err != nil { t.Fatalf("assumePod failed: %v", err) } } for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Fatalf("AddPod failed: %v", err) } } for _, podToUpdate := range tt.podsToUpdate { - if err := cache.UpdatePod(podToUpdate[0], podToUpdate[1]); err != nil { + if err := cache.UpdatePod(logger, podToUpdate[0], podToUpdate[1]); err != nil { t.Fatalf("UpdatePod failed: %v", err) } } @@ -512,17 +530,20 @@ func TestAddPodAfterExpiration(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() now := time.Now() - cache := newCache(ttl, time.Second, nil) - if err := assumeAndFinishBinding(cache, tt.pod, now); err != nil { + cache := newCache(ctx, ttl, time.Second) + if err := assumeAndFinishBinding(logger, cache, tt.pod, now); err != nil { t.Fatalf("assumePod failed: %v", err) } - cache.cleanupAssumedPods(now.Add(2 * ttl)) + cache.cleanupAssumedPods(logger, now.Add(2*ttl)) // It should be expired and removed. if err := isForgottenFromCache(tt.pod, cache); err != nil { t.Error(err) } - if err := cache.AddPod(tt.pod); err != nil { + if err := cache.AddPod(logger, tt.pod); err != nil { t.Fatalf("AddPod failed: %v", err) } // check after expiration. confirmed pods shouldn't be expired. @@ -579,9 +600,12 @@ func TestUpdatePod(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Fatalf("AddPod failed: %v", err) } } @@ -590,7 +614,7 @@ func TestUpdatePod(t *testing.T) { if j == 0 { continue } - if err := cache.UpdatePod(tt.podsToUpdate[j-1], tt.podsToUpdate[j]); err != nil { + if err := cache.UpdatePod(logger, tt.podsToUpdate[j-1], tt.podsToUpdate[j]); err != nil { t.Fatalf("UpdatePod failed: %v", err) } // check after expiration. confirmed pods shouldn't be expired. @@ -615,7 +639,7 @@ func TestUpdatePodAndGet(t *testing.T) { pod *v1.Pod podToUpdate *v1.Pod - handler func(cache Cache, pod *v1.Pod) error + handler func(logger klog.Logger, cache Cache, pod *v1.Pod) error assumePod bool }{ @@ -623,8 +647,8 @@ func TestUpdatePodAndGet(t *testing.T) { pod: testPods[0], podToUpdate: testPods[0], - handler: func(cache Cache, pod *v1.Pod) error { - return cache.AssumePod(pod) + handler: func(logger klog.Logger, cache Cache, pod *v1.Pod) error { + return cache.AssumePod(logger, pod) }, assumePod: true, }, @@ -632,8 +656,8 @@ func TestUpdatePodAndGet(t *testing.T) { pod: testPods[0], podToUpdate: testPods[1], - handler: func(cache Cache, pod *v1.Pod) error { - return cache.AddPod(pod) + handler: func(logger klog.Logger, cache Cache, pod *v1.Pod) error { + return cache.AddPod(logger, pod) }, assumePod: false, }, @@ -641,14 +665,17 @@ func TestUpdatePodAndGet(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) - if err := tt.handler(cache, tt.pod); err != nil { + if err := tt.handler(logger, cache, tt.pod); err != nil { t.Fatalf("unexpected err: %v", err) } if !tt.assumePod { - if err := cache.UpdatePod(tt.pod, tt.podToUpdate); err != nil { + if err := cache.UpdatePod(logger, tt.pod, tt.podToUpdate); err != nil { t.Fatalf("UpdatePod failed: %v", err) } } @@ -711,17 +738,20 @@ func TestExpireAddUpdatePod(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() now := time.Now() - cache := newCache(ttl, time.Second, nil) + cache := newCache(ctx, ttl, time.Second) for _, podToAssume := range tt.podsToAssume { - if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, podToAssume, now); err != nil { t.Fatalf("assumePod failed: %v", err) } } - cache.cleanupAssumedPods(now.Add(2 * ttl)) + cache.cleanupAssumedPods(logger, now.Add(2*ttl)) for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Fatalf("AddPod failed: %v", err) } } @@ -730,7 +760,7 @@ func TestExpireAddUpdatePod(t *testing.T) { if j == 0 { continue } - if err := cache.UpdatePod(tt.podsToUpdate[j-1], tt.podsToUpdate[j]); err != nil { + if err := cache.UpdatePod(logger, tt.podsToUpdate[j-1], tt.podsToUpdate[j]); err != nil { t.Fatalf("UpdatePod failed: %v", err) } // check after expiration. confirmed pods shouldn't be expired. @@ -776,8 +806,11 @@ func TestEphemeralStorageResource(t *testing.T) { } for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(time.Second, time.Second, nil) - if err := cache.AddPod(tt.pod); err != nil { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, time.Second, time.Second) + if err := cache.AddPod(logger, tt.pod); err != nil { t.Fatalf("AddPod failed: %v", err) } n := cache.nodes[nodeName] @@ -785,7 +818,7 @@ func TestEphemeralStorageResource(t *testing.T) { t.Error(err) } - if err := cache.RemovePod(tt.pod); err != nil { + if err := cache.RemovePod(logger, tt.pod); err != nil { t.Fatalf("RemovePod failed: %v", err) } if _, err := cache.GetPod(tt.pod); err == nil { @@ -828,15 +861,18 @@ func TestRemovePod(t *testing.T) { for name, tt := range tests { t.Run(name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() nodeName := pod.Spec.NodeName - cache := newCache(time.Second, time.Second, nil) + cache := newCache(ctx, time.Second, time.Second) // Add/Assume pod succeeds even before adding the nodes. if tt.assume { - if err := cache.AddPod(pod); err != nil { + if err := cache.AddPod(logger, pod); err != nil { t.Fatalf("AddPod failed: %v", err) } } else { - if err := cache.AssumePod(pod); err != nil { + if err := cache.AssumePod(logger, pod); err != nil { t.Fatalf("AssumePod failed: %v", err) } } @@ -845,10 +881,10 @@ func TestRemovePod(t *testing.T) { t.Error(err) } for _, n := range nodes { - cache.AddNode(n) + cache.AddNode(logger, n) } - if err := cache.RemovePod(pod); err != nil { + if err := cache.RemovePod(logger, pod); err != nil { t.Fatalf("RemovePod failed: %v", err) } @@ -870,10 +906,13 @@ func TestForgetPod(t *testing.T) { pods := []*v1.Pod{basePod} now := time.Now() ttl := 10 * time.Second + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() - cache := newCache(ttl, time.Second, nil) + cache := newCache(ctx, ttl, time.Second) for _, pod := range pods { - if err := assumeAndFinishBinding(cache, pod, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, pod, now); err != nil { t.Fatalf("assumePod failed: %v", err) } isAssumed, err := cache.IsAssumedPod(pod) @@ -895,7 +934,7 @@ func TestForgetPod(t *testing.T) { } } for _, pod := range pods { - if err := cache.ForgetPod(pod); err != nil { + if err := cache.ForgetPod(logger, pod); err != nil { t.Fatalf("ForgetPod failed: %v", err) } if err := isForgottenFromCache(pod, cache); err != nil { @@ -1050,13 +1089,16 @@ func TestNodeOperators(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() expected := buildNodeInfo(test.node, test.pods) node := test.node - cache := newCache(time.Second, time.Second, nil) - cache.AddNode(node) + cache := newCache(ctx, time.Second, time.Second) + cache.AddNode(logger, node) for _, pod := range test.pods { - if err := cache.AddPod(pod); err != nil { + if err := cache.AddPod(logger, pod); err != nil { t.Fatal(err) } } @@ -1082,7 +1124,7 @@ func TestNodeOperators(t *testing.T) { // Step 2: dump cached nodes successfully. cachedNodes := NewEmptySnapshot() - if err := cache.UpdateSnapshot(cachedNodes); err != nil { + if err := cache.UpdateSnapshot(logger, cachedNodes); err != nil { t.Error(err) } newNode, found := cachedNodes.nodeInfoMap[node.Name] @@ -1098,7 +1140,7 @@ func TestNodeOperators(t *testing.T) { node.Status.Allocatable[v1.ResourceMemory] = mem50m expected.Allocatable.Memory = mem50m.Value() - cache.UpdateNode(nil, node) + cache.UpdateNode(logger, nil, node) got, found = cache.nodes[node.Name] if !found { t.Errorf("Failed to find node %v in schedulertypes after UpdateNode.", node.Name) @@ -1121,7 +1163,7 @@ func TestNodeOperators(t *testing.T) { } // Step 4: the node can be removed even if it still has pods. - if err := cache.RemoveNode(node); err != nil { + if err := cache.RemoveNode(logger, node); err != nil { t.Error(err) } if n, err := cache.getNodeInfo(node.Name); err != nil { @@ -1146,7 +1188,7 @@ func TestNodeOperators(t *testing.T) { // Step 5: removing pods for the removed node still succeeds. for _, p := range test.pods { - if err := cache.RemovePod(p); err != nil { + if err := cache.RemovePod(logger, p); err != nil { t.Error(err) } if _, err := cache.GetPod(p); err == nil { @@ -1158,6 +1200,8 @@ func TestNodeOperators(t *testing.T) { } func TestSchedulerCache_UpdateSnapshot(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) + // Create a few nodes to be used in tests. var nodes []*v1.Node for i := 0; i < 10; i++ { @@ -1224,73 +1268,73 @@ func TestSchedulerCache_UpdateSnapshot(t *testing.T) { addNode := func(i int) operation { return func(t *testing.T) { - cache.AddNode(nodes[i]) + cache.AddNode(logger, nodes[i]) } } removeNode := func(i int) operation { return func(t *testing.T) { - if err := cache.RemoveNode(nodes[i]); err != nil { + if err := cache.RemoveNode(logger, nodes[i]); err != nil { t.Error(err) } } } updateNode := func(i int) operation { return func(t *testing.T) { - cache.UpdateNode(nodes[i], updatedNodes[i]) + cache.UpdateNode(logger, nodes[i], updatedNodes[i]) } } addPod := func(i int) operation { return func(t *testing.T) { - if err := cache.AddPod(pods[i]); err != nil { + if err := cache.AddPod(logger, pods[i]); err != nil { t.Error(err) } } } addPodWithAffinity := func(i int) operation { return func(t *testing.T) { - if err := cache.AddPod(podsWithAffinity[i]); err != nil { + if err := cache.AddPod(logger, podsWithAffinity[i]); err != nil { t.Error(err) } } } addPodWithPVC := func(i int) operation { return func(t *testing.T) { - if err := cache.AddPod(podsWithPVC[i]); err != nil { + if err := cache.AddPod(logger, podsWithPVC[i]); err != nil { t.Error(err) } } } removePod := func(i int) operation { return func(t *testing.T) { - if err := cache.RemovePod(pods[i]); err != nil { + if err := cache.RemovePod(logger, pods[i]); err != nil { t.Error(err) } } } removePodWithAffinity := func(i int) operation { return func(t *testing.T) { - if err := cache.RemovePod(podsWithAffinity[i]); err != nil { + if err := cache.RemovePod(logger, podsWithAffinity[i]); err != nil { t.Error(err) } } } removePodWithPVC := func(i int) operation { return func(t *testing.T) { - if err := cache.RemovePod(podsWithPVC[i]); err != nil { + if err := cache.RemovePod(logger, podsWithPVC[i]); err != nil { t.Error(err) } } } updatePod := func(i int) operation { return func(t *testing.T) { - if err := cache.UpdatePod(pods[i], updatedPods[i]); err != nil { + if err := cache.UpdatePod(logger, pods[i], updatedPods[i]); err != nil { t.Error(err) } } } updateSnapshot := func() operation { return func(t *testing.T) { - cache.UpdateSnapshot(snapshot) + cache.UpdateSnapshot(logger, snapshot) if err := compareCacheWithNodeInfoSnapshot(t, cache, snapshot); err != nil { t.Error(err) } @@ -1512,7 +1556,10 @@ func TestSchedulerCache_UpdateSnapshot(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cache = newCache(time.Second, time.Second, nil) + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache = newCache(ctx, time.Second, time.Second) snapshot = NewEmptySnapshot() for _, op := range test.operations { @@ -1546,7 +1593,7 @@ func TestSchedulerCache_UpdateSnapshot(t *testing.T) { } // Always update the snapshot at the end of operations and compare it. - if err := cache.UpdateSnapshot(snapshot); err != nil { + if err := cache.UpdateSnapshot(logger, snapshot); err != nil { t.Error(err) } if err := compareCacheWithNodeInfoSnapshot(t, cache, snapshot); err != nil { @@ -1621,6 +1668,8 @@ func compareCacheWithNodeInfoSnapshot(t *testing.T, cache *cacheImpl, snapshot * } func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) + // Create a few nodes to be used in tests. var nodes []*v1.Node i := 0 @@ -1644,7 +1693,7 @@ func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { var snapshot *Snapshot addNode := func(t *testing.T, i int) { - cache.AddNode(nodes[i]) + cache.AddNode(logger, nodes[i]) _, ok := snapshot.nodeInfoMap[nodes[i].Name] if !ok { snapshot.nodeInfoMap[nodes[i].Name] = cache.nodes[nodes[i].Name].info @@ -1652,7 +1701,7 @@ func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { } updateSnapshot := func(t *testing.T) { - cache.updateNodeInfoSnapshotList(snapshot, true) + cache.updateNodeInfoSnapshotList(logger, snapshot, true) if err := compareCacheWithNodeInfoSnapshot(t, cache, snapshot); err != nil { t.Error(err) } @@ -1742,13 +1791,16 @@ func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cache = newCache(time.Second, time.Second, nil) + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache = newCache(ctx, time.Second, time.Second) snapshot = NewEmptySnapshot() test.operations(t) // Always update the snapshot at the end of operations and compare it. - cache.updateNodeInfoSnapshotList(snapshot, true) + cache.updateNodeInfoSnapshotList(logger, snapshot, true) if err := compareCacheWithNodeInfoSnapshot(t, cache, snapshot); err != nil { t.Error(err) } @@ -1764,11 +1816,12 @@ func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { } func BenchmarkUpdate1kNodes30kPods(b *testing.B) { + logger, _ := ktesting.NewTestContext(b) cache := setupCacheOf1kNodes30kPods(b) b.ResetTimer() for n := 0; n < b.N; n++ { cachedNodes := NewEmptySnapshot() - cache.UpdateSnapshot(cachedNodes) + cache.UpdateSnapshot(logger, cachedNodes) } } @@ -1787,12 +1840,13 @@ func BenchmarkExpirePods(b *testing.B) { } func benchmarkExpire(b *testing.B, podNum int) { + logger, _ := ktesting.NewTestContext(b) now := time.Now() for n := 0; n < b.N; n++ { b.StopTimer() cache := setupCacheWithAssumedPods(b, podNum, now) b.StartTimer() - cache.cleanupAssumedPods(now.Add(2 * time.Second)) + cache.cleanupAssumedPods(logger, now.Add(2*time.Second)) } } @@ -1821,14 +1875,17 @@ func makeBasePod(t testingMode, nodeName, objName, cpu, mem, extended string, po } func setupCacheOf1kNodes30kPods(b *testing.B) Cache { - cache := newCache(time.Second, time.Second, nil) + logger, ctx := ktesting.NewTestContext(b) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, time.Second, time.Second) for i := 0; i < 1000; i++ { nodeName := fmt.Sprintf("node-%d", i) for j := 0; j < 30; j++ { objName := fmt.Sprintf("%s-pod-%d", nodeName, j) pod := makeBasePod(b, nodeName, objName, "0", "0", "", nil) - if err := cache.AddPod(pod); err != nil { + if err := cache.AddPod(logger, pod); err != nil { b.Fatalf("AddPod failed: %v", err) } } @@ -1837,13 +1894,16 @@ func setupCacheOf1kNodes30kPods(b *testing.B) Cache { } func setupCacheWithAssumedPods(b *testing.B, podNum int, assumedTime time.Time) *cacheImpl { - cache := newCache(time.Second, time.Second, nil) + logger, ctx := ktesting.NewTestContext(b) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, time.Second, time.Second) for i := 0; i < podNum; i++ { nodeName := fmt.Sprintf("node-%d", i/10) objName := fmt.Sprintf("%s-pod-%d", nodeName, i%10) pod := makeBasePod(b, nodeName, objName, "0", "0", "", nil) - err := assumeAndFinishBinding(cache, pod, assumedTime) + err := assumeAndFinishBinding(logger, cache, pod, assumedTime) if err != nil { b.Fatalf("assumePod failed: %v", err) } diff --git a/pkg/scheduler/internal/cache/debugger/comparer.go b/pkg/scheduler/internal/cache/debugger/comparer.go index f348977b4f90..5d52c9675f4a 100644 --- a/pkg/scheduler/internal/cache/debugger/comparer.go +++ b/pkg/scheduler/internal/cache/debugger/comparer.go @@ -20,7 +20,7 @@ import ( "sort" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/klog/v2" @@ -38,9 +38,9 @@ type CacheComparer struct { } // Compare compares the nodes and pods of NodeLister with Cache.Snapshot. -func (c *CacheComparer) Compare() error { - klog.V(3).InfoS("Cache comparer started") - defer klog.V(3).InfoS("Cache comparer finished") +func (c *CacheComparer) Compare(logger klog.Logger) error { + logger.V(3).Info("Cache comparer started") + defer logger.V(3).Info("Cache comparer finished") nodes, err := c.NodeLister.List(labels.Everything()) if err != nil { @@ -57,11 +57,11 @@ func (c *CacheComparer) Compare() error { pendingPods := c.PodQueue.PendingPods() if missed, redundant := c.CompareNodes(nodes, dump.Nodes); len(missed)+len(redundant) != 0 { - klog.InfoS("Cache mismatch", "missedNodes", missed, "redundantNodes", redundant) + logger.Info("Cache mismatch", "missedNodes", missed, "redundantNodes", redundant) } if missed, redundant := c.ComparePods(pods, pendingPods, dump.Nodes); len(missed)+len(redundant) != 0 { - klog.InfoS("Cache mismatch", "missedPods", missed, "redundantPods", redundant) + logger.Info("Cache mismatch", "missedPods", missed, "redundantPods", redundant) } return nil diff --git a/pkg/scheduler/internal/cache/debugger/debugger.go b/pkg/scheduler/internal/cache/debugger/debugger.go index d8839ec67e83..8a1f80c88b42 100644 --- a/pkg/scheduler/internal/cache/debugger/debugger.go +++ b/pkg/scheduler/internal/cache/debugger/debugger.go @@ -17,10 +17,12 @@ limitations under the License. package debugger import ( + "context" "os" "os/signal" corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/klog/v2" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" ) @@ -54,7 +56,9 @@ func New( // ListenForSignal starts a goroutine that will trigger the CacheDebugger's // behavior when the process receives SIGINT (Windows) or SIGUSER2 (non-Windows). -func (d *CacheDebugger) ListenForSignal(stopCh <-chan struct{}) { +func (d *CacheDebugger) ListenForSignal(ctx context.Context) { + logger := klog.FromContext(ctx) + stopCh := ctx.Done() ch := make(chan os.Signal, 1) signal.Notify(ch, compareSignal) @@ -64,8 +68,8 @@ func (d *CacheDebugger) ListenForSignal(stopCh <-chan struct{}) { case <-stopCh: return case <-ch: - d.Comparer.Compare() - d.Dumper.DumpAll() + d.Comparer.Compare(logger) + d.Dumper.DumpAll(logger) } } }() diff --git a/pkg/scheduler/internal/cache/debugger/dumper.go b/pkg/scheduler/internal/cache/debugger/dumper.go index b2424b67888c..5fb6239249d6 100644 --- a/pkg/scheduler/internal/cache/debugger/dumper.go +++ b/pkg/scheduler/internal/cache/debugger/dumper.go @@ -22,7 +22,7 @@ import ( "k8s.io/klog/v2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/scheduler/framework" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" "k8s.io/kubernetes/pkg/scheduler/internal/queue" @@ -36,34 +36,34 @@ type CacheDumper struct { } // DumpAll writes cached nodes and scheduling queue information to the scheduler logs. -func (d *CacheDumper) DumpAll() { - d.dumpNodes() - d.dumpSchedulingQueue() +func (d *CacheDumper) DumpAll(logger klog.Logger) { + d.dumpNodes(logger) + d.dumpSchedulingQueue(logger) } // dumpNodes writes NodeInfo to the scheduler logs. -func (d *CacheDumper) dumpNodes() { +func (d *CacheDumper) dumpNodes(logger klog.Logger) { dump := d.cache.Dump() nodeInfos := make([]string, 0, len(dump.Nodes)) for name, nodeInfo := range dump.Nodes { - nodeInfos = append(nodeInfos, d.printNodeInfo(name, nodeInfo)) + nodeInfos = append(nodeInfos, d.printNodeInfo(logger, name, nodeInfo)) } // Extra blank line added between node entries for readability. - klog.InfoS("Dump of cached NodeInfo", "nodes", strings.Join(nodeInfos, "\n\n")) + logger.Info("Dump of cached NodeInfo", "nodes", strings.Join(nodeInfos, "\n\n")) } // dumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs. -func (d *CacheDumper) dumpSchedulingQueue() { +func (d *CacheDumper) dumpSchedulingQueue(logger klog.Logger) { pendingPods := d.podQueue.PendingPods() var podData strings.Builder for _, p := range pendingPods { podData.WriteString(printPod(p)) } - klog.InfoS("Dump of scheduling queue", "pods", podData.String()) + logger.Info("Dump of scheduling queue", "pods", podData.String()) } // printNodeInfo writes parts of NodeInfo to a string. -func (d *CacheDumper) printNodeInfo(name string, n *framework.NodeInfo) string { +func (d *CacheDumper) printNodeInfo(logger klog.Logger, name string, n *framework.NodeInfo) string { var nodeData strings.Builder nodeData.WriteString(fmt.Sprintf("Node name: %s\nDeleted: %t\nRequested Resources: %+v\nAllocatable Resources:%+v\nScheduled Pods(number: %v):\n", name, n.Node() == nil, n.Requested, n.Allocatable, len(n.Pods))) @@ -72,7 +72,7 @@ func (d *CacheDumper) printNodeInfo(name string, n *framework.NodeInfo) string { nodeData.WriteString(printPod(p.Pod)) } // Dumping nominated pods info on the node - nominatedPodInfos := d.podQueue.NominatedPodsForNode(name) + nominatedPodInfos := d.podQueue.NominatedPodsForNode(logger, name) if len(nominatedPodInfos) != 0 { nodeData.WriteString(fmt.Sprintf("Nominated Pods(number: %v):\n", len(nominatedPodInfos))) for _, pi := range nominatedPodInfos { diff --git a/pkg/scheduler/internal/cache/fake/fake_cache.go b/pkg/scheduler/internal/cache/fake/fake_cache.go index 0bcf7e13ae69..1a39f9cef51d 100644 --- a/pkg/scheduler/internal/cache/fake/fake_cache.go +++ b/pkg/scheduler/internal/cache/fake/fake_cache.go @@ -18,6 +18,7 @@ package fake import ( v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/framework" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" ) @@ -31,28 +32,28 @@ type Cache struct { } // AssumePod is a fake method for testing. -func (c *Cache) AssumePod(pod *v1.Pod) error { +func (c *Cache) AssumePod(logger klog.Logger, pod *v1.Pod) error { c.AssumeFunc(pod) return nil } // FinishBinding is a fake method for testing. -func (c *Cache) FinishBinding(pod *v1.Pod) error { return nil } +func (c *Cache) FinishBinding(logger klog.Logger, pod *v1.Pod) error { return nil } // ForgetPod is a fake method for testing. -func (c *Cache) ForgetPod(pod *v1.Pod) error { +func (c *Cache) ForgetPod(logger klog.Logger, pod *v1.Pod) error { c.ForgetFunc(pod) return nil } // AddPod is a fake method for testing. -func (c *Cache) AddPod(pod *v1.Pod) error { return nil } +func (c *Cache) AddPod(logger klog.Logger, pod *v1.Pod) error { return nil } // UpdatePod is a fake method for testing. -func (c *Cache) UpdatePod(oldPod, newPod *v1.Pod) error { return nil } +func (c *Cache) UpdatePod(logger klog.Logger, oldPod, newPod *v1.Pod) error { return nil } // RemovePod is a fake method for testing. -func (c *Cache) RemovePod(pod *v1.Pod) error { return nil } +func (c *Cache) RemovePod(logger klog.Logger, pod *v1.Pod) error { return nil } // IsAssumedPod is a fake method for testing. func (c *Cache) IsAssumedPod(pod *v1.Pod) (bool, error) { @@ -65,16 +66,18 @@ func (c *Cache) GetPod(pod *v1.Pod) (*v1.Pod, error) { } // AddNode is a fake method for testing. -func (c *Cache) AddNode(node *v1.Node) *framework.NodeInfo { return nil } +func (c *Cache) AddNode(logger klog.Logger, node *v1.Node) *framework.NodeInfo { return nil } // UpdateNode is a fake method for testing. -func (c *Cache) UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInfo { return nil } +func (c *Cache) UpdateNode(logger klog.Logger, oldNode, newNode *v1.Node) *framework.NodeInfo { + return nil +} // RemoveNode is a fake method for testing. -func (c *Cache) RemoveNode(node *v1.Node) error { return nil } +func (c *Cache) RemoveNode(logger klog.Logger, node *v1.Node) error { return nil } // UpdateSnapshot is a fake method for testing. -func (c *Cache) UpdateSnapshot(snapshot *internalcache.Snapshot) error { +func (c *Cache) UpdateSnapshot(logger klog.Logger, snapshot *internalcache.Snapshot) error { return nil } diff --git a/pkg/scheduler/internal/cache/interface.go b/pkg/scheduler/internal/cache/interface.go index 4cd6d8f88692..2d09922bcd6a 100644 --- a/pkg/scheduler/internal/cache/interface.go +++ b/pkg/scheduler/internal/cache/interface.go @@ -19,6 +19,7 @@ package cache import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -68,23 +69,23 @@ type Cache interface { // AssumePod assumes a pod scheduled and aggregates the pod's information into its node. // The implementation also decides the policy to expire pod before being confirmed (receiving Add event). // After expiration, its information would be subtracted. - AssumePod(pod *v1.Pod) error + AssumePod(logger klog.Logger, pod *v1.Pod) error // FinishBinding signals that cache for assumed pod can be expired - FinishBinding(pod *v1.Pod) error + FinishBinding(logger klog.Logger, pod *v1.Pod) error // ForgetPod removes an assumed pod from cache. - ForgetPod(pod *v1.Pod) error + ForgetPod(logger klog.Logger, pod *v1.Pod) error // AddPod either confirms a pod if it's assumed, or adds it back if it's expired. // If added back, the pod's information would be added again. - AddPod(pod *v1.Pod) error + AddPod(logger klog.Logger, pod *v1.Pod) error // UpdatePod removes oldPod's information and adds newPod's information. - UpdatePod(oldPod, newPod *v1.Pod) error + UpdatePod(logger klog.Logger, oldPod, newPod *v1.Pod) error // RemovePod removes a pod. The pod's information would be subtracted from assigned node. - RemovePod(pod *v1.Pod) error + RemovePod(logger klog.Logger, pod *v1.Pod) error // GetPod returns the pod from the cache with the same namespace and the // same name of the specified pod. @@ -95,21 +96,21 @@ type Cache interface { // AddNode adds overall information about node. // It returns a clone of added NodeInfo object. - AddNode(node *v1.Node) *framework.NodeInfo + AddNode(logger klog.Logger, node *v1.Node) *framework.NodeInfo // UpdateNode updates overall information about node. // It returns a clone of updated NodeInfo object. - UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInfo + UpdateNode(logger klog.Logger, oldNode, newNode *v1.Node) *framework.NodeInfo // RemoveNode removes overall information about node. - RemoveNode(node *v1.Node) error + RemoveNode(logger klog.Logger, node *v1.Node) error // UpdateSnapshot updates the passed infoSnapshot to the current contents of Cache. // The node info contains aggregated information of pods scheduled (including assumed to be) // on this node. // The snapshot only includes Nodes that are not deleted at the time this function is called. // nodeinfo.Node() is guaranteed to be not nil for all the nodes in the snapshot. - UpdateSnapshot(nodeSnapshot *Snapshot) error + UpdateSnapshot(logger klog.Logger, nodeSnapshot *Snapshot) error // Dump produces a dump of the current cache. Dump() *Dump diff --git a/pkg/scheduler/internal/cache/node_tree.go b/pkg/scheduler/internal/cache/node_tree.go index a0005ddeed50..b56f4d69a649 100644 --- a/pkg/scheduler/internal/cache/node_tree.go +++ b/pkg/scheduler/internal/cache/node_tree.go @@ -36,24 +36,24 @@ type nodeTree struct { } // newNodeTree creates a NodeTree from nodes. -func newNodeTree(nodes []*v1.Node) *nodeTree { +func newNodeTree(logger klog.Logger, nodes []*v1.Node) *nodeTree { nt := &nodeTree{ tree: make(map[string][]string), } for _, n := range nodes { - nt.addNode(n) + nt.addNode(logger, n) } return nt } // addNode adds a node and its corresponding zone to the tree. If the zone already exists, the node // is added to the array of nodes in that zone. -func (nt *nodeTree) addNode(n *v1.Node) { +func (nt *nodeTree) addNode(logger klog.Logger, n *v1.Node) { zone := utilnode.GetZoneKey(n) if na, ok := nt.tree[zone]; ok { for _, nodeName := range na { if nodeName == n.Name { - klog.InfoS("Node already exists in the NodeTree", "node", klog.KObj(n)) + logger.Info("Node already exists in the NodeTree", "node", klog.KObj(n)) return } } @@ -62,12 +62,12 @@ func (nt *nodeTree) addNode(n *v1.Node) { nt.zones = append(nt.zones, zone) nt.tree[zone] = []string{n.Name} } - klog.V(2).InfoS("Added node in listed group to NodeTree", "node", klog.KObj(n), "zone", zone) + logger.V(2).Info("Added node in listed group to NodeTree", "node", klog.KObj(n), "zone", zone) nt.numNodes++ } // removeNode removes a node from the NodeTree. -func (nt *nodeTree) removeNode(n *v1.Node) error { +func (nt *nodeTree) removeNode(logger klog.Logger, n *v1.Node) error { zone := utilnode.GetZoneKey(n) if na, ok := nt.tree[zone]; ok { for i, nodeName := range na { @@ -76,13 +76,13 @@ func (nt *nodeTree) removeNode(n *v1.Node) error { if len(nt.tree[zone]) == 0 { nt.removeZone(zone) } - klog.V(2).InfoS("Removed node in listed group from NodeTree", "node", klog.KObj(n), "zone", zone) + logger.V(2).Info("Removed node in listed group from NodeTree", "node", klog.KObj(n), "zone", zone) nt.numNodes-- return nil } } } - klog.ErrorS(nil, "Node in listed group was not found", "node", klog.KObj(n), "zone", zone) + logger.Error(nil, "Node in listed group was not found", "node", klog.KObj(n), "zone", zone) return fmt.Errorf("node %q in group %q was not found", n.Name, zone) } @@ -99,7 +99,7 @@ func (nt *nodeTree) removeZone(zone string) { } // updateNode updates a node in the NodeTree. -func (nt *nodeTree) updateNode(old, new *v1.Node) { +func (nt *nodeTree) updateNode(logger klog.Logger, old, new *v1.Node) { var oldZone string if old != nil { oldZone = utilnode.GetZoneKey(old) @@ -110,8 +110,8 @@ func (nt *nodeTree) updateNode(old, new *v1.Node) { if oldZone == newZone { return } - nt.removeNode(old) // No error checking. We ignore whether the old node exists or not. - nt.addNode(new) + nt.removeNode(logger, old) // No error checking. We ignore whether the old node exists or not. + nt.addNode(logger, new) } // list returns the list of names of the node. NodeTree iterates over zones and in each zone iterates diff --git a/pkg/scheduler/internal/cache/node_tree_test.go b/pkg/scheduler/internal/cache/node_tree_test.go index a316a0ef4cf7..64749cc70464 100644 --- a/pkg/scheduler/internal/cache/node_tree_test.go +++ b/pkg/scheduler/internal/cache/node_tree_test.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2/ktesting" ) var allNodes = []*v1.Node{ @@ -199,9 +200,10 @@ func TestNodeTree_AddNode(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(nil) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, nil) for _, n := range test.nodesToAdd { - nt.addNode(n) + nt.addNode(logger, n) } verifyNodeTree(t, nt, test.expectedTree) }) @@ -256,9 +258,10 @@ func TestNodeTree_RemoveNode(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(test.existingNodes) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, test.existingNodes) for _, n := range test.nodesToRemove { - err := nt.removeNode(n) + err := nt.removeNode(logger, n) if test.expectError == (err == nil) { t.Errorf("unexpected returned error value: %v", err) } @@ -332,7 +335,8 @@ func TestNodeTree_UpdateNode(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(test.existingNodes) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, test.existingNodes) var oldNode *v1.Node for _, n := range allNodes { if n.Name == test.nodeToUpdate.Name { @@ -343,7 +347,7 @@ func TestNodeTree_UpdateNode(t *testing.T) { if oldNode == nil { oldNode = &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "nonexisting-node"}} } - nt.updateNode(oldNode, test.nodeToUpdate) + nt.updateNode(logger, oldNode, test.nodeToUpdate) verifyNodeTree(t, nt, test.expectedTree) }) } @@ -379,7 +383,8 @@ func TestNodeTree_List(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(test.nodesToAdd) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, test.nodesToAdd) output, err := nt.list() if err != nil { @@ -393,7 +398,8 @@ func TestNodeTree_List(t *testing.T) { } func TestNodeTree_List_Exhausted(t *testing.T) { - nt := newNodeTree(allNodes[:9]) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, allNodes[:9]) nt.numNodes++ _, err := nt.list() if err == nil { @@ -448,7 +454,8 @@ func TestNodeTreeMultiOperations(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(nil) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, nil) addIndex := 0 removeIndex := 0 for _, op := range test.operations { @@ -457,14 +464,14 @@ func TestNodeTreeMultiOperations(t *testing.T) { if addIndex >= len(test.nodesToAdd) { t.Error("more add operations than nodesToAdd") } else { - nt.addNode(test.nodesToAdd[addIndex]) + nt.addNode(logger, test.nodesToAdd[addIndex]) addIndex++ } case "remove": if removeIndex >= len(test.nodesToRemove) { t.Error("more remove operations than nodesToRemove") } else { - nt.removeNode(test.nodesToRemove[removeIndex]) + nt.removeNode(logger, test.nodesToRemove[removeIndex]) removeIndex++ } default: diff --git a/pkg/scheduler/internal/queue/scheduling_queue.go b/pkg/scheduler/internal/queue/scheduling_queue.go index a5a4b30ecadd..edbbb5e420c1 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue.go +++ b/pkg/scheduler/internal/queue/scheduling_queue.go @@ -81,15 +81,15 @@ type PreEnqueueCheck func(pod *v1.Pod) bool // makes it easy to use those data structures as a SchedulingQueue. type SchedulingQueue interface { framework.PodNominator - Add(pod *v1.Pod) error + Add(logger klog.Logger, pod *v1.Pod) error // Activate moves the given pods to activeQ iff they're in unschedulablePods or backoffQ. // The passed-in pods are originally compiled from plugins that want to activate Pods, // by injecting the pods through a reserved CycleState struct (PodsToActivate). - Activate(pods map[string]*v1.Pod) + Activate(logger klog.Logger, pods map[string]*v1.Pod) // AddUnschedulableIfNotPresent adds an unschedulable pod back to scheduling queue. // The podSchedulingCycle represents the current scheduling cycle number which can be // returned by calling SchedulingCycle(). - AddUnschedulableIfNotPresent(pod *framework.QueuedPodInfo, podSchedulingCycle int64) error + AddUnschedulableIfNotPresent(logger klog.Logger, pod *framework.QueuedPodInfo, podSchedulingCycle int64) error // SchedulingCycle returns the current number of scheduling cycle which is // cached by scheduling queue. Normally, incrementing this number whenever // a pod is popped (e.g. called Pop()) is enough. @@ -97,17 +97,17 @@ type SchedulingQueue interface { // Pop removes the head of the queue and returns it. It blocks if the // queue is empty and waits until a new item is added to the queue. Pop() (*framework.QueuedPodInfo, error) - Update(oldPod, newPod *v1.Pod) error - Delete(pod *v1.Pod) error - MoveAllToActiveOrBackoffQueue(event framework.ClusterEvent, preCheck PreEnqueueCheck) - AssignedPodAdded(pod *v1.Pod) - AssignedPodUpdated(pod *v1.Pod) + Update(logger klog.Logger, oldPod, newPod *v1.Pod) error + Delete(logger klog.Logger, pod *v1.Pod) error + MoveAllToActiveOrBackoffQueue(logger klog.Logger, event framework.ClusterEvent, preCheck PreEnqueueCheck) + AssignedPodAdded(logger klog.Logger, pod *v1.Pod) + AssignedPodUpdated(logger klog.Logger, pod *v1.Pod) PendingPods() []*v1.Pod // Close closes the SchedulingQueue so that the goroutine which is // waiting to pop items can exit gracefully. Close() // Run starts the goroutines managing the queue. - Run() + Run(logger klog.Logger) } // NewSchedulingQueue initializes a priority queue as a new scheduling queue. @@ -290,44 +290,48 @@ func NewPriorityQueue( } // Run starts the goroutine to pump from podBackoffQ to activeQ -func (p *PriorityQueue) Run() { - go wait.Until(p.flushBackoffQCompleted, 1.0*time.Second, p.stop) - go wait.Until(p.flushUnschedulablePodsLeftover, 30*time.Second, p.stop) +func (p *PriorityQueue) Run(logger klog.Logger) { + go wait.Until(func() { + p.flushBackoffQCompleted(logger) + }, 1.0*time.Second, p.stop) + go wait.Until(func() { + p.flushUnschedulablePodsLeftover(logger) + }, 30*time.Second, p.stop) } // Add adds a pod to the active queue. It should be called only when a new pod // is added so there is no chance the pod is already in active/unschedulable/backoff queues -func (p *PriorityQueue) Add(pod *v1.Pod) error { +func (p *PriorityQueue) Add(logger klog.Logger, pod *v1.Pod) error { p.lock.Lock() defer p.lock.Unlock() pInfo := p.newQueuedPodInfo(pod) if err := p.activeQ.Add(pInfo); err != nil { - klog.ErrorS(err, "Error adding pod to the active queue", "pod", klog.KObj(pod)) + logger.Error(err, "Error adding pod to the active queue", "pod", klog.KObj(pod)) return err } if p.unschedulablePods.get(pod) != nil { - klog.ErrorS(nil, "Error: pod is already in the unschedulable queue", "pod", klog.KObj(pod)) + logger.Error(nil, "Error: pod is already in the unschedulable queue", "pod", klog.KObj(pod)) p.unschedulablePods.delete(pod) } // Delete pod from backoffQ if it is backing off if err := p.podBackoffQ.Delete(pInfo); err == nil { - klog.ErrorS(nil, "Error: pod is already in the podBackoff queue", "pod", klog.KObj(pod)) + logger.Error(nil, "Error: pod is already in the podBackoff queue", "pod", klog.KObj(pod)) } metrics.SchedulerQueueIncomingPods.WithLabelValues("active", PodAdd).Inc() - p.PodNominator.AddNominatedPod(pInfo.PodInfo, nil) + p.PodNominator.AddNominatedPod(logger, pInfo.PodInfo, nil) p.cond.Broadcast() return nil } // Activate moves the given pods to activeQ iff they're in unschedulablePods or backoffQ. -func (p *PriorityQueue) Activate(pods map[string]*v1.Pod) { +func (p *PriorityQueue) Activate(logger klog.Logger, pods map[string]*v1.Pod) { p.lock.Lock() defer p.lock.Unlock() activated := false for _, pod := range pods { - if p.activate(pod) { + if p.activate(logger, pod) { activated = true } } @@ -337,7 +341,7 @@ func (p *PriorityQueue) Activate(pods map[string]*v1.Pod) { } } -func (p *PriorityQueue) activate(pod *v1.Pod) bool { +func (p *PriorityQueue) activate(logger klog.Logger, pod *v1.Pod) bool { // Verify if the pod is present in activeQ. if _, exists, _ := p.activeQ.Get(newQueuedPodInfoForLookup(pod)); exists { // No need to activate if it's already present in activeQ. @@ -348,7 +352,7 @@ func (p *PriorityQueue) activate(pod *v1.Pod) bool { if pInfo = p.unschedulablePods.get(pod); pInfo == nil { // If the pod doesn't belong to unschedulablePods or backoffQ, don't activate it. if obj, exists, _ := p.podBackoffQ.Get(newQueuedPodInfoForLookup(pod)); !exists { - klog.ErrorS(nil, "To-activate pod does not exist in unschedulablePods or backoffQ", "pod", klog.KObj(pod)) + logger.Error(nil, "To-activate pod does not exist in unschedulablePods or backoffQ", "pod", klog.KObj(pod)) return false } else { pInfo = obj.(*framework.QueuedPodInfo) @@ -357,18 +361,18 @@ func (p *PriorityQueue) activate(pod *v1.Pod) bool { if pInfo == nil { // Redundant safe check. We shouldn't reach here. - klog.ErrorS(nil, "Internal error: cannot obtain pInfo") + logger.Error(nil, "Internal error: cannot obtain pInfo") return false } if err := p.activeQ.Add(pInfo); err != nil { - klog.ErrorS(err, "Error adding pod to the scheduling queue", "pod", klog.KObj(pod)) + logger.Error(err, "Error adding pod to the scheduling queue", "pod", klog.KObj(pod)) return false } p.unschedulablePods.delete(pod) p.podBackoffQ.Delete(pInfo) metrics.SchedulerQueueIncomingPods.WithLabelValues("active", ForceActivate).Inc() - p.PodNominator.AddNominatedPod(pInfo.PodInfo, nil) + p.PodNominator.AddNominatedPod(logger, pInfo.PodInfo, nil) return true } @@ -390,7 +394,7 @@ func (p *PriorityQueue) SchedulingCycle() int64 { // the queue, unless it is already in the queue. Normally, PriorityQueue puts // unschedulable pods in `unschedulablePods`. But if there has been a recent move // request, then the pod is put in `podBackoffQ`. -func (p *PriorityQueue) AddUnschedulableIfNotPresent(pInfo *framework.QueuedPodInfo, podSchedulingCycle int64) error { +func (p *PriorityQueue) AddUnschedulableIfNotPresent(logger klog.Logger, pInfo *framework.QueuedPodInfo, podSchedulingCycle int64) error { p.lock.Lock() defer p.lock.Unlock() pod := pInfo.Pod @@ -424,12 +428,12 @@ func (p *PriorityQueue) AddUnschedulableIfNotPresent(pInfo *framework.QueuedPodI } - p.PodNominator.AddNominatedPod(pInfo.PodInfo, nil) + p.PodNominator.AddNominatedPod(logger, pInfo.PodInfo, nil) return nil } // flushBackoffQCompleted Moves all pods from backoffQ which have completed backoff in to activeQ -func (p *PriorityQueue) flushBackoffQCompleted() { +func (p *PriorityQueue) flushBackoffQCompleted(logger klog.Logger) { p.lock.Lock() defer p.lock.Unlock() activated := false @@ -445,7 +449,7 @@ func (p *PriorityQueue) flushBackoffQCompleted() { } _, err := p.podBackoffQ.Pop() if err != nil { - klog.ErrorS(err, "Unable to pop pod from backoff queue despite backoff completion", "pod", klog.KObj(pod)) + logger.Error(err, "Unable to pop pod from backoff queue despite backoff completion", "pod", klog.KObj(pod)) break } p.activeQ.Add(rawPodInfo) @@ -460,7 +464,7 @@ func (p *PriorityQueue) flushBackoffQCompleted() { // flushUnschedulablePodsLeftover moves pods which stay in unschedulablePods // longer than podMaxInUnschedulablePodsDuration to backoffQ or activeQ. -func (p *PriorityQueue) flushUnschedulablePodsLeftover() { +func (p *PriorityQueue) flushUnschedulablePodsLeftover(logger klog.Logger) { p.lock.Lock() defer p.lock.Unlock() @@ -474,7 +478,7 @@ func (p *PriorityQueue) flushUnschedulablePodsLeftover() { } if len(podsToMove) > 0 { - p.movePodsToActiveOrBackoffQueue(podsToMove, UnschedulableTimeout) + p.movePodsToActiveOrBackoffQueue(logger, podsToMove, UnschedulableTimeout) } } @@ -522,7 +526,7 @@ func isPodUpdated(oldPod, newPod *v1.Pod) bool { // the item from the unschedulable queue if pod is updated in a way that it may // become schedulable and adds the updated one to the active queue. // If pod is not present in any of the queues, it is added to the active queue. -func (p *PriorityQueue) Update(oldPod, newPod *v1.Pod) error { +func (p *PriorityQueue) Update(logger klog.Logger, oldPod, newPod *v1.Pod) error { p.lock.Lock() defer p.lock.Unlock() @@ -531,14 +535,14 @@ func (p *PriorityQueue) Update(oldPod, newPod *v1.Pod) error { // If the pod is already in the active queue, just update it there. if oldPodInfo, exists, _ := p.activeQ.Get(oldPodInfo); exists { pInfo := updatePod(oldPodInfo, newPod) - p.PodNominator.UpdateNominatedPod(oldPod, pInfo.PodInfo) + p.PodNominator.UpdateNominatedPod(logger, oldPod, pInfo.PodInfo) return p.activeQ.Update(pInfo) } // If the pod is in the backoff queue, update it there. if oldPodInfo, exists, _ := p.podBackoffQ.Get(oldPodInfo); exists { pInfo := updatePod(oldPodInfo, newPod) - p.PodNominator.UpdateNominatedPod(oldPod, pInfo.PodInfo) + p.PodNominator.UpdateNominatedPod(logger, oldPod, pInfo.PodInfo) return p.podBackoffQ.Update(pInfo) } } @@ -546,7 +550,7 @@ func (p *PriorityQueue) Update(oldPod, newPod *v1.Pod) error { // If the pod is in the unschedulable queue, updating it may make it schedulable. if usPodInfo := p.unschedulablePods.get(newPod); usPodInfo != nil { pInfo := updatePod(usPodInfo, newPod) - p.PodNominator.UpdateNominatedPod(oldPod, pInfo.PodInfo) + p.PodNominator.UpdateNominatedPod(logger, oldPod, pInfo.PodInfo) if isPodUpdated(oldPod, newPod) { if p.isPodBackingoff(usPodInfo) { if err := p.podBackoffQ.Add(pInfo); err != nil { @@ -572,17 +576,17 @@ func (p *PriorityQueue) Update(oldPod, newPod *v1.Pod) error { if err := p.activeQ.Add(pInfo); err != nil { return err } - p.PodNominator.AddNominatedPod(pInfo.PodInfo, nil) + p.PodNominator.AddNominatedPod(logger, pInfo.PodInfo, nil) p.cond.Broadcast() return nil } // Delete deletes the item from either of the two queues. It assumes the pod is // only in one queue. -func (p *PriorityQueue) Delete(pod *v1.Pod) error { +func (p *PriorityQueue) Delete(logger klog.Logger, pod *v1.Pod) error { p.lock.Lock() defer p.lock.Unlock() - p.PodNominator.DeleteNominatedPodIfExists(pod) + p.PodNominator.DeleteNominatedPodIfExists(logger, pod) if err := p.activeQ.Delete(newQueuedPodInfoForLookup(pod)); err != nil { // The item was probably not found in the activeQ. p.podBackoffQ.Delete(newQueuedPodInfoForLookup(pod)) @@ -593,17 +597,17 @@ func (p *PriorityQueue) Delete(pod *v1.Pod) error { // AssignedPodAdded is called when a bound pod is added. Creation of this pod // may make pending pods with matching affinity terms schedulable. -func (p *PriorityQueue) AssignedPodAdded(pod *v1.Pod) { +func (p *PriorityQueue) AssignedPodAdded(logger klog.Logger, pod *v1.Pod) { p.lock.Lock() - p.movePodsToActiveOrBackoffQueue(p.getUnschedulablePodsWithMatchingAffinityTerm(pod), AssignedPodAdd) + p.movePodsToActiveOrBackoffQueue(logger, p.getUnschedulablePodsWithMatchingAffinityTerm(logger, pod), AssignedPodAdd) p.lock.Unlock() } // AssignedPodUpdated is called when a bound pod is updated. Change of labels // may make pending pods with matching affinity terms schedulable. -func (p *PriorityQueue) AssignedPodUpdated(pod *v1.Pod) { +func (p *PriorityQueue) AssignedPodUpdated(logger klog.Logger, pod *v1.Pod) { p.lock.Lock() - p.movePodsToActiveOrBackoffQueue(p.getUnschedulablePodsWithMatchingAffinityTerm(pod), AssignedPodUpdate) + p.movePodsToActiveOrBackoffQueue(logger, p.getUnschedulablePodsWithMatchingAffinityTerm(logger, pod), AssignedPodUpdate) p.lock.Unlock() } @@ -611,7 +615,7 @@ func (p *PriorityQueue) AssignedPodUpdated(pod *v1.Pod) { // This function adds all pods and then signals the condition variable to ensure that // if Pop() is waiting for an item, it receives the signal after all the pods are in the // queue and the head is the highest priority pod. -func (p *PriorityQueue) MoveAllToActiveOrBackoffQueue(event framework.ClusterEvent, preCheck PreEnqueueCheck) { +func (p *PriorityQueue) MoveAllToActiveOrBackoffQueue(logger klog.Logger, event framework.ClusterEvent, preCheck PreEnqueueCheck) { p.lock.Lock() defer p.lock.Unlock() unschedulablePods := make([]*framework.QueuedPodInfo, 0, len(p.unschedulablePods.podInfoMap)) @@ -620,11 +624,11 @@ func (p *PriorityQueue) MoveAllToActiveOrBackoffQueue(event framework.ClusterEve unschedulablePods = append(unschedulablePods, pInfo) } } - p.movePodsToActiveOrBackoffQueue(unschedulablePods, event) + p.movePodsToActiveOrBackoffQueue(logger, unschedulablePods, event) } // NOTE: this function assumes lock has been acquired in caller -func (p *PriorityQueue) movePodsToActiveOrBackoffQueue(podInfoList []*framework.QueuedPodInfo, event framework.ClusterEvent) { +func (p *PriorityQueue) movePodsToActiveOrBackoffQueue(logger klog.Logger, podInfoList []*framework.QueuedPodInfo, event framework.ClusterEvent) { activated := false for _, pInfo := range podInfoList { // If the event doesn't help making the Pod schedulable, continue. @@ -637,14 +641,14 @@ func (p *PriorityQueue) movePodsToActiveOrBackoffQueue(podInfoList []*framework. pod := pInfo.Pod if p.isPodBackingoff(pInfo) { if err := p.podBackoffQ.Add(pInfo); err != nil { - klog.ErrorS(err, "Error adding pod to the backoff queue", "pod", klog.KObj(pod)) + logger.Error(err, "Error adding pod to the backoff queue", "pod", klog.KObj(pod)) } else { metrics.SchedulerQueueIncomingPods.WithLabelValues("backoff", event.Label).Inc() p.unschedulablePods.delete(pod) } } else { if err := p.activeQ.Add(pInfo); err != nil { - klog.ErrorS(err, "Error adding pod to the scheduling queue", "pod", klog.KObj(pod)) + logger.Error(err, "Error adding pod to the scheduling queue", "pod", klog.KObj(pod)) } else { activated = true metrics.SchedulerQueueIncomingPods.WithLabelValues("active", event.Label).Inc() @@ -661,9 +665,9 @@ func (p *PriorityQueue) movePodsToActiveOrBackoffQueue(podInfoList []*framework. // getUnschedulablePodsWithMatchingAffinityTerm returns unschedulable pods which have // any affinity term that matches "pod". // NOTE: this function assumes lock has been acquired in caller. -func (p *PriorityQueue) getUnschedulablePodsWithMatchingAffinityTerm(pod *v1.Pod) []*framework.QueuedPodInfo { +func (p *PriorityQueue) getUnschedulablePodsWithMatchingAffinityTerm(logger klog.Logger, pod *v1.Pod) []*framework.QueuedPodInfo { var nsLabels labels.Set - nsLabels = interpodaffinity.GetNamespaceLabelsSnapshot(pod.Namespace, p.nsLister) + nsLabels = interpodaffinity.GetNamespaceLabelsSnapshot(logger, pod.Namespace, p.nsLister) var podsToMove []*framework.QueuedPodInfo for _, pInfo := range p.unschedulablePods.podInfoMap { @@ -706,7 +710,7 @@ func (p *PriorityQueue) Close() { } // DeleteNominatedPodIfExists deletes from nominatedPods. -func (npm *nominator) DeleteNominatedPodIfExists(pod *v1.Pod) { +func (npm *nominator) DeleteNominatedPodIfExists(logger klog.Logger, pod *v1.Pod) { npm.Lock() npm.delete(pod) npm.Unlock() @@ -716,15 +720,15 @@ func (npm *nominator) DeleteNominatedPodIfExists(pod *v1.Pod) { // This is called during the preemption process after a node is nominated to run // the pod. We update the structure before sending a request to update the pod // object to avoid races with the following scheduling cycles. -func (npm *nominator) AddNominatedPod(pi *framework.PodInfo, nominatingInfo *framework.NominatingInfo) { +func (npm *nominator) AddNominatedPod(logger klog.Logger, pi *framework.PodInfo, nominatingInfo *framework.NominatingInfo) { npm.Lock() - npm.add(pi, nominatingInfo) + npm.add(logger, pi, nominatingInfo) npm.Unlock() } // NominatedPodsForNode returns a copy of pods that are nominated to run on the given node, // but they are waiting for other pods to be removed from the node. -func (npm *nominator) NominatedPodsForNode(nodeName string) []*framework.PodInfo { +func (npm *nominator) NominatedPodsForNode(logger klog.Logger, nodeName string) []*framework.PodInfo { npm.RLock() defer npm.RUnlock() // Make a copy of the nominated Pods so the caller can mutate safely. @@ -855,7 +859,7 @@ type nominator struct { sync.RWMutex } -func (npm *nominator) add(pi *framework.PodInfo, nominatingInfo *framework.NominatingInfo) { +func (npm *nominator) add(logger klog.Logger, pi *framework.PodInfo, nominatingInfo *framework.NominatingInfo) { // Always delete the pod if it already exists, to ensure we never store more than // one instance of the pod. npm.delete(pi.Pod) @@ -874,11 +878,11 @@ func (npm *nominator) add(pi *framework.PodInfo, nominatingInfo *framework.Nomin // If the pod was removed or if it was already scheduled, don't nominate it. updatedPod, err := npm.podLister.Pods(pi.Pod.Namespace).Get(pi.Pod.Name) if err != nil { - klog.V(4).InfoS("Pod doesn't exist in podLister, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod)) + logger.V(4).Info("Pod doesn't exist in podLister, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod)) return } if updatedPod.Spec.NodeName != "" { - klog.V(4).InfoS("Pod is already scheduled to a node, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod), "node", updatedPod.Spec.NodeName) + logger.V(4).Info("Pod is already scheduled to a node, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod), "node", updatedPod.Spec.NodeName) return } } @@ -886,7 +890,7 @@ func (npm *nominator) add(pi *framework.PodInfo, nominatingInfo *framework.Nomin npm.nominatedPodToNode[pi.Pod.UID] = nodeName for _, npi := range npm.nominatedPods[nodeName] { if npi.Pod.UID == pi.Pod.UID { - klog.V(4).InfoS("Pod already exists in the nominator", "pod", klog.KObj(npi.Pod)) + logger.V(4).Info("Pod already exists in the nominator", "pod", klog.KObj(npi.Pod)) return } } @@ -911,7 +915,7 @@ func (npm *nominator) delete(p *v1.Pod) { } // UpdateNominatedPod updates the with . -func (npm *nominator) UpdateNominatedPod(oldPod *v1.Pod, newPodInfo *framework.PodInfo) { +func (npm *nominator) UpdateNominatedPod(logger klog.Logger, oldPod *v1.Pod, newPodInfo *framework.PodInfo) { npm.Lock() defer npm.Unlock() // In some cases, an Update event with no "NominatedNode" present is received right @@ -934,7 +938,7 @@ func (npm *nominator) UpdateNominatedPod(oldPod *v1.Pod, newPodInfo *framework.P // We update irrespective of the nominatedNodeName changed or not, to ensure // that pod pointer is updated. npm.delete(oldPod) - npm.add(newPodInfo, nominatingInfo) + npm.add(logger, newPodInfo, nominatingInfo) } // NewPodNominator creates a nominator as a backing of framework.PodNominator. @@ -950,17 +954,17 @@ func NewPodNominator(podLister listersv1.PodLister) framework.PodNominator { // MakeNextPodFunc returns a function to retrieve the next pod from a given // scheduling queue -func MakeNextPodFunc(queue SchedulingQueue) func() *framework.QueuedPodInfo { +func MakeNextPodFunc(logger klog.Logger, queue SchedulingQueue) func() *framework.QueuedPodInfo { return func() *framework.QueuedPodInfo { podInfo, err := queue.Pop() if err == nil { - klog.V(4).InfoS("About to try and schedule pod", "pod", klog.KObj(podInfo.Pod)) + logger.V(4).Info("About to try and schedule pod", "pod", klog.KObj(podInfo.Pod)) for plugin := range podInfo.UnschedulablePlugins { metrics.UnschedulableReason(plugin, podInfo.Pod.Spec.SchedulerName).Dec() } return podInfo } - klog.ErrorS(err, "Error while retrieving next pod from scheduling queue") + logger.Error(err, "Error while retrieving next pod from scheduling queue") return nil } } diff --git a/pkg/scheduler/internal/queue/scheduling_queue_test.go b/pkg/scheduler/internal/queue/scheduling_queue_test.go index d1d56980a8e6..8435976c64fb 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue_test.go +++ b/pkg/scheduler/internal/queue/scheduling_queue_test.go @@ -36,6 +36,8 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "k8s.io/component-base/metrics/testutil" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" @@ -88,16 +90,17 @@ func getUnschedulablePod(p *PriorityQueue, pod *v1.Pod) *v1.Pod { func TestPriorityQueue_Add(t *testing.T) { objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - if err := q.Add(medPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, medPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } - if err := q.Add(unschedulablePodInfo.Pod); err != nil { + if err := q.Add(logger, unschedulablePodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } - if err := q.Add(highPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, highPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } expectedNominatedPods := &nominator{ @@ -133,13 +136,14 @@ func newDefaultQueueSort() framework.LessFunc { func TestPriorityQueue_AddWithReversePriorityLessFunc(t *testing.T) { objs := []runtime.Object{medPriorityPodInfo.Pod, highPriorityPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - if err := q.Add(medPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, medPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } - if err := q.Add(highPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, highPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } if p, err := q.Pop(); err != nil || p.Pod != highPriorityPodInfo.Pod { @@ -152,12 +156,13 @@ func TestPriorityQueue_AddWithReversePriorityLessFunc(t *testing.T) { func TestPriorityQueue_AddUnschedulableIfNotPresent(t *testing.T) { objs := []runtime.Object{highPriNominatedPodInfo.Pod, unschedulablePodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - q.Add(highPriNominatedPodInfo.Pod) - q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(highPriNominatedPodInfo.Pod), q.SchedulingCycle()) // Must not add anything. - q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(unschedulablePodInfo.Pod), q.SchedulingCycle()) + q.Add(logger, highPriNominatedPodInfo.Pod) + q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(highPriNominatedPodInfo.Pod), q.SchedulingCycle()) // Must not add anything. + q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(unschedulablePodInfo.Pod), q.SchedulingCycle()) expectedNominatedPods := &nominator{ nominatedPodToNode: map[types.UID]string{ unschedulablePodInfo.Pod.UID: "node1", @@ -186,7 +191,8 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent(t *testing.T) { // Pods in and before current scheduling cycle will be put back to activeQueue // if we were trying to schedule them when we received move request. func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(time.Now()))) totalNum := 10 @@ -196,7 +202,7 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { p := st.MakePod().Name(fmt.Sprintf("pod%d", i)).Namespace(fmt.Sprintf("ns%d", i)).UID(fmt.Sprintf("upns%d", i)).Priority(priority).Obj() expectedPods = append(expectedPods, *p) // priority is to make pods ordered in the PriorityQueue - q.Add(p) + q.Add(logger, p) } // Pop all pods except for the first one @@ -208,7 +214,7 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { } // move all pods to active queue when we were trying to schedule them - q.MoveAllToActiveOrBackoffQueue(TestEvent, nil) + q.MoveAllToActiveOrBackoffQueue(logger, TestEvent, nil) oldCycle := q.SchedulingCycle() firstPod, _ := q.Pop() @@ -229,7 +235,7 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { }, } - if err := q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(unschedulablePod), oldCycle); err != nil { + if err := q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(unschedulablePod), oldCycle); err != nil { t.Errorf("Failed to call AddUnschedulableIfNotPresent(%v): %v", unschedulablePod.Name, err) } } @@ -245,7 +251,8 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { func TestPriorityQueue_Pop(t *testing.T) { objs := []runtime.Object{medPriorityPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) wg := sync.WaitGroup{} @@ -259,17 +266,18 @@ func TestPriorityQueue_Pop(t *testing.T) { t.Errorf("Expected medPriorityPodInfo to be present in nomindatePods: %v", q.PodNominator.(*nominator).nominatedPods["node1"]) } }() - q.Add(medPriorityPodInfo.Pod) + q.Add(logger, medPriorityPodInfo.Pod) wg.Wait() } func TestPriorityQueue_Update(t *testing.T) { objs := []runtime.Object{highPriorityPodInfo.Pod, unschedulablePodInfo.Pod, medPriorityPodInfo.Pod} c := testingclock.NewFakeClock(time.Now()) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs, WithClock(c)) - q.Update(nil, highPriorityPodInfo.Pod) + q.Update(logger, nil, highPriorityPodInfo.Pod) if _, exists, _ := q.activeQ.Get(newQueuedPodInfoForLookup(highPriorityPodInfo.Pod)); !exists { t.Errorf("Expected %v to be added to activeQ.", highPriorityPodInfo.Pod.Name) } @@ -277,7 +285,7 @@ func TestPriorityQueue_Update(t *testing.T) { t.Errorf("Expected nomindatePods to be empty: %v", q.PodNominator) } // Update highPriorityPodInfo and add a nominatedNodeName to it. - q.Update(highPriorityPodInfo.Pod, highPriNominatedPodInfo.Pod) + q.Update(logger, highPriorityPodInfo.Pod, highPriNominatedPodInfo.Pod) if q.activeQ.Len() != 1 { t.Error("Expected only one item in activeQ.") } @@ -286,12 +294,12 @@ func TestPriorityQueue_Update(t *testing.T) { } // Updating an unschedulable pod which is not in any of the two queues, should // add the pod to activeQ. - q.Update(unschedulablePodInfo.Pod, unschedulablePodInfo.Pod) + q.Update(logger, unschedulablePodInfo.Pod, unschedulablePodInfo.Pod) if _, exists, _ := q.activeQ.Get(newQueuedPodInfoForLookup(unschedulablePodInfo.Pod)); !exists { t.Errorf("Expected %v to be added to activeQ.", unschedulablePodInfo.Pod.Name) } // Updating a pod that is already in activeQ, should not change it. - q.Update(unschedulablePodInfo.Pod, unschedulablePodInfo.Pod) + q.Update(logger, unschedulablePodInfo.Pod, unschedulablePodInfo.Pod) if len(q.unschedulablePods.podInfoMap) != 0 { t.Error("Expected unschedulablePods to be empty.") } @@ -308,7 +316,7 @@ func TestPriorityQueue_Update(t *testing.T) { if err := q.podBackoffQ.Add(podInfo); err != nil { t.Errorf("adding pod to backoff queue error: %v", err) } - q.Update(podInfo.Pod, podInfo.Pod) + q.Update(logger, podInfo.Pod, podInfo.Pod) rawPodInfo, err := q.podBackoffQ.Pop() podGotFromBackoffQ := rawPodInfo.(*framework.QueuedPodInfo).Pod if err != nil || podGotFromBackoffQ != medPriorityPodInfo.Pod { @@ -317,13 +325,13 @@ func TestPriorityQueue_Update(t *testing.T) { // updating a pod which is in unschedulable queue, and it is still backing off, // we will move it to backoff queue - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(medPriorityPodInfo.Pod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(medPriorityPodInfo.Pod), q.SchedulingCycle()) if len(q.unschedulablePods.podInfoMap) != 1 { t.Error("Expected unschedulablePods to be 1.") } updatedPod := medPriorityPodInfo.Pod.DeepCopy() updatedPod.Annotations["foo"] = "test" - q.Update(medPriorityPodInfo.Pod, updatedPod) + q.Update(logger, medPriorityPodInfo.Pod, updatedPod) rawPodInfo, err = q.podBackoffQ.Pop() podGotFromBackoffQ = rawPodInfo.(*framework.QueuedPodInfo).Pod if err != nil || podGotFromBackoffQ != updatedPod { @@ -332,7 +340,7 @@ func TestPriorityQueue_Update(t *testing.T) { // updating a pod which is in unschedulable queue, and it is not backing off, // we will move it to active queue - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(medPriorityPodInfo.Pod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(medPriorityPodInfo.Pod), q.SchedulingCycle()) if len(q.unschedulablePods.podInfoMap) != 1 { t.Error("Expected unschedulablePods to be 1.") } @@ -341,7 +349,7 @@ func TestPriorityQueue_Update(t *testing.T) { // Move clock by podInitialBackoffDuration, so that pods in the unschedulablePods would pass the backing off, // and the pods will be moved into activeQ. c.Step(q.podInitialBackoffDuration) - q.Update(medPriorityPodInfo.Pod, updatedPod) + q.Update(logger, medPriorityPodInfo.Pod, updatedPod) if p, err := q.Pop(); err != nil || p.Pod != updatedPod { t.Errorf("Expected: %v after Pop, but got: %v", updatedPod.Name, p.Pod.Name) } @@ -349,12 +357,13 @@ func TestPriorityQueue_Update(t *testing.T) { func TestPriorityQueue_Delete(t *testing.T) { objs := []runtime.Object{highPriorityPodInfo.Pod, unschedulablePodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - q.Update(highPriorityPodInfo.Pod, highPriNominatedPodInfo.Pod) - q.Add(unschedulablePodInfo.Pod) - if err := q.Delete(highPriNominatedPodInfo.Pod); err != nil { + q.Update(logger, highPriorityPodInfo.Pod, highPriNominatedPodInfo.Pod) + q.Add(logger, unschedulablePodInfo.Pod) + if err := q.Delete(logger, highPriNominatedPodInfo.Pod); err != nil { t.Errorf("delete failed: %v", err) } if _, exists, _ := q.activeQ.Get(newQueuedPodInfoForLookup(unschedulablePodInfo.Pod)); !exists { @@ -366,7 +375,7 @@ func TestPriorityQueue_Delete(t *testing.T) { if len(q.PodNominator.(*nominator).nominatedPods) != 1 { t.Errorf("Expected nomindatePods to have only 'unschedulablePodInfo': %v", q.PodNominator.(*nominator).nominatedPods) } - if err := q.Delete(unschedulablePodInfo.Pod); err != nil { + if err := q.Delete(logger, unschedulablePodInfo.Pod); err != nil { t.Errorf("delete failed: %v", err) } if len(q.PodNominator.(*nominator).nominatedPods) != 0 { @@ -411,7 +420,8 @@ func TestPriorityQueue_Activate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var objs []runtime.Object - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) @@ -429,7 +439,7 @@ func TestPriorityQueue_Activate(t *testing.T) { } // Activate specific pod according to the table - q.Activate(map[string]*v1.Pod{"test_pod": tt.qPodInfoToActivate.PodInfo.Pod}) + q.Activate(logger, map[string]*v1.Pod{"test_pod": tt.qPodInfoToActivate.PodInfo.Pod}) // Check the result after activation by the length of activeQ if wantLen := len(tt.want); q.activeQ.Len() != wantLen { @@ -500,6 +510,7 @@ func BenchmarkMoveAllToActiveOrBackoffQueue(b *testing.B) { for _, tt := range tests { for _, podsInUnschedulablePods := range []int{1000, 5000} { b.Run(fmt.Sprintf("%v-%v", tt.name, podsInUnschedulablePods), func(b *testing.B) { + logger, _ := ktesting.NewTestContext(b) for i := 0; i < b.N; i++ { b.StopTimer() c := testingclock.NewFakeClock(time.Now()) @@ -545,15 +556,15 @@ func BenchmarkMoveAllToActiveOrBackoffQueue(b *testing.B) { // Random case. podInfo = q.newQueuedPodInfo(p, plugins[j%len(plugins)]) } - q.AddUnschedulableIfNotPresent(podInfo, q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, podInfo, q.SchedulingCycle()) } b.StartTimer() if tt.moveEvent.Resource != "" { - q.MoveAllToActiveOrBackoffQueue(tt.moveEvent, nil) + q.MoveAllToActiveOrBackoffQueue(logger, tt.moveEvent, nil) } else { // Random case. - q.MoveAllToActiveOrBackoffQueue(events[i%len(events)], nil) + q.MoveAllToActiveOrBackoffQueue(logger, events[i%len(events)], nil) } } }) @@ -566,22 +577,23 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueue(t *testing.T) { m := map[framework.ClusterEvent]sets.String{ {Resource: framework.Node, ActionType: framework.Add}: sets.NewString("fooPlugin"), } - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c), WithClusterEventMap(m)) - q.Add(medPriorityPodInfo.Pod) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) + q.Add(logger, medPriorityPodInfo.Pod) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) // Construct a Pod, but don't associate its scheduler failure to any plugin hpp1 := highPriorityPodInfo.Pod.DeepCopy() hpp1.Name = "hpp1" - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(hpp1), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(hpp1), q.SchedulingCycle()) // Construct another Pod, and associate its scheduler failure to plugin "barPlugin". hpp2 := highPriorityPodInfo.Pod.DeepCopy() hpp2.Name = "hpp2" - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(hpp2, "barPlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(hpp2, "barPlugin"), q.SchedulingCycle()) // Pods is still backing off, move the pod into backoffQ. - q.MoveAllToActiveOrBackoffQueue(NodeAdd, nil) + q.MoveAllToActiveOrBackoffQueue(logger, NodeAdd, nil) if q.activeQ.Len() != 1 { t.Errorf("Expected 1 item to be in activeQ, but got: %v", q.activeQ.Len()) } @@ -596,9 +608,9 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueue(t *testing.T) { } q.schedulingCycle++ - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(hpp1), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(hpp1), q.SchedulingCycle()) for _, pod := range []*v1.Pod{unschedulablePodInfo.Pod, highPriorityPodInfo.Pod, hpp1, hpp2} { if q.unschedulablePods.get(pod) == nil { t.Errorf("Expected %v in the unschedulablePods", pod.Name) @@ -607,7 +619,7 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueue(t *testing.T) { // Move clock by podInitialBackoffDuration, so that pods in the unschedulablePods would pass the backing off, // and the pods will be moved into activeQ. c.Step(q.podInitialBackoffDuration) - q.MoveAllToActiveOrBackoffQueue(NodeAdd, nil) + q.MoveAllToActiveOrBackoffQueue(logger, NodeAdd, nil) // hpp2 won't be moved regardless of its backoff timer. if q.activeQ.Len() != 4 { t.Errorf("Expected 4 items to be in activeQ, but got: %v", q.activeQ.Len()) @@ -621,24 +633,26 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueue(t *testing.T) { // when a pod with pod affinity is in unschedulablePods and another pod with a // matching label is added, the unschedulable pod is moved to activeQ. func TestPriorityQueue_AssignedPodAdded(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + affinityPod := st.MakePod().Name("afp").Namespace("ns1").UID("upns1").Annotation("annot2", "val2").Priority(mediumPriority).NominatedNodeName("node1").PodAffinityExists("service", "region", st.PodAffinityWithRequiredReq).Obj() labelPod := st.MakePod().Name("lbp").Namespace(affinityPod.Namespace).Label("service", "securityscan").Node("node1").Obj() c := testingclock.NewFakeClock(time.Now()) m := map[framework.ClusterEvent]sets.String{AssignedPodAdd: sets.NewString("fakePlugin")} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c), WithClusterEventMap(m)) - q.Add(medPriorityPodInfo.Pod) + q.Add(logger, medPriorityPodInfo.Pod) // Add a couple of pods to the unschedulablePods. - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fakePlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(affinityPod, "fakePlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fakePlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(affinityPod, "fakePlugin"), q.SchedulingCycle()) // Move clock to make the unschedulable pods complete backoff. c.Step(DefaultPodInitialBackoffDuration + time.Second) // Simulate addition of an assigned pod. The pod has matching labels for // affinityPod. So, affinityPod should go to activeQ. - q.AssignedPodAdded(labelPod) + q.AssignedPodAdded(logger, labelPod) if getUnschedulablePod(q, affinityPod) != nil { t.Error("affinityPod is still in the unschedulablePods.") } @@ -653,25 +667,26 @@ func TestPriorityQueue_AssignedPodAdded(t *testing.T) { func TestPriorityQueue_NominatedPodsForNode(t *testing.T) { objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - q.Add(medPriorityPodInfo.Pod) - q.Add(unschedulablePodInfo.Pod) - q.Add(highPriorityPodInfo.Pod) + q.Add(logger, medPriorityPodInfo.Pod) + q.Add(logger, unschedulablePodInfo.Pod) + q.Add(logger, highPriorityPodInfo.Pod) if p, err := q.Pop(); err != nil || p.Pod != highPriorityPodInfo.Pod { t.Errorf("Expected: %v after Pop, but got: %v", highPriorityPodInfo.Pod.Name, p.Pod.Name) } expectedList := []*framework.PodInfo{medPriorityPodInfo, unschedulablePodInfo} - podInfos := q.NominatedPodsForNode("node1") + podInfos := q.NominatedPodsForNode(logger, "node1") if diff := cmp.Diff(expectedList, podInfos); diff != "" { t.Errorf("Unexpected list of nominated Pods for node: (-want, +got):\n%s", diff) } podInfos[0].Pod.Name = "not mpp" - if diff := cmp.Diff(podInfos, q.NominatedPodsForNode("node1")); diff == "" { + if diff := cmp.Diff(podInfos, q.NominatedPodsForNode(logger, "node1")); diff == "" { t.Error("Expected list of nominated Pods for node2 is different from podInfos") } - if len(q.NominatedPodsForNode("node2")) != 0 { + if len(q.NominatedPodsForNode(logger, "node2")) != 0 { t.Error("Expected list of nominated Pods for node2 to be empty.") } } @@ -703,6 +718,7 @@ func TestPriorityQueue_NominatedPodDeleted(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) cs := fake.NewSimpleClientset(tt.podInfo.Pod) informerFactory := informers.NewSharedInformerFactory(cs, 0) podLister := informerFactory.Core().V1().Pods().Lister() @@ -719,9 +735,9 @@ func TestPriorityQueue_NominatedPodDeleted(t *testing.T) { informerFactory.Core().V1().Pods().Informer().GetStore().Delete(tt.podInfo.Pod) } - q.AddNominatedPod(tt.podInfo, nil) + q.AddNominatedPod(logger, tt.podInfo, nil) - if got := len(q.NominatedPodsForNode(tt.podInfo.Pod.Status.NominatedNodeName)) == 1; got != tt.want { + if got := len(q.NominatedPodsForNode(logger, tt.podInfo.Pod.Status.NominatedNodeName)) == 1; got != tt.want { t.Errorf("Want %v, but got %v", tt.want, got) } }) @@ -737,38 +753,40 @@ func TestPriorityQueue_PendingPods(t *testing.T) { return pendingSet } - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort()) - q.Add(medPriorityPodInfo.Pod) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(unschedulablePodInfo.Pod), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPriorityPodInfo.Pod), q.SchedulingCycle()) + q.Add(logger, medPriorityPodInfo.Pod) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(unschedulablePodInfo.Pod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(highPriorityPodInfo.Pod), q.SchedulingCycle()) expectedSet := makeSet([]*v1.Pod{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod}) if !reflect.DeepEqual(expectedSet, makeSet(q.PendingPods())) { t.Error("Unexpected list of pending Pods.") } // Move all to active queue. We should still see the same set of pods. - q.MoveAllToActiveOrBackoffQueue(TestEvent, nil) + q.MoveAllToActiveOrBackoffQueue(logger, TestEvent, nil) if !reflect.DeepEqual(expectedSet, makeSet(q.PendingPods())) { t.Error("Unexpected list of pending Pods...") } } func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { - objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod, scheduledPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() + objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod, scheduledPodInfo.Pod} q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - if err := q.Add(medPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, medPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } // Update unschedulablePodInfo on a different node than specified in the pod. - q.AddNominatedPod(framework.NewPodInfo(unschedulablePodInfo.Pod), + q.AddNominatedPod(logger, framework.NewPodInfo(unschedulablePodInfo.Pod), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node5"}) // Update nominated node name of a pod on a node that is not specified in the pod object. - q.AddNominatedPod(framework.NewPodInfo(highPriorityPodInfo.Pod), + q.AddNominatedPod(logger, framework.NewPodInfo(highPriorityPodInfo.Pod), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node2"}) expectedNominatedPods := &nominator{ nominatedPodToNode: map[types.UID]string{ @@ -794,7 +812,7 @@ func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { } // Update one of the nominated pods that doesn't have nominatedNodeName in the // pod object. It should be updated correctly. - q.AddNominatedPod(highPriorityPodInfo, &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node4"}) + q.AddNominatedPod(logger, highPriorityPodInfo, &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node4"}) expectedNominatedPods = &nominator{ nominatedPodToNode: map[types.UID]string{ medPriorityPodInfo.Pod.UID: "node1", @@ -813,7 +831,7 @@ func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { // Attempt to nominate a pod that was deleted from the informer cache. // Nothing should change. - q.AddNominatedPod(nonExistentPodInfo, &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) + q.AddNominatedPod(logger, nonExistentPodInfo, &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominator{}), cmpopts.IgnoreFields(nominator{}, "podLister", "RWMutex")); diff != "" { t.Errorf("Unexpected diff after nominating a deleted pod (-want, +got):\n%s", diff) } @@ -821,14 +839,14 @@ func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { // Nothing should change. scheduledPodCopy := scheduledPodInfo.Pod.DeepCopy() scheduledPodInfo.Pod.Spec.NodeName = "" - q.AddNominatedPod(framework.NewPodInfo(scheduledPodCopy), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) + q.AddNominatedPod(logger, framework.NewPodInfo(scheduledPodCopy), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominator{}), cmpopts.IgnoreFields(nominator{}, "podLister", "RWMutex")); diff != "" { t.Errorf("Unexpected diff after nominating a scheduled pod (-want, +got):\n%s", diff) } // Delete a nominated pod that doesn't have nominatedNodeName in the pod // object. It should be deleted. - q.DeleteNominatedPodIfExists(highPriorityPodInfo.Pod) + q.DeleteNominatedPodIfExists(logger, highPriorityPodInfo.Pod) expectedNominatedPods = &nominator{ nominatedPodToNode: map[types.UID]string{ medPriorityPodInfo.Pod.UID: "node1", @@ -1001,13 +1019,14 @@ func TestSchedulingQueue_Close(t *testing.T) { // are frequent events that move pods to the active queue. func TestRecentlyTriedPodsGoBack(t *testing.T) { c := testingclock.NewFakeClock(time.Now()) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) // Add a few pods to priority queue. for i := 0; i < 5; i++ { p := st.MakePod().Name(fmt.Sprintf("test-pod-%v", i)).Namespace("ns1").UID(fmt.Sprintf("tp00%v", i)).Priority(highPriority).Node("node1").NominatedNodeName("node1").Obj() - q.Add(p) + q.Add(logger, p) } c.Step(time.Microsecond) // Simulate a pod being popped by the scheduler, determined unschedulable, and @@ -1025,10 +1044,10 @@ func TestRecentlyTriedPodsGoBack(t *testing.T) { LastProbeTime: metav1.Now(), }) // Put in the unschedulable queue. - q.AddUnschedulableIfNotPresent(p1, q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, p1, q.SchedulingCycle()) c.Step(DefaultPodInitialBackoffDuration) // Move all unschedulable pods to the active queue. - q.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + q.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) // Simulation is over. Now let's pop all pods. The pod popped first should be // the last one we pop here. for i := 0; i < 5; i++ { @@ -1048,7 +1067,8 @@ func TestRecentlyTriedPodsGoBack(t *testing.T) { // are frequent events that move pods to the active queue. func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { c := testingclock.NewFakeClock(time.Now()) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) @@ -1066,11 +1086,11 @@ func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { }) // Put in the unschedulable queue - q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(unschedulablePod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(unschedulablePod), q.SchedulingCycle()) // Move clock to make the unschedulable pods complete backoff. c.Step(DefaultPodInitialBackoffDuration + time.Second) // Move all unschedulable pods to the active queue. - q.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + q.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) // Simulate a pod being popped by the scheduler, // At this time, unschedulable pod should be popped. @@ -1085,7 +1105,7 @@ func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { // Assume newer pod was added just after unschedulable pod // being popped and before being pushed back to the queue. newerPod := st.MakePod().Name("test-newer-pod").Namespace("ns1").UID("tp002").CreationTimestamp(metav1.Now()).Priority(highPriority).NominatedNodeName("node1").Obj() - q.Add(newerPod) + q.Add(logger, newerPod) // And then unschedulablePodInfo was determined as unschedulable AGAIN. podutil.UpdatePodCondition(&unschedulablePod.Status, &v1.PodCondition{ @@ -1096,11 +1116,11 @@ func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { }) // And then, put unschedulable pod to the unschedulable queue - q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(unschedulablePod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(unschedulablePod), q.SchedulingCycle()) // Move clock to make the unschedulable pods complete backoff. c.Step(DefaultPodInitialBackoffDuration + time.Second) // Move all unschedulable pods to the active queue. - q.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + q.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) // At this time, newerPod should be popped // because it is the oldest tried pod. @@ -1116,14 +1136,15 @@ func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { // TestHighPriorityBackoff tests that a high priority pod does not block // other pods if it is unschedulable func TestHighPriorityBackoff(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort()) midPod := st.MakePod().Name("test-midpod").Namespace("ns1").UID("tp-mid").Priority(midPriority).NominatedNodeName("node1").Obj() highPod := st.MakePod().Name("test-highpod").Namespace("ns1").UID("tp-high").Priority(highPriority).NominatedNodeName("node1").Obj() - q.Add(midPod) - q.Add(highPod) + q.Add(logger, midPod) + q.Add(logger, highPod) // Simulate a pod being popped by the scheduler, determined unschedulable, and // then moved back to the active queue. p, err := q.Pop() @@ -1141,9 +1162,9 @@ func TestHighPriorityBackoff(t *testing.T) { Message: "fake scheduling failure", }) // Put in the unschedulable queue. - q.AddUnschedulableIfNotPresent(p, q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, p, q.SchedulingCycle()) // Move all unschedulable pods to the active queue. - q.MoveAllToActiveOrBackoffQueue(TestEvent, nil) + q.MoveAllToActiveOrBackoffQueue(logger, TestEvent, nil) p, err = q.Pop() if err != nil { @@ -1161,7 +1182,8 @@ func TestHighPriorityFlushUnschedulablePodsLeftover(t *testing.T) { m := map[framework.ClusterEvent]sets.String{ NodeAdd: sets.NewString("fakePlugin"), } - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c), WithClusterEventMap(m)) midPod := st.MakePod().Name("test-midpod").Namespace("ns1").UID("tp-mid").Priority(midPriority).NominatedNodeName("node1").Obj() @@ -1183,10 +1205,10 @@ func TestHighPriorityFlushUnschedulablePodsLeftover(t *testing.T) { Message: "fake scheduling failure", }) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPod, "fakePlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(midPod, "fakePlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(highPod, "fakePlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(midPod, "fakePlugin"), q.SchedulingCycle()) c.Step(DefaultPodMaxInUnschedulablePodsDuration + time.Second) - q.flushUnschedulablePodsLeftover() + q.flushUnschedulablePodsLeftover(logger) if p, err := q.Pop(); err != nil || p.Pod != highPod { t.Errorf("Expected: %v after Pop, but got: %v", highPriorityPodInfo.Pod.Name, p.Pod.Name) @@ -1242,7 +1264,8 @@ func TestPriorityQueue_initPodMaxInUnschedulablePodsDuration(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() var queue *PriorityQueue if test.podMaxInUnschedulablePodsDuration > 0 { @@ -1257,7 +1280,7 @@ func TestPriorityQueue_initPodMaxInUnschedulablePodsDuration(t *testing.T) { var podInfoList []*framework.QueuedPodInfo for i, op := range test.operations { - op(queue, test.operands[i]) + op(logger, queue, test.operands[i]) } expectedLen := len(test.expected) @@ -1280,25 +1303,25 @@ func TestPriorityQueue_initPodMaxInUnschedulablePodsDuration(t *testing.T) { } } -type operation func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) +type operation func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) var ( - add = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { - queue.Add(pInfo.Pod) + add = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + queue.Add(logger, pInfo.Pod) } - addUnschedulablePodBackToUnschedulablePods = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { - queue.AddUnschedulableIfNotPresent(pInfo, 0) + addUnschedulablePodBackToUnschedulablePods = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + queue.AddUnschedulableIfNotPresent(logger, pInfo, 0) } - addUnschedulablePodBackToBackoffQ = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { - queue.AddUnschedulableIfNotPresent(pInfo, -1) + addUnschedulablePodBackToBackoffQ = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + queue.AddUnschedulableIfNotPresent(logger, pInfo, -1) } - addPodActiveQ = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + addPodActiveQ = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { queue.activeQ.Add(pInfo) } - updatePodActiveQ = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + updatePodActiveQ = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { queue.activeQ.Update(pInfo) } - addPodUnschedulablePods = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + addPodUnschedulablePods = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { // Update pod condition to unschedulable. podutil.UpdatePodCondition(&pInfo.Pod.Status, &v1.PodCondition{ Type: v1.PodScheduled, @@ -1308,22 +1331,22 @@ var ( }) queue.unschedulablePods.addOrUpdate(pInfo) } - addPodBackoffQ = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + addPodBackoffQ = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { queue.podBackoffQ.Add(pInfo) } - moveAllToActiveOrBackoffQ = func(queue *PriorityQueue, _ *framework.QueuedPodInfo) { - queue.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + moveAllToActiveOrBackoffQ = func(logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) { + queue.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) } - flushBackoffQ = func(queue *PriorityQueue, _ *framework.QueuedPodInfo) { + flushBackoffQ = func(logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) { queue.clock.(*testingclock.FakeClock).Step(2 * time.Second) - queue.flushBackoffQCompleted() + queue.flushBackoffQCompleted(logger) } - moveClockForward = func(queue *PriorityQueue, _ *framework.QueuedPodInfo) { + moveClockForward = func(logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) { queue.clock.(*testingclock.FakeClock).Step(2 * time.Second) } - flushUnschedulerQ = func(queue *PriorityQueue, _ *framework.QueuedPodInfo) { + flushUnschedulerQ = func(logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) { queue.clock.(*testingclock.FakeClock).Step(queue.podMaxInUnschedulablePodsDuration) - queue.flushUnschedulablePodsLeftover() + queue.flushUnschedulablePodsLeftover(logger) } ) @@ -1392,13 +1415,14 @@ func TestPodTimestamp(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(timestamp))) var podInfoList []*framework.QueuedPodInfo for i, op := range test.operations { - op(queue, test.operands[i]) + op(logger, queue, test.operands[i]) } expectedLen := len(test.expected) @@ -1556,12 +1580,13 @@ scheduler_pending_pods{queue="unschedulable"} 0 for _, test := range tests { t.Run(test.name, func(t *testing.T) { resetMetrics() - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(timestamp))) for i, op := range test.operations { for _, pInfo := range test.operands[i] { - op(queue, pInfo) + op(logger, queue, pInfo) } } @@ -1581,10 +1606,11 @@ func TestPerPodSchedulingMetrics(t *testing.T) { // Case 1: A pod is created and scheduled after 1 attempt. The queue operations are // Add -> Pop. c := testingclock.NewFakeClock(timestamp) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) - queue.Add(pod) + queue.Add(logger, pod) pInfo, err := queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) @@ -1595,16 +1621,16 @@ func TestPerPodSchedulingMetrics(t *testing.T) { // Add -> Pop -> AddUnschedulableIfNotPresent -> flushUnschedulablePodsLeftover -> Pop. c = testingclock.NewFakeClock(timestamp) queue = NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) - queue.Add(pod) + queue.Add(logger, pod) pInfo, err = queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) } - queue.AddUnschedulableIfNotPresent(pInfo, 1) + queue.AddUnschedulableIfNotPresent(logger, pInfo, 1) // Override clock to exceed the DefaultPodMaxInUnschedulablePodsDuration so that unschedulable pods // will be moved to activeQ c.SetTime(timestamp.Add(DefaultPodMaxInUnschedulablePodsDuration + 1)) - queue.flushUnschedulablePodsLeftover() + queue.flushUnschedulablePodsLeftover(logger) pInfo, err = queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) @@ -1615,19 +1641,19 @@ func TestPerPodSchedulingMetrics(t *testing.T) { // Add -> Pop -> AddUnschedulableIfNotPresent -> flushUnschedulablePodsLeftover -> Update -> Pop. c = testingclock.NewFakeClock(timestamp) queue = NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) - queue.Add(pod) + queue.Add(logger, pod) pInfo, err = queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) } - queue.AddUnschedulableIfNotPresent(pInfo, 1) + queue.AddUnschedulableIfNotPresent(logger, pInfo, 1) // Override clock to exceed the DefaultPodMaxInUnschedulablePodsDuration so that unschedulable pods // will be moved to activeQ c.SetTime(timestamp.Add(DefaultPodMaxInUnschedulablePodsDuration + 1)) - queue.flushUnschedulablePodsLeftover() + queue.flushUnschedulablePodsLeftover(logger) newPod := pod.DeepCopy() newPod.Generation = 1 - queue.Update(pod, newPod) + queue.Update(logger, pod, newPod) pInfo, err = queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) @@ -1707,12 +1733,13 @@ func TestIncomingPodsMetrics(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { metrics.SchedulerQueueIncomingPods.Reset() - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(timestamp))) for _, op := range test.operations { for _, pInfo := range pInfos { - op(queue, pInfo) + op(logger, queue, pInfo) } } metricName := metrics.SchedulerSubsystem + "_" + metrics.SchedulerQueueIncomingPods.Name @@ -1735,7 +1762,8 @@ func checkPerPodSchedulingMetrics(name string, t *testing.T, pInfo *framework.Qu func TestBackOffFlow(t *testing.T) { cl := testingclock.NewFakeClock(time.Now()) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(cl)) steps := []struct { @@ -1755,7 +1783,7 @@ func TestBackOffFlow(t *testing.T) { Namespace: pod.Namespace, Name: pod.Name, } - if err := q.Add(pod); err != nil { + if err := q.Add(logger, pod); err != nil { t.Fatal(err) } @@ -1770,12 +1798,12 @@ func TestBackOffFlow(t *testing.T) { if podInfo.Attempts != i+1 { t.Errorf("got attempts %d, want %d", podInfo.Attempts, i+1) } - if err := q.AddUnschedulableIfNotPresent(podInfo, int64(i)); err != nil { + if err := q.AddUnschedulableIfNotPresent(logger, podInfo, int64(i)); err != nil { t.Fatal(err) } // An event happens. - q.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + q.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) if _, ok, _ := q.podBackoffQ.Get(podInfo); !ok { t.Errorf("pod %v is not in the backoff queue", podID) @@ -1790,14 +1818,14 @@ func TestBackOffFlow(t *testing.T) { // Simulate routine that continuously flushes the backoff queue. cl.Step(time.Millisecond) - q.flushBackoffQCompleted() + q.flushBackoffQCompleted(logger) // Still in backoff queue after an early flush. if _, ok, _ := q.podBackoffQ.Get(podInfo); !ok { t.Errorf("pod %v is not in the backoff queue", podID) } // Moved out of the backoff queue after timeout. cl.Step(backoff) - q.flushBackoffQCompleted() + q.flushBackoffQCompleted(logger) if _, ok, _ := q.podBackoffQ.Get(podInfo); ok { t.Errorf("pod %v is still in the backoff queue", podID) } @@ -1946,13 +1974,14 @@ func TestMoveAllToActiveOrBackoffQueue_PreEnqueueChecks(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort()) for _, podInfo := range tt.podInfos { - q.AddUnschedulableIfNotPresent(podInfo, q.schedulingCycle) + q.AddUnschedulableIfNotPresent(logger, podInfo, q.schedulingCycle) } - q.MoveAllToActiveOrBackoffQueue(TestEvent, tt.preEnqueueCheck) + q.MoveAllToActiveOrBackoffQueue(logger, TestEvent, tt.preEnqueueCheck) var got []string for q.podBackoffQ.Len() != 0 { obj, err := q.podBackoffQ.Pop() diff --git a/pkg/scheduler/schedule_one.go b/pkg/scheduler/schedule_one.go index dbb739778f6c..cdbf9c703afa 100644 --- a/pkg/scheduler/schedule_one.go +++ b/pkg/scheduler/schedule_one.go @@ -64,6 +64,7 @@ var clearNominatedNode = &framework.NominatingInfo{NominatingMode: framework.Mod // scheduleOne does the entire scheduling workflow for a single pod. It is serialized on the scheduling algorithm's host fitting. func (sched *Scheduler) scheduleOne(ctx context.Context) { + logger := klog.FromContext(ctx) podInfo := sched.NextPod() // pod could be nil when schedulerQueue is closed if podInfo == nil || podInfo.Pod == nil { @@ -74,14 +75,14 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { if err != nil { // This shouldn't happen, because we only accept for scheduling the pods // which specify a scheduler name that matches one of the profiles. - klog.ErrorS(err, "Error occurred") + logger.Error(err, "Error occurred") return } - if sched.skipPodSchedule(fwk, pod) { + if sched.skipPodSchedule(ctx, fwk, pod) { return } - klog.V(3).InfoS("Attempting to schedule pod", "pod", klog.KObj(pod)) + logger.V(3).Info("Attempting to schedule pod", "pod", klog.KObj(pod)) // Synchronously attempt to find a fit for the pod. start := time.Now() @@ -102,15 +103,15 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { var nominatingInfo *framework.NominatingInfo if fitError, ok := err.(*framework.FitError); ok { if !fwk.HasPostFilterPlugins() { - klog.V(3).InfoS("No PostFilter plugins are registered, so no preemption will be performed") + logger.V(3).Info("No PostFilter plugins are registered, so no preemption will be performed") } else { // Run PostFilter plugins to try to make the pod schedulable in a future scheduling cycle. result, status := fwk.RunPostFilterPlugins(ctx, state, pod, fitError.Diagnosis.NodeToStatusMap) if status.Code() == framework.Error { - klog.ErrorS(nil, "Status after running PostFilter plugins for pod", "pod", klog.KObj(pod), "status", status) + logger.Error(nil, "Status after running PostFilter plugins for pod", "pod", klog.KObj(pod), "status", status) } else { fitError.Diagnosis.PostFilterMsg = status.Message() - klog.V(5).InfoS("Status after running PostFilter plugins for pod", "pod", klog.KObj(pod), "status", status) + logger.V(5).Info("Status after running PostFilter plugins for pod", "pod", klog.KObj(pod), "status", status) } if result != nil { nominatingInfo = result.NominatingInfo @@ -126,7 +127,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { metrics.PodUnschedulable(fwk.ProfileName(), metrics.SinceInSeconds(start)) } else { nominatingInfo = clearNominatedNode - klog.ErrorS(err, "Error selecting node for pod", "pod", klog.KObj(pod)) + logger.Error(err, "Error selecting node for pod", "pod", klog.KObj(pod)) metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) } sched.handleSchedulingFailure(ctx, fwk, podInfo, err, v1.PodReasonUnschedulable, nominatingInfo) @@ -138,7 +139,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { assumedPodInfo := podInfo.DeepCopy() assumedPod := assumedPodInfo.Pod // assume modifies `assumedPod` by setting NodeName=scheduleResult.SuggestedHost - err = sched.assume(assumedPod, scheduleResult.SuggestedHost) + err = sched.assume(logger, assumedPod, scheduleResult.SuggestedHost) if err != nil { metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) // This is most probably result of a BUG in retrying logic. @@ -155,8 +156,8 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) // trigger un-reserve to clean up state associated with the reserved Pod fwk.RunReservePluginsUnreserve(schedulingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if forgetErr := sched.Cache.ForgetPod(assumedPod); forgetErr != nil { - klog.ErrorS(forgetErr, "Scheduler cache ForgetPod failed") + if forgetErr := sched.Cache.ForgetPod(logger, assumedPod); forgetErr != nil { + logger.Error(forgetErr, "Scheduler cache ForgetPod failed") } sched.handleSchedulingFailure(ctx, fwk, assumedPodInfo, sts.AsError(), SchedulerError, clearNominatedNode) return @@ -175,8 +176,8 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { } // One of the plugins returned status different than success or wait. fwk.RunReservePluginsUnreserve(schedulingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if forgetErr := sched.Cache.ForgetPod(assumedPod); forgetErr != nil { - klog.ErrorS(forgetErr, "Scheduler cache ForgetPod failed") + if forgetErr := sched.Cache.ForgetPod(logger, assumedPod); forgetErr != nil { + logger.Error(forgetErr, "Scheduler cache ForgetPod failed") } sched.handleSchedulingFailure(ctx, fwk, assumedPodInfo, runPermitStatus.AsError(), reason, clearNominatedNode) return @@ -184,7 +185,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { // At the end of a successful scheduling cycle, pop and move up Pods if needed. if len(podsToActivate.Map) != 0 { - sched.SchedulingQueue.Activate(podsToActivate.Map) + sched.SchedulingQueue.Activate(logger, podsToActivate.Map) // Clear the entries after activation. podsToActivate.Map = make(map[string]*v1.Pod) } @@ -208,8 +209,8 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { } // trigger un-reserve plugins to clean up state associated with the reserved Pod fwk.RunReservePluginsUnreserve(bindingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if forgetErr := sched.Cache.ForgetPod(assumedPod); forgetErr != nil { - klog.ErrorS(forgetErr, "scheduler cache ForgetPod failed") + if forgetErr := sched.Cache.ForgetPod(logger, assumedPod); forgetErr != nil { + logger.Error(forgetErr, "scheduler cache ForgetPod failed") } else { // "Forget"ing an assumed Pod in binding cycle should be treated as a PodDelete event, // as the assumed Pod had occupied a certain amount of resources in scheduler cache. @@ -217,7 +218,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { // Avoid moving the assumed Pod itself as it's always Unschedulable. // It's intentional to "defer" this operation; otherwise MoveAllToActiveOrBackoffQueue() would // update `q.moveRequest` and thus move the assumed pod to backoffQ anyways. - defer sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(internalqueue.AssignedPodDelete, func(pod *v1.Pod) bool { + defer sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, internalqueue.AssignedPodDelete, func(pod *v1.Pod) bool { return assumedPod.UID != pod.UID }) } @@ -231,13 +232,13 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) // trigger un-reserve plugins to clean up state associated with the reserved Pod fwk.RunReservePluginsUnreserve(bindingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if forgetErr := sched.Cache.ForgetPod(assumedPod); forgetErr != nil { - klog.ErrorS(forgetErr, "scheduler cache ForgetPod failed") + if forgetErr := sched.Cache.ForgetPod(logger, assumedPod); forgetErr != nil { + logger.Error(forgetErr, "scheduler cache ForgetPod failed") } else { // "Forget"ing an assumed Pod in binding cycle should be treated as a PodDelete event, // as the assumed Pod had occupied a certain amount of resources in scheduler cache. // TODO(#103853): de-duplicate the logic. - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(internalqueue.AssignedPodDelete, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, internalqueue.AssignedPodDelete, nil) } sched.handleSchedulingFailure(ctx, fwk, assumedPodInfo, preBindStatus.AsError(), SchedulerError, clearNominatedNode) return @@ -248,19 +249,19 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) // trigger un-reserve plugins to clean up state associated with the reserved Pod fwk.RunReservePluginsUnreserve(bindingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if err := sched.Cache.ForgetPod(assumedPod); err != nil { - klog.ErrorS(err, "scheduler cache ForgetPod failed") + if err := sched.Cache.ForgetPod(logger, assumedPod); err != nil { + logger.Error(err, "scheduler cache ForgetPod failed") } else { // "Forget"ing an assumed Pod in binding cycle should be treated as a PodDelete event, // as the assumed Pod had occupied a certain amount of resources in scheduler cache. // TODO(#103853): de-duplicate the logic. - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(internalqueue.AssignedPodDelete, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, internalqueue.AssignedPodDelete, nil) } sched.handleSchedulingFailure(ctx, fwk, assumedPodInfo, fmt.Errorf("binding rejected: %w", err), SchedulerError, clearNominatedNode) return } // Calculating nodeResourceString can be heavy. Avoid it if klog verbosity is below 2. - klog.V(2).InfoS("Successfully bound pod to node", "pod", klog.KObj(pod), "node", scheduleResult.SuggestedHost, "evaluatedNodes", scheduleResult.EvaluatedNodes, "feasibleNodes", scheduleResult.FeasibleNodes) + logger.V(2).Info("Successfully bound pod to node", "pod", klog.KObj(pod), "node", scheduleResult.SuggestedHost, "evaluatedNodes", scheduleResult.EvaluatedNodes, "feasibleNodes", scheduleResult.FeasibleNodes) metrics.PodScheduled(fwk.ProfileName(), metrics.SinceInSeconds(start)) metrics.PodSchedulingAttempts.Observe(float64(podInfo.Attempts)) metrics.PodSchedulingDuration.WithLabelValues(getAttemptsLabel(podInfo)).Observe(metrics.SinceInSeconds(podInfo.InitialAttemptTimestamp)) @@ -270,7 +271,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { // At the end of a successful binding cycle, move up Pods if needed. if len(podsToActivate.Map) != 0 { - sched.SchedulingQueue.Activate(podsToActivate.Map) + sched.SchedulingQueue.Activate(logger, podsToActivate.Map) // Unlike the logic in scheduling cycle, we don't bother deleting the entries // as `podsToActivate.Map` is no longer consumed. } @@ -286,11 +287,11 @@ func (sched *Scheduler) frameworkForPod(pod *v1.Pod) (framework.Framework, error } // skipPodSchedule returns true if we could skip scheduling the pod for specified cases. -func (sched *Scheduler) skipPodSchedule(fwk framework.Framework, pod *v1.Pod) bool { +func (sched *Scheduler) skipPodSchedule(ctx context.Context, fwk framework.Framework, pod *v1.Pod) bool { // Case 1: pod is being deleted. if pod.DeletionTimestamp != nil { fwk.EventRecorder().Eventf(pod, nil, v1.EventTypeWarning, "FailedScheduling", "Scheduling", "skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name) - klog.V(3).InfoS("Skip schedule deleting pod", "pod", klog.KObj(pod)) + klog.FromContext(ctx).V(3).Info("Skip schedule deleting pod", "pod", klog.KObj(pod)) return true } @@ -299,6 +300,7 @@ func (sched *Scheduler) skipPodSchedule(fwk framework.Framework, pod *v1.Pod) bo // during its previous scheduling cycle but before getting assumed. isAssumed, err := sched.Cache.IsAssumedPod(pod) if err != nil { + // TODO: pass ctx into a revised HandleError utilruntime.HandleError(fmt.Errorf("failed to check whether pod %s/%s is assumed: %v", pod.Namespace, pod.Name, err)) return false } @@ -311,8 +313,9 @@ func (sched *Scheduler) skipPodSchedule(fwk framework.Framework, pod *v1.Pod) bo func (sched *Scheduler) schedulePod(ctx context.Context, fwk framework.Framework, state *framework.CycleState, pod *v1.Pod) (result ScheduleResult, err error) { trace := utiltrace.New("Scheduling", utiltrace.Field{Key: "namespace", Value: pod.Namespace}, utiltrace.Field{Key: "name", Value: pod.Name}) defer trace.LogIfLong(100 * time.Millisecond) + logger := klog.FromContext(ctx) - if err := sched.Cache.UpdateSnapshot(sched.nodeInfoSnapshot); err != nil { + if err := sched.Cache.UpdateSnapshot(logger, sched.nodeInfoSnapshot); err != nil { return result, err } trace.Step("Snapshotting scheduler cache and node infos done") @@ -362,6 +365,7 @@ func (sched *Scheduler) schedulePod(ctx context.Context, fwk framework.Framework // Filters the nodes to find the ones that fit the pod based on the framework // filter plugins and filter extenders. func (sched *Scheduler) findNodesThatFitPod(ctx context.Context, fwk framework.Framework, state *framework.CycleState, pod *v1.Pod) ([]*v1.Node, framework.Diagnosis, error) { + logger := klog.FromContext(ctx) diagnosis := framework.Diagnosis{ NodeToStatusMap: make(framework.NodeToStatusMap), UnschedulablePlugins: sets.NewString(), @@ -394,7 +398,7 @@ func (sched *Scheduler) findNodesThatFitPod(ctx context.Context, fwk framework.F if len(pod.Status.NominatedNodeName) > 0 { feasibleNodes, err := sched.evaluateNominatedNode(ctx, pod, fwk, state, diagnosis) if err != nil { - klog.ErrorS(err, "Evaluation failed on nominated node", "pod", klog.KObj(pod), "node", pod.Status.NominatedNodeName) + logger.Error(err, "Evaluation failed on nominated node", "pod", klog.KObj(pod), "node", pod.Status.NominatedNodeName) } // Nominated node passes all the filters, scheduler is good to assign this node to the pod. if len(feasibleNodes) != 0 { @@ -418,7 +422,7 @@ func (sched *Scheduler) findNodesThatFitPod(ctx context.Context, fwk framework.F return nil, diagnosis, err } - feasibleNodes, err = findNodesThatPassExtenders(sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) + feasibleNodes, err = findNodesThatPassExtenders(ctx, sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) if err != nil { return nil, diagnosis, err } @@ -437,7 +441,7 @@ func (sched *Scheduler) evaluateNominatedNode(ctx context.Context, pod *v1.Pod, return nil, err } - feasibleNodes, err = findNodesThatPassExtenders(sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) + feasibleNodes, err = findNodesThatPassExtenders(ctx, sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) if err != nil { return nil, err } @@ -544,7 +548,8 @@ func (sched *Scheduler) numFeasibleNodesToFind(numAllNodes int32) (numNodes int3 return numNodes } -func findNodesThatPassExtenders(extenders []framework.Extender, pod *v1.Pod, feasibleNodes []*v1.Node, statuses framework.NodeToStatusMap) ([]*v1.Node, error) { +func findNodesThatPassExtenders(ctx context.Context, extenders []framework.Extender, pod *v1.Pod, feasibleNodes []*v1.Node, statuses framework.NodeToStatusMap) ([]*v1.Node, error) { + logger := klog.FromContext(ctx) // Extenders are called sequentially. // Nodes in original feasibleNodes can be excluded in one extender, and pass on to the next // extender in a decreasing manner. @@ -564,7 +569,7 @@ func findNodesThatPassExtenders(extenders []framework.Extender, pod *v1.Pod, fea feasibleList, failedMap, failedAndUnresolvableMap, err := extender.Filter(pod, feasibleNodes) if err != nil { if extender.IsIgnorable() { - klog.InfoS("Skipping extender as it returned error and has ignorable flag set", "extender", extender, "err", err) + logger.Info("Skipping extender as it returned error and has ignorable flag set", "extender", extender, "err", err) continue } return nil, err @@ -636,11 +641,12 @@ func prioritizeNodes( } // Additional details logged at level 10 if enabled. - klogV := klog.V(10) - if klogV.Enabled() { + logger := klog.FromContext(ctx) + loggerV := logger.V(10) + if loggerV.Enabled() { for plugin, nodeScoreList := range scoresMap { for _, nodeScore := range nodeScoreList { - klogV.InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", plugin, "node", nodeScore.Name, "score", nodeScore.Score) + loggerV.Info("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", plugin, "node", nodeScore.Name, "score", nodeScore.Score) } } } @@ -673,14 +679,14 @@ func prioritizeNodes( prioritizedList, weight, err := extenders[extIndex].Prioritize(pod, nodes) if err != nil { // Prioritization errors from extender can be ignored, let k8s/other extenders determine the priorities - klog.V(5).InfoS("Failed to run extender's priority function. No score given by this extender.", "error", err, "pod", klog.KObj(pod), "extender", extenders[extIndex].Name()) + logger.V(5).Info("Failed to run extender's priority function. No score given by this extender.", "error", err, "pod", klog.KObj(pod), "extender", extenders[extIndex].Name()) return } mu.Lock() for i := range *prioritizedList { host, score := (*prioritizedList)[i].Host, (*prioritizedList)[i].Score - if klogV.Enabled() { - klogV.InfoS("Extender scored node for pod", "pod", klog.KObj(pod), "extender", extenders[extIndex].Name(), "node", host, "score", score) + if loggerV.Enabled() { + loggerV.Info("Extender scored node for pod", "pod", klog.KObj(pod), "extender", extenders[extIndex].Name(), "node", host, "score", score) } combinedScores[host] += score * weight } @@ -696,9 +702,9 @@ func prioritizeNodes( } } - if klogV.Enabled() { + if loggerV.Enabled() { for i := range result { - klogV.InfoS("Calculated node's final score for pod", "pod", klog.KObj(pod), "node", result[i].Name, "score", result[i].Score) + loggerV.Info("Calculated node's final score for pod", "pod", klog.KObj(pod), "node", result[i].Name, "score", result[i].Score) } } return result, nil @@ -731,20 +737,20 @@ func selectHost(nodeScoreList framework.NodeScoreList) (string, error) { // assume signals to the cache that a pod is already in the cache, so that binding can be asynchronous. // assume modifies `assumed`. -func (sched *Scheduler) assume(assumed *v1.Pod, host string) error { +func (sched *Scheduler) assume(logger klog.Logger, assumed *v1.Pod, host string) error { // Optimistically assume that the binding will succeed and send it to apiserver // in the background. // If the binding fails, scheduler will release resources allocated to assumed pod // immediately. assumed.Spec.NodeName = host - if err := sched.Cache.AssumePod(assumed); err != nil { - klog.ErrorS(err, "Scheduler cache AssumePod failed") + if err := sched.Cache.AssumePod(logger, assumed); err != nil { + logger.Error(err, "Scheduler cache AssumePod failed") return err } // if "assumed" is a nominated pod, we should remove it from internal cache if sched.SchedulingQueue != nil { - sched.SchedulingQueue.DeleteNominatedPodIfExists(assumed) + sched.SchedulingQueue.DeleteNominatedPodIfExists(logger, assumed) } return nil @@ -754,8 +760,9 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error { // The precedence for binding is: (1) extenders and (2) framework plugins. // We expect this to run asynchronously, so we handle binding metrics internally. func (sched *Scheduler) bind(ctx context.Context, fwk framework.Framework, assumed *v1.Pod, targetNode string, state *framework.CycleState) (err error) { + logger := klog.FromContext(ctx) defer func() { - sched.finishBinding(fwk, assumed, targetNode, err) + sched.finishBinding(logger, fwk, assumed, targetNode, err) }() bound, err := sched.extendersBinding(assumed, targetNode) @@ -786,12 +793,12 @@ func (sched *Scheduler) extendersBinding(pod *v1.Pod, node string) (bool, error) return false, nil } -func (sched *Scheduler) finishBinding(fwk framework.Framework, assumed *v1.Pod, targetNode string, err error) { - if finErr := sched.Cache.FinishBinding(assumed); finErr != nil { - klog.ErrorS(finErr, "Scheduler cache FinishBinding failed") +func (sched *Scheduler) finishBinding(logger klog.Logger, fwk framework.Framework, assumed *v1.Pod, targetNode string, err error) { + if finErr := sched.Cache.FinishBinding(logger, assumed); finErr != nil { + logger.Error(finErr, "Scheduler cache FinishBinding failed") } if err != nil { - klog.V(1).InfoS("Failed to bind pod", "pod", klog.KObj(assumed)) + logger.V(1).Info("Failed to bind pod", "pod", klog.KObj(assumed)) return } @@ -810,14 +817,15 @@ func getAttemptsLabel(p *framework.QueuedPodInfo) string { // handleSchedulingFailure records an event for the pod that indicates the // pod has failed to schedule. Also, update the pod condition and nominated node name if set. func (sched *Scheduler) handleSchedulingFailure(ctx context.Context, fwk framework.Framework, podInfo *framework.QueuedPodInfo, err error, reason string, nominatingInfo *framework.NominatingInfo) { - sched.Error(podInfo, err) + sched.Error(ctx, podInfo, err) // Update the scheduling queue with the nominated pod information. Without // this, there would be a race condition between the next scheduling cycle // and the time the scheduler receives a Pod Update for the nominated pod. // Here we check for nil only for tests. if sched.SchedulingQueue != nil { - sched.SchedulingQueue.AddNominatedPod(podInfo.PodInfo, nominatingInfo) + logger := klog.FromContext(ctx) + sched.SchedulingQueue.AddNominatedPod(logger, podInfo.PodInfo, nominatingInfo) } pod := podInfo.Pod @@ -829,7 +837,7 @@ func (sched *Scheduler) handleSchedulingFailure(ctx context.Context, fwk framewo Reason: reason, Message: err.Error(), }, nominatingInfo); err != nil { - klog.ErrorS(err, "Error updating pod", "pod", klog.KObj(pod)) + klog.FromContext(ctx).Error(err, "Error updating pod", "pod", klog.KObj(pod)) } } @@ -844,7 +852,8 @@ func truncateMessage(message string) string { } func updatePod(ctx context.Context, client clientset.Interface, pod *v1.Pod, condition *v1.PodCondition, nominatingInfo *framework.NominatingInfo) error { - klog.V(3).InfoS("Updating pod condition", "pod", klog.KObj(pod), "conditionType", condition.Type, "conditionStatus", condition.Status, "conditionReason", condition.Reason) + logger := klog.FromContext(ctx) + logger.V(3).Info("Updating pod condition", "pod", klog.KObj(pod), "conditionType", condition.Type, "conditionStatus", condition.Status, "conditionReason", condition.Reason) podStatusCopy := pod.Status.DeepCopy() // NominatedNodeName is updated only if we are trying to set it, and the value is // different from the existing one. diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go index c563e6c89524..48f0e69f26b5 100644 --- a/pkg/scheduler/schedule_one_test.go +++ b/pkg/scheduler/schedule_one_test.go @@ -44,6 +44,8 @@ import ( clientcache "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" "k8s.io/component-helpers/storage/volume" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" extenderv1 "k8s.io/kube-scheduler/extender/v1" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" @@ -96,6 +98,7 @@ func (f *fakeExtender) IsIgnorable() bool { } func (f *fakeExtender) ProcessPreemption( + _ klog.Logger, _ *v1.Pod, _ map[string]*extenderv1.Victims, _ framework.NodeInfoLister, @@ -355,11 +358,11 @@ func TestSchedulerMultipleProfilesScheduling(t *testing.T) { informerFactory := informers.NewSharedInformerFactory(client, 0) sched, err := New( + ctx, client, informerFactory, nil, profile.NewRecorderFactory(broadcaster), - ctx.Done(), WithProfiles( schedulerapi.KubeSchedulerProfile{SchedulerName: "match-node2", Plugins: &schedulerapi.Plugins{ @@ -542,6 +545,10 @@ func TestSchedulerScheduleOne(t *testing.T) { for _, item := range table { t.Run(item.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var gotError error var gotPod *v1.Pod var gotForgetPod *v1.Pod @@ -581,16 +588,14 @@ func TestSchedulerScheduleOne(t *testing.T) { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s := newScheduler( + logger, cache, nil, func() *framework.QueuedPodInfo { return &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(item.sendPod)} }, - func(p *framework.QueuedPodInfo, err error) { + func(_ context.Context, p *framework.QueuedPodInfo, err error) { gotPod = p.Pod gotError = err }, @@ -636,13 +641,14 @@ func TestSchedulerScheduleOne(t *testing.T) { } func TestSchedulerNoPhantomPodAfterExpire(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queuedPodStore := clientcache.NewFIFO(clientcache.MetaNamespaceKeyFunc) - scache := internalcache.New(100*time.Millisecond, ctx.Done()) + scache := internalcache.New(ctx, 100*time.Millisecond) pod := podWithPort("pod.Name", "", 8080) node := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1", UID: types.UID("node1")}} - scache.AddNode(&node) + scache.AddNode(logger, &node) fns := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), @@ -701,13 +707,14 @@ func TestSchedulerNoPhantomPodAfterExpire(t *testing.T) { } func TestSchedulerNoPhantomPodAfterDelete(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queuedPodStore := clientcache.NewFIFO(clientcache.MetaNamespaceKeyFunc) - scache := internalcache.New(10*time.Minute, ctx.Done()) + scache := internalcache.New(ctx, 10*time.Minute) firstPod := podWithPort("pod.Name", "", 8080) node := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1", UID: types.UID("node1")}} - scache.AddNode(&node) + scache.AddNode(logger, &node) fns := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), @@ -746,10 +753,10 @@ func TestSchedulerNoPhantomPodAfterDelete(t *testing.T) { // and would be removed itself (without any explicit actions on schedulernodeinfo). Even in that case, // explicitly AddPod will as well correct the behavior. firstPod.Spec.NodeName = node.Name - if err := scache.AddPod(firstPod); err != nil { + if err := scache.AddPod(logger, firstPod); err != nil { t.Fatalf("err: %v", err) } - if err := scache.RemovePod(firstPod); err != nil { + if err := scache.RemovePod(logger, firstPod); err != nil { t.Fatalf("err: %v", err) } @@ -770,10 +777,11 @@ func TestSchedulerNoPhantomPodAfterDelete(t *testing.T) { } func TestSchedulerFailedSchedulingReasons(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queuedPodStore := clientcache.NewFIFO(clientcache.MetaNamespaceKeyFunc) - scache := internalcache.New(10*time.Minute, ctx.Done()) + scache := internalcache.New(ctx, 10*time.Minute) // Design the baseline for the pods, and we will make nodes that don't fit it later. var cpu = int64(4) @@ -805,7 +813,7 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) { v1.ResourcePods: *(resource.NewQuantity(10, resource.DecimalSI)), }}, } - scache.AddNode(&node) + scache.AddNode(logger, &node) nodes = append(nodes, &node) objects = append(objects, &node) } @@ -1034,6 +1042,10 @@ func TestSchedulerBinding(t *testing.T) { for _, test := range table { t.Run(test.name, func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + pod := st.MakePod().Name(test.podName).Obj() defaultBound := false client := clientsetfake.NewSimpleClientset(pod) @@ -1050,15 +1062,13 @@ func TestSchedulerBinding(t *testing.T) { if err != nil { t.Fatal(err) } - stop := make(chan struct{}) - defer close(stop) sched := &Scheduler{ Extenders: test.extenders, - Cache: internalcache.New(100*time.Millisecond, stop), + Cache: internalcache.New(ctx, 100*time.Millisecond), nodeInfoSnapshot: nil, percentageOfNodesToScore: 0, } - err = sched.bind(context.Background(), fwk, pod, "node", nil) + err = sched.bind(ctx, fwk, pod, "node", nil) if err != nil { t.Error(err) } @@ -1481,13 +1491,14 @@ func TestFindNodesThatPassExtenders(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) var extenders []framework.Extender for ii := range tt.extenders { extenders = append(extenders, &tt.extenders[ii]) } pod := st.MakePod().Name("1").UID("1").Obj() - got, err := findNodesThatPassExtenders(extenders, pod, tt.nodes, tt.filteredNodesStatuses) + got, err := findNodesThatPassExtenders(ctx, extenders, pod, tt.nodes, tt.filteredNodesStatuses) if tt.expectsErr { if err == nil { t.Error("Unexpected non-error") @@ -1984,19 +1995,21 @@ func TestSchedulerSchedulePod(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cache := internalcache.New(time.Duration(0), wait.NeverStop) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + cache := internalcache.New(ctx, time.Duration(0)) for _, pod := range test.pods { - cache.AddPod(pod) + cache.AddPod(logger, pod) } var nodes []*v1.Node for _, name := range test.nodes { node := &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"hostname": name}}} nodes = append(nodes, node) - cache.AddNode(node) + cache.AddNode(logger, node) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() cs := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(cs, 0) for _, pvc := range test.pvcs { @@ -2019,6 +2032,7 @@ func TestSchedulerSchedulePod(t *testing.T) { } scheduler := newScheduler( + logger, cache, nil, nil, @@ -2059,8 +2073,9 @@ func TestSchedulerSchedulePod(t *testing.T) { } func TestFindFitAllError(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) nodes := makeNodeList([]string{"3", "2", "1"}) - scheduler := makeScheduler(nodes) + scheduler := makeScheduler(ctx, nodes) fwk, err := st.NewFramework( []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), @@ -2094,8 +2109,9 @@ func TestFindFitAllError(t *testing.T) { } func TestFindFitSomeError(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) nodes := makeNodeList([]string{"3", "2", "1"}) - scheduler := makeScheduler(nodes) + scheduler := makeScheduler(ctx, nodes) fwk, err := st.NewFramework( []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), @@ -2161,6 +2177,10 @@ func TestFindFitPredicateCallCounts(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + nodes := makeNodeList([]string{"1"}) plugin := st.FakeFilterPlugin{} @@ -2183,11 +2203,11 @@ func TestFindFitPredicateCallCounts(t *testing.T) { t.Fatal(err) } - scheduler := makeScheduler(nodes) - if err := scheduler.Cache.UpdateSnapshot(scheduler.nodeInfoSnapshot); err != nil { + scheduler := makeScheduler(ctx, nodes) + if err := scheduler.Cache.UpdateSnapshot(logger, scheduler.nodeInfoSnapshot); err != nil { t.Fatal(err) } - fwk.AddNominatedPod(framework.NewPodInfo(st.MakePod().UID("nominated").Priority(midPriority).Obj()), + fwk.AddNominatedPod(logger, framework.NewPodInfo(st.MakePod().UID("nominated").Priority(midPriority).Obj()), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "1"}) _, _, err = scheduler.findNodesThatFitPod(context.Background(), fwk, framework.NewCycleState(), test.pod) @@ -2296,6 +2316,10 @@ func TestZeroRequest(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) @@ -2321,6 +2345,7 @@ func TestZeroRequest(t *testing.T) { } scheduler := newScheduler( + logger, nil, nil, nil, @@ -2332,7 +2357,6 @@ func TestZeroRequest(t *testing.T) { snapshot, schedulerapi.DefaultPercentageOfNodesToScore) - ctx := context.Background() state := framework.NewCycleState() _, _, err = scheduler.findNodesThatFitPod(ctx, fwk, state, test.pod) if err != nil { @@ -2408,13 +2432,17 @@ func TestNumFeasibleNodesToFind(t *testing.T) { } func TestFairEvaluationForNodes(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + numAllNodes := 500 nodeNames := make([]string, 0, numAllNodes) for i := 0; i < numAllNodes; i++ { nodeNames = append(nodeNames, strconv.Itoa(i)) } nodes := makeNodeList(nodeNames) - sched := makeScheduler(nodes) + sched := makeScheduler(ctx, nodes) fwk, err := st.NewFramework( []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), @@ -2475,13 +2503,17 @@ func TestPreferNominatedNodeFilterCallCounts(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + // create three nodes in the cluster. nodes := makeNodeList([]string{"node1", "node2", "node3"}) client := clientsetfake.NewSimpleClientset(test.pod) informerFactory := informers.NewSharedInformerFactory(client, 0) - cache := internalcache.New(time.Duration(0), wait.NeverStop) + cache := internalcache.New(ctx, time.Duration(0)) for _, n := range nodes { - cache.AddNode(n) + cache.AddNode(logger, n) } plugin := st.FakeFilterPlugin{FailedNodeReturnCodeMap: test.nodeReturnCodeMap} registerFakeFilterFunc := st.RegisterFilterPlugin( @@ -2505,6 +2537,7 @@ func TestPreferNominatedNodeFilterCallCounts(t *testing.T) { } snapshot := internalcache.NewSnapshot(nil, nodes) scheduler := newScheduler( + logger, cache, nil, nil, @@ -2560,13 +2593,15 @@ func makeNodeList(nodeNames []string) []*v1.Node { } // makeScheduler makes a simple Scheduler for testing. -func makeScheduler(nodes []*v1.Node) *Scheduler { - cache := internalcache.New(time.Duration(0), wait.NeverStop) +func makeScheduler(ctx context.Context, nodes []*v1.Node) *Scheduler { + logger := klog.FromContext(ctx) + cache := internalcache.New(ctx, time.Duration(0)) for _, n := range nodes { - cache.AddNode(n) + cache.AddNode(logger, n) } s := newScheduler( + logger, cache, nil, nil, @@ -2577,7 +2612,7 @@ func makeScheduler(nodes []*v1.Node) *Scheduler { nil, emptySnapshot, schedulerapi.DefaultPercentageOfNodesToScore) - cache.UpdateSnapshot(s.nodeInfoSnapshot) + cache.UpdateSnapshot(logger, s.nodeInfoSnapshot) return s } @@ -2632,6 +2667,7 @@ func setupTestSchedulerWithOnePodOnNode(ctx context.Context, t *testing.T, queue // queuedPodStore: pods queued before processing. // scache: scheduler cache that might contain assumed pods. func setupTestScheduler(ctx context.Context, queuedPodStore *clientcache.FIFO, cache internalcache.Cache, informerFactory informers.SharedInformerFactory, broadcaster events.EventBroadcaster, fns ...st.RegisterPluginFunc) (*Scheduler, chan *v1.Binding, chan error) { + logger := klog.FromContext(ctx) bindingChan := make(chan *v1.Binding, 1) client := clientsetfake.NewSimpleClientset() client.PrependReactor("create", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { @@ -2666,12 +2702,13 @@ func setupTestScheduler(ctx context.Context, queuedPodStore *clientcache.FIFO, c errChan := make(chan error, 1) sched := newScheduler( + logger, cache, nil, func() *framework.QueuedPodInfo { return &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(clientcache.Pop(queuedPodStore).(*v1.Pod))} }, - func(p *framework.QueuedPodInfo, err error) { + func(_ context.Context, p *framework.QueuedPodInfo, err error) { errChan <- err }, nil, @@ -2686,6 +2723,7 @@ func setupTestScheduler(ctx context.Context, queuedPodStore *clientcache.FIFO, c } func setupTestSchedulerWithVolumeBinding(ctx context.Context, volumeBinder volumebinding.SchedulerVolumeBinder, broadcaster events.EventBroadcaster) (*Scheduler, chan *v1.Binding, chan error) { + logger := klog.FromContext(ctx) testNode := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1", UID: types.UID("node1")}} queuedPodStore := clientcache.NewFIFO(clientcache.MetaNamespaceKeyFunc) pod := podWithID("foo", "") @@ -2693,8 +2731,8 @@ func setupTestSchedulerWithVolumeBinding(ctx context.Context, volumeBinder volum pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{Name: "testVol", VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: "testPVC"}}}) queuedPodStore.Add(pod) - scache := internalcache.New(10*time.Minute, ctx.Done()) - scache.AddNode(&testNode) + scache := internalcache.New(ctx, 10*time.Minute) + scache.AddNode(logger, &testNode) testPVC := v1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "testPVC", Namespace: pod.Namespace, UID: types.UID("testPVC")}} client := clientsetfake.NewSimpleClientset(&testNode, &testPVC) informerFactory := informers.NewSharedInformerFactory(client, 0) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index c165b34ffc29..9e9315cab080 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -76,7 +76,7 @@ type Scheduler struct { // Error is called if there is an error. It is passed the pod in // question, and the error - Error func(*framework.QueuedPodInfo, error) + Error func(context.Context, *framework.QueuedPodInfo, error) // SchedulePod tries to schedule the given pod to one of the nodes in the node list. // Return a struct of ScheduleResult with the name of suggested host on success, @@ -99,6 +99,11 @@ type Scheduler struct { percentageOfNodesToScore int32 nextStartNodeIndex int + + // logger *must* be initialized when creating a Scheduler, + // otherwise logging functions will access a nil sink and + // panic. + logger klog.Logger } type schedulerOptions struct { @@ -231,17 +236,15 @@ var defaultSchedulerOptions = schedulerOptions{ } // New returns a Scheduler -func New(client clientset.Interface, +func New(ctx context.Context, + client clientset.Interface, informerFactory informers.SharedInformerFactory, dynInformerFactory dynamicinformer.DynamicSharedInformerFactory, recorderFactory profile.RecorderFactory, - stopCh <-chan struct{}, opts ...Option) (*Scheduler, error) { - stopEverything := stopCh - if stopEverything == nil { - stopEverything = wait.NeverStop - } + logger := klog.FromContext(ctx) + stopEverything := ctx.Done() options := defaultSchedulerOptions for _, opt := range opts { @@ -265,7 +268,7 @@ func New(client clientset.Interface, metrics.Register() - extenders, err := buildExtenders(options.extenders, options.profiles) + extenders, err := buildExtenders(logger, options.extenders, options.profiles) if err != nil { return nil, fmt.Errorf("couldn't build extenders: %w", err) } @@ -278,7 +281,7 @@ func New(client clientset.Interface, snapshot := internalcache.NewEmptySnapshot() clusterEventMap := make(map[framework.ClusterEvent]sets.String) - profiles, err := profile.NewMap(options.profiles, registry, recorderFactory, stopCh, + profiles, err := profile.NewMap(options.profiles, registry, recorderFactory, stopEverything, frameworkruntime.WithComponentConfigVersion(options.componentConfigVersion), frameworkruntime.WithClientSet(client), frameworkruntime.WithKubeConfig(options.kubeConfig), @@ -308,16 +311,17 @@ func New(client clientset.Interface, internalqueue.WithPodMaxInUnschedulablePodsDuration(options.podMaxInUnschedulablePodsDuration), ) - schedulerCache := internalcache.New(durationToExpireAssumedPod, stopEverything) + schedulerCache := internalcache.New(ctx, durationToExpireAssumedPod) // Setup cache debugger. debugger := cachedebugger.New(nodeLister, podLister, schedulerCache, podQueue) - debugger.ListenForSignal(stopEverything) + debugger.ListenForSignal(ctx) sched := newScheduler( + logger, schedulerCache, extenders, - internalqueue.MakeNextPodFunc(podQueue), + internalqueue.MakeNextPodFunc(logger, podQueue), MakeDefaultErrorFunc(client, podLister, podQueue, schedulerCache), stopEverything, podQueue, @@ -334,7 +338,8 @@ func New(client clientset.Interface, // Run begins watching and scheduling. It starts scheduling and blocked until the context is done. func (sched *Scheduler) Run(ctx context.Context) { - sched.SchedulingQueue.Run() + logger := klog.FromContext(ctx) + sched.SchedulingQueue.Run(logger) // We need to start scheduleOne loop in a dedicated goroutine, // because scheduleOne function hangs on getting the next item @@ -349,17 +354,18 @@ func (sched *Scheduler) Run(ctx context.Context) { } // MakeDefaultErrorFunc construct a function to handle pod scheduler error -func MakeDefaultErrorFunc(client clientset.Interface, podLister corelisters.PodLister, podQueue internalqueue.SchedulingQueue, schedulerCache internalcache.Cache) func(*framework.QueuedPodInfo, error) { - return func(podInfo *framework.QueuedPodInfo, err error) { +func MakeDefaultErrorFunc(client clientset.Interface, podLister corelisters.PodLister, podQueue internalqueue.SchedulingQueue, schedulerCache internalcache.Cache) func(context.Context, *framework.QueuedPodInfo, error) { + return func(ctx context.Context, podInfo *framework.QueuedPodInfo, err error) { + logger := klog.FromContext(ctx) pod := podInfo.Pod if err == ErrNoNodesAvailable { - klog.V(2).InfoS("Unable to schedule pod; no nodes are registered to the cluster; waiting", "pod", klog.KObj(pod)) + logger.V(2).Info("Unable to schedule pod; no nodes are registered to the cluster; waiting", "pod", klog.KObj(pod)) } else if fitError, ok := err.(*framework.FitError); ok { // Inject UnschedulablePlugins to PodInfo, which will be used later for moving Pods between queues efficiently. podInfo.UnschedulablePlugins = fitError.Diagnosis.UnschedulablePlugins - klog.V(2).InfoS("Unable to schedule pod; no fit; waiting", "pod", klog.KObj(pod), "err", err) + logger.V(2).Info("Unable to schedule pod; no fit; waiting", "pod", klog.KObj(pod), "err", err) } else if apierrors.IsNotFound(err) { - klog.V(2).InfoS("Unable to schedule pod, possibly due to node not found; waiting", "pod", klog.KObj(pod), "err", err) + logger.V(2).Info("Unable to schedule pod, possibly due to node not found; waiting", "pod", klog.KObj(pod), "err", err) if errStatus, ok := err.(apierrors.APIStatus); ok && errStatus.Status().Details.Kind == "node" { nodeName := errStatus.Status().Details.Name // when node is not found, We do not remove the node right away. Trying again to get @@ -367,33 +373,33 @@ func MakeDefaultErrorFunc(client clientset.Interface, podLister corelisters.PodL _, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil && apierrors.IsNotFound(err) { node := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}} - if err := schedulerCache.RemoveNode(&node); err != nil { - klog.V(4).InfoS("Node is not found; failed to remove it from the cache", "node", node.Name) + if err := schedulerCache.RemoveNode(logger, &node); err != nil { + logger.V(4).Info("Node is not found; failed to remove it from the cache", "node", node.Name) } } } } else { - klog.ErrorS(err, "Error scheduling pod; retrying", "pod", klog.KObj(pod)) + logger.Error(err, "Error scheduling pod; retrying", "pod", klog.KObj(pod)) } // Check if the Pod exists in informer cache. cachedPod, err := podLister.Pods(pod.Namespace).Get(pod.Name) if err != nil { - klog.InfoS("Pod doesn't exist in informer cache", "pod", klog.KObj(pod), "err", err) + logger.Info("Pod doesn't exist in informer cache", "pod", klog.KObj(pod), "err", err) return } // In the case of extender, the pod may have been bound successfully, but timed out returning its response to the scheduler. // It could result in the live version to carry .spec.nodeName, and that's inconsistent with the internal-queued version. if len(cachedPod.Spec.NodeName) != 0 { - klog.InfoS("Pod has been assigned to node. Abort adding it back to queue.", "pod", klog.KObj(pod), "node", cachedPod.Spec.NodeName) + logger.Info("Pod has been assigned to node. Abort adding it back to queue.", "pod", klog.KObj(pod), "node", cachedPod.Spec.NodeName) return } // As is from SharedInformer, we need to do a DeepCopy() here. podInfo.PodInfo = framework.NewPodInfo(cachedPod.DeepCopy()) - if err := podQueue.AddUnschedulableIfNotPresent(podInfo, podQueue.SchedulingCycle()); err != nil { - klog.ErrorS(err, "Error occurred") + if err := podQueue.AddUnschedulableIfNotPresent(logger, podInfo, podQueue.SchedulingCycle()); err != nil { + logger.Error(err, "Error occurred") } } } @@ -406,7 +412,7 @@ func NewInformerFactory(cs clientset.Interface, resyncPeriod time.Duration) info return informerFactory } -func buildExtenders(extenders []schedulerapi.Extender, profiles []schedulerapi.KubeSchedulerProfile) ([]framework.Extender, error) { +func buildExtenders(logger klog.Logger, extenders []schedulerapi.Extender, profiles []schedulerapi.KubeSchedulerProfile) ([]framework.Extender, error) { var fExtenders []framework.Extender if len(extenders) == 0 { return nil, nil @@ -415,7 +421,7 @@ func buildExtenders(extenders []schedulerapi.Extender, profiles []schedulerapi.K var ignoredExtendedResources []string var ignorableExtenders []framework.Extender for i := range extenders { - klog.V(2).InfoS("Creating extender", "extender", extenders[i]) + logger.V(2).Info("Creating extender", "extender", extenders[i]) extender, err := NewHTTPExtender(&extenders[i]) if err != nil { return nil, err @@ -466,10 +472,11 @@ func buildExtenders(extenders []schedulerapi.Extender, profiles []schedulerapi.K // newScheduler creates a Scheduler object. func newScheduler( + logger klog.Logger, cache internalcache.Cache, extenders []framework.Extender, nextPod func() *framework.QueuedPodInfo, - Error func(*framework.QueuedPodInfo, error), + Error func(context.Context, *framework.QueuedPodInfo, error), stopEverything <-chan struct{}, schedulingQueue internalqueue.SchedulingQueue, profiles profile.Map, @@ -487,6 +494,7 @@ func newScheduler( client: client, nodeInfoSnapshot: nodeInfoSnapshot, percentageOfNodesToScore: percentageOfNodesToScore, + logger: logger, } sched.SchedulePod = sched.schedulePod return &sched diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 47298d5869a4..7c8c1f9aaed6 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" + "k8s.io/klog/v2/ktesting" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" @@ -163,19 +164,20 @@ func TestSchedulerCreation(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() client := fake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()}) - stopCh := make(chan struct{}) - defer close(stopCh) s, err := New( + ctx, client, informerFactory, nil, profile.NewRecorderFactory(eventBroadcaster), - stopCh, tc.opts..., ) @@ -259,8 +261,9 @@ func TestDefaultErrorFunc(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - stopCh := make(chan struct{}) - defer close(stopCh) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() client := fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testPod}}) informerFactory := informers.NewSharedInformerFactory(client, 0) @@ -269,23 +272,23 @@ func TestDefaultErrorFunc(t *testing.T) { podInformer.Informer().GetStore().Add(testPod) queue := internalqueue.NewPriorityQueue(nil, informerFactory, internalqueue.WithClock(testingclock.NewFakeClock(time.Now()))) - schedulerCache := internalcache.New(30*time.Second, stopCh) + schedulerCache := internalcache.New(ctx, 30*time.Second) - queue.Add(testPod) + queue.Add(logger, testPod) queue.Pop() if tt.podUpdatedDuringScheduling { podInformer.Informer().GetStore().Update(testPodUpdated) - queue.Update(testPod, testPodUpdated) + queue.Update(logger, testPod, testPodUpdated) } if tt.podDeletedDuringScheduling { podInformer.Informer().GetStore().Delete(testPod) - queue.Delete(testPod) + queue.Delete(logger, testPod) } testPodInfo := &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(testPod)} errFunc := MakeDefaultErrorFunc(client, podInformer.Lister(), queue, schedulerCache) - errFunc(testPodInfo, tt.injectErr) + errFunc(ctx, testPodInfo, tt.injectErr) var got *v1.Pod if tt.podUpdatedDuringScheduling { @@ -333,7 +336,8 @@ func TestDefaultErrorFunc_NodeNotFound(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() client := fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testPod}}, &v1.NodeList{Items: tt.nodes}) @@ -343,12 +347,12 @@ func TestDefaultErrorFunc_NodeNotFound(t *testing.T) { podInformer.Informer().GetStore().Add(testPod) queue := internalqueue.NewPriorityQueue(nil, informerFactory, internalqueue.WithClock(testingclock.NewFakeClock(time.Now()))) - schedulerCache := internalcache.New(30*time.Second, ctx.Done()) + schedulerCache := internalcache.New(ctx, 30*time.Second) for i := range tt.nodes { node := tt.nodes[i] // Add node to schedulerCache no matter it's deleted in API server or not. - schedulerCache.AddNode(&node) + schedulerCache.AddNode(logger, &node) if node.Name == tt.nodeNameToDelete { client.CoreV1().Nodes().Delete(ctx, node.Name, metav1.DeleteOptions{}) } @@ -356,7 +360,7 @@ func TestDefaultErrorFunc_NodeNotFound(t *testing.T) { testPodInfo := &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(testPod)} errFunc := MakeDefaultErrorFunc(client, podInformer.Lister(), queue, schedulerCache) - errFunc(testPodInfo, tt.injectErr) + errFunc(ctx, testPodInfo, tt.injectErr) gotNodes := schedulerCache.Dump().Nodes gotNodeNames := sets.NewString() @@ -371,8 +375,9 @@ func TestDefaultErrorFunc_NodeNotFound(t *testing.T) { } func TestDefaultErrorFunc_PodAlreadyBound(t *testing.T) { - stopCh := make(chan struct{}) - defer close(stopCh) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() nodeFoo := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} testPod := st.MakePod().Name("test-pod").Namespace(v1.NamespaceDefault).Node("foo").Obj() @@ -384,14 +389,14 @@ func TestDefaultErrorFunc_PodAlreadyBound(t *testing.T) { podInformer.Informer().GetStore().Add(testPod) queue := internalqueue.NewPriorityQueue(nil, informerFactory, internalqueue.WithClock(testingclock.NewFakeClock(time.Now()))) - schedulerCache := internalcache.New(30*time.Second, stopCh) + schedulerCache := internalcache.New(ctx, 30*time.Second) // Add node to schedulerCache no matter it's deleted in API server or not. - schedulerCache.AddNode(&nodeFoo) + schedulerCache.AddNode(logger, &nodeFoo) testPodInfo := &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(testPod)} errFunc := MakeDefaultErrorFunc(client, podInformer.Lister(), queue, schedulerCache) - errFunc(testPodInfo, fmt.Errorf("binding rejected: timeout")) + errFunc(ctx, testPodInfo, fmt.Errorf("binding rejected: timeout")) pod := getPodFromPriorityQueue(queue, testPod) if pod != nil { diff --git a/pkg/scheduler/testing/fake_extender.go b/pkg/scheduler/testing/fake_extender.go index ab66d6c8ca80..e9d933ccea13 100644 --- a/pkg/scheduler/testing/fake_extender.go +++ b/pkg/scheduler/testing/fake_extender.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" corev1helpers "k8s.io/component-helpers/scheduling/corev1" + "k8s.io/klog/v2" extenderv1 "k8s.io/kube-scheduler/extender/v1" "k8s.io/kubernetes/pkg/scheduler/framework" frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" @@ -182,6 +183,7 @@ func (f *FakeExtender) SupportsPreemption() bool { // ProcessPreemption implements the extender preempt function. func (f *FakeExtender) ProcessPreemption( + logger klog.Logger, pod *v1.Pod, nodeNameToVictims map[string]*extenderv1.Victims, nodeInfos framework.NodeInfoLister, @@ -199,7 +201,7 @@ func (f *FakeExtender) ProcessPreemption( for nodeName, victims := range nodeNameToVictimsCopy { // Try to do preemption on extender side. nodeInfo, _ := nodeInfos.Get(nodeName) - extenderVictimPods, extenderPDBViolations, fits, err := f.selectVictimsOnNodeByExtender(pod, nodeInfo.Node()) + extenderVictimPods, extenderPDBViolations, fits, err := f.selectVictimsOnNodeByExtender(logger, pod, nodeInfo.Node()) if err != nil { return nil, err } @@ -221,7 +223,7 @@ func (f *FakeExtender) ProcessPreemption( // 1. More victim pods (if any) amended by preemption phase of extender. // 2. Number of violating victim (used to calculate PDB). // 3. Fits or not after preemption phase on extender's side. -func (f *FakeExtender) selectVictimsOnNodeByExtender(pod *v1.Pod, node *v1.Node) ([]*v1.Pod, int, bool, error) { +func (f *FakeExtender) selectVictimsOnNodeByExtender(logger klog.Logger, pod *v1.Pod, node *v1.Node) ([]*v1.Pod, int, bool, error) { // If a extender support preemption but have no cached node info, let's run filter to make sure // default scheduler's decision still stand with given pod and node. if !f.NodeCacheCapable { @@ -242,7 +244,7 @@ func (f *FakeExtender) selectVictimsOnNodeByExtender(pod *v1.Pod, node *v1.Node) var potentialVictims []*v1.Pod removePod := func(rp *v1.Pod) { - nodeInfoCopy.RemovePod(rp) + nodeInfoCopy.RemovePod(logger, rp) } addPod := func(ap *v1.Pod) { nodeInfoCopy.AddPod(ap) diff --git a/pkg/scheduler/util/utils.go b/pkg/scheduler/util/utils.go index 929dfb8fa443..d431a953ac33 100644 --- a/pkg/scheduler/util/utils.go +++ b/pkg/scheduler/util/utils.go @@ -56,7 +56,7 @@ func GetPodStartTime(pod *v1.Pod) *metav1.Time { func GetEarliestPodStartTime(victims *extenderv1.Victims) *metav1.Time { if len(victims.Pods) == 0 { // should not reach here. - klog.ErrorS(fmt.Errorf("victims.Pods is empty. Should not reach here"), "") + klog.Background().Error(nil, "victims.Pods is empty. Should not reach here") return nil } diff --git a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go index 35d5f0a866be..8a9a03f3a24d 100644 --- a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go +++ b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go @@ -33,6 +33,7 @@ import ( "k8s.io/cloud-provider/app/config" "k8s.io/cloud-provider/options" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog/v2" ) // TearDownFunc is to be called to tear down a test server. @@ -47,20 +48,14 @@ type TestServer struct { TmpDir string // Temp Dir used, by the apiserver } -// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie -type Logger interface { - Errorf(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) - Logf(format string, args ...interface{}) -} - // StartTestServer starts a cloud-controller-manager. A rest client config and a tear-down func, // and location of the tmpdir are returned. // // Note: we return a tear-down func instead of a stop channel because the later will leak temporary // files that because Golang testing's call to os.Exit will not give a stop channel go routine // enough time to remove temporary files. -func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { +func StartTestServer(ctx context.Context, customFlags []string) (result TestServer, err error) { + logger := klog.FromContext(ctx) stopCh := make(chan struct{}) configDoneCh := make(chan struct{}) var capturedConfig config.CompletedConfig @@ -93,11 +88,11 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) if err != nil { - t.Fatalf("Cloud provider could not be initialized: %v", err) + panic(fmt.Errorf("Cloud provider could not be initialized: %v", err)) } s.SecureServing.ServerCert.CertDirectory = result.TmpDir if cloud == nil { - t.Fatalf("Cloud provider is nil") + panic("Cloud provider is nil") } return cloud } @@ -129,7 +124,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err commandArgs = append(commandArgs, fmt.Sprintf("--secure-port=%d", bindPort)) commandArgs = append(commandArgs, fmt.Sprintf("--cert-dir=%s", result.TmpDir)) - t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort) + logger.Info("cloud-controller-manager will listen securely", "port", bindPort) } for _, listener := range listeners { listener.Close() @@ -151,7 +146,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err return result, err } - t.Logf("Waiting for /healthz to be ok...") + logger.Info("Waiting for /healthz to be ok...") client, err := kubernetes.NewForConfig(capturedConfig.LoopbackClientConfig) if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) @@ -185,14 +180,13 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t Logger, flags []string) *TestServer { - result, err := StartTestServer(t, flags) +func StartTestServerOrDie(ctx context.Context, flags []string) *TestServer { + result, err := StartTestServer(ctx, flags) if err == nil { return &result } - t.Fatalf("failed to launch server: %v", err) - return nil + panic(fmt.Errorf("failed to launch server: %v", err)) } func createListenerOnFreePort() (net.Listener, int, error) { diff --git a/test/integration/daemonset/daemonset_test.go b/test/integration/daemonset/daemonset_test.go index fe1cb4e0cb06..72aa494447b4 100644 --- a/test/integration/daemonset/daemonset_test.go +++ b/test/integration/daemonset/daemonset_test.go @@ -81,11 +81,11 @@ func setup(t *testing.T) (context.Context, kubeapiservertesting.TearDownFunc, *d }) sched, err := scheduler.New( + ctx, clientSet, informers, nil, profile.NewRecorderFactory(eventBroadcaster), - ctx.Done(), ) if err != nil { t.Fatalf("Couldn't create scheduler: %v", err) diff --git a/test/integration/scheduler/queue_test.go b/test/integration/scheduler/queue_test.go index 8c9364deb4fe..a0b37d408fa6 100644 --- a/test/integration/scheduler/queue_test.go +++ b/test/integration/scheduler/queue_test.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" "k8s.io/kube-scheduler/config/v1beta3" apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/scheduler" @@ -105,7 +106,7 @@ func TestCoreResourceEnqueue(t *testing.T) { if fitError == nil { t.Fatalf("Expect Pod %v to fail at scheduling.", podInfo.Pod.Name) } - testCtx.Scheduler.Error(podInfo, fitError) + testCtx.Scheduler.Error(ctx, podInfo, fitError) } // Trigger a NodeTaintChange event. @@ -248,6 +249,7 @@ func TestCustomResourceEnqueue(t *testing.T) { defer testutils.CleanupTest(t, testCtx) cs, ns, ctx := testCtx.ClientSet, testCtx.NS.Name, testCtx.Ctx + logger := klog.FromContext(ctx) // Create one Node. node := st.MakeNode().Name("fake-node").Obj() if _, err := cs.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}); err != nil { @@ -280,11 +282,11 @@ func TestCustomResourceEnqueue(t *testing.T) { if fitError == nil { t.Fatalf("Expect Pod %v to fail at scheduling.", podInfo.Pod.Name) } - testCtx.Scheduler.Error(podInfo, fitError) + testCtx.Scheduler.Error(ctx, podInfo, fitError) // Scheduling cycle is incremented from 0 to 1 after NextPod() is called, so // pass a number larger than 1 to move Pod to unschedulablePods. - testCtx.Scheduler.SchedulingQueue.AddUnschedulableIfNotPresent(podInfo, 10) + testCtx.Scheduler.SchedulingQueue.AddUnschedulableIfNotPresent(logger, podInfo, 10) // Trigger a Custom Resource event. // We expect this event to trigger moving the test Pod from unschedulablePods to activeQ. diff --git a/test/integration/scheduler_perf/main_test.go b/test/integration/scheduler_perf/main_test.go index 16275396eba4..7df208790d4e 100644 --- a/test/integration/scheduler_perf/main_test.go +++ b/test/integration/scheduler_perf/main_test.go @@ -17,11 +17,16 @@ limitations under the License. package benchmark import ( + "flag" "testing" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/test/integration/framework" ) func TestMain(m *testing.M) { + ktesting.DefaultConfig = ktesting.NewConfig(ktesting.Verbosity(0)) // Run with -v=0 by default. + ktesting.DefaultConfig.AddFlags(flag.CommandLine) + framework.EtcdMain(m.Run) } diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go index e4f2e067d8ad..b96acb7135d0 100644 --- a/test/integration/scheduler_perf/scheduler_perf_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_test.go @@ -634,7 +634,7 @@ func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem { b.Fatalf("validate scheduler config file failed: %v", err) } } - finalFunc, podInformer, client, dynClient := mustSetupScheduler(cfg) + finalFunc, podInformer, client, dynClient := mustSetupScheduler(b, cfg) b.Cleanup(finalFunc) var mu sync.Mutex diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index 114c119a660d..82dd9587012b 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -40,6 +40,7 @@ import ( "k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/testutil" "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta2" "k8s.io/kubernetes/pkg/scheduler/apis/config" kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" @@ -74,7 +75,7 @@ func newDefaultComponentConfig() (*config.KubeSchedulerConfiguration, error) { // remove resources after finished. // Notes on rate limiter: // - client rate limit is set to 5000. -func mustSetupScheduler(config *config.KubeSchedulerConfiguration) (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface, dynamic.Interface) { +func mustSetupScheduler(tl ktesting.TL, config *config.KubeSchedulerConfiguration) (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface, dynamic.Interface) { // Run API server with minimimal logging by default. Can be raised with -v. framework.MinVerbosity = 0 apiURL, apiShutdown := util.StartApiserver() @@ -102,7 +103,7 @@ func mustSetupScheduler(config *config.KubeSchedulerConfiguration) (util.Shutdow // Not all config options will be effective but only those mostly related with scheduler performance will // be applied to start a scheduler, most of them are defined in `scheduler.schedulerOptions`. - _, podInformer, schedulerShutdown := util.StartScheduler(client, cfg, config) + _, podInformer, schedulerShutdown := util.StartScheduler(tl, client, cfg, config) fakePVControllerShutdown := util.StartFakePVController(client) shutdownFunc := func() { diff --git a/test/integration/serving/serving_test.go b/test/integration/serving/serving_test.go index 0e2bd16e07b9..4640c518540e 100644 --- a/test/integration/serving/serving_test.go +++ b/test/integration/serving/serving_test.go @@ -17,6 +17,7 @@ limitations under the License. package serving import ( + "context" "crypto/tls" "crypto/x509" "fmt" @@ -32,6 +33,7 @@ import ( cloudprovider "k8s.io/cloud-provider" cloudctrlmgrtesting "k8s.io/cloud-provider/app/testing" "k8s.io/cloud-provider/fake" + "k8s.io/klog/v2/ktesting" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing" @@ -39,15 +41,15 @@ import ( ) type componentTester interface { - StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) + StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) } type kubeControllerManagerTester struct{} -func (kubeControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { +func (kubeControllerManagerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { // avoid starting any controller loops, we're just testing serving customFlags = append([]string{"--controllers="}, customFlags...) - gotResult, err := kubectrlmgrtesting.StartTestServer(t, customFlags) + gotResult, err := kubectrlmgrtesting.StartTestServer(ctx, customFlags) if err != nil { return nil, nil, nil, nil, err } @@ -56,8 +58,8 @@ func (kubeControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, type cloudControllerManagerTester struct{} -func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { - gotResult, err := cloudctrlmgrtesting.StartTestServer(t, customFlags) +func (cloudControllerManagerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := cloudctrlmgrtesting.StartTestServer(ctx, customFlags) if err != nil { return nil, nil, nil, nil, err } @@ -66,8 +68,8 @@ func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, type kubeSchedulerTester struct{} -func (kubeSchedulerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { - gotResult, err := kubeschedulertesting.StartTestServer(t, customFlags) +func (kubeSchedulerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := kubeschedulertesting.StartTestServer(ctx, customFlags) if err != nil { return nil, nil, nil, nil, err } @@ -252,7 +254,8 @@ func testComponent(t *testing.T, tester componentTester, kubeconfig, brokenKubec } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - secureOptions, secureInfo, insecureInfo, tearDownFn, err := tester.StartTestServer(t, append(append([]string{}, tt.flags...), extraFlags...)) + _, ctx := ktesting.NewTestContext(t) + secureOptions, secureInfo, insecureInfo, tearDownFn, err := tester.StartTestServer(ctx, append(append([]string{}, tt.flags...), extraFlags...)) if tearDownFn != nil { defer tearDownFn() } @@ -375,7 +378,8 @@ func testComponentWithSecureServing(t *testing.T, tester componentTester, kubeco } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - secureOptions, secureInfo, _, tearDownFn, err := tester.StartTestServer(t, append(append([]string{}, tt.flags...), extraFlags...)) + _, ctx := ktesting.NewTestContext(t) + secureOptions, secureInfo, _, tearDownFn, err := tester.StartTestServer(ctx, append(append([]string{}, tt.flags...), extraFlags...)) if tearDownFn != nil { defer tearDownFn() } diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 27dfbd37a160..d486df53eb6b 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -44,6 +44,7 @@ import ( "k8s.io/client-go/tools/events" pvutil "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" podutil "k8s.io/kubernetes/pkg/api/v1/pod" @@ -79,8 +80,9 @@ func StartApiserver() (string, ShutdownFunc) { // StartScheduler configures and starts a scheduler given a handle to the clientSet interface // and event broadcaster. It returns the running scheduler, podInformer and the shutdown function to stop it. -func StartScheduler(clientSet clientset.Interface, kubeConfig *restclient.Config, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) { - ctx, cancel := context.WithCancel(context.Background()) +func StartScheduler(tl ktesting.TL, clientSet clientset.Interface, kubeConfig *restclient.Config, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) { + _, ctx := ktesting.NewTestContext(tl) + ctx, cancel := context.WithCancel(ctx) informerFactory := scheduler.NewInformerFactory(clientSet, 0) evtBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{ @@ -89,11 +91,11 @@ func StartScheduler(clientSet clientset.Interface, kubeConfig *restclient.Config evtBroadcaster.StartRecordingToSink(ctx.Done()) sched, err := scheduler.New( + ctx, clientSet, informerFactory, nil, profile.NewRecorderFactory(evtBroadcaster), - ctx.Done(), scheduler.WithKubeConfig(kubeConfig), scheduler.WithProfiles(cfg.Profiles...), scheduler.WithPercentageOfNodesToScore(cfg.PercentageOfNodesToScore), @@ -405,11 +407,11 @@ func InitTestSchedulerWithOptions( opts = append(opts, scheduler.WithKubeConfig(testCtx.KubeConfig)) testCtx.Scheduler, err = scheduler.New( + testCtx.Ctx, testCtx.ClientSet, testCtx.InformerFactory, testCtx.DynInformerFactory, profile.NewRecorderFactory(eventBroadcaster), - testCtx.Ctx.Done(), opts..., ) From 408a28e76769b05636fa63f1e1fdef13cdc913be Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Sun, 23 Jan 2022 11:10:48 +0100 Subject: [PATCH 2/5] scheduler_perf: support JSON This enables benchmarking with JSON selected as output. --- test/integration/scheduler_perf/main_test.go | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/integration/scheduler_perf/main_test.go b/test/integration/scheduler_perf/main_test.go index 7df208790d4e..d32ce1287807 100644 --- a/test/integration/scheduler_perf/main_test.go +++ b/test/integration/scheduler_perf/main_test.go @@ -18,8 +18,14 @@ package benchmark import ( "flag" + "fmt" + "os" "testing" + "github.com/spf13/pflag" + + logsapi "k8s.io/component-base/logs/api/v1" + _ "k8s.io/component-base/logs/json/register" "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/test/integration/framework" ) @@ -28,5 +34,22 @@ func TestMain(m *testing.M) { ktesting.DefaultConfig = ktesting.NewConfig(ktesting.Verbosity(0)) // Run with -v=0 by default. ktesting.DefaultConfig.AddFlags(flag.CommandLine) + c := logsapi.NewLoggingConfiguration() + + // component-base only supports pflag at the moment. + var fs pflag.FlagSet + logsapi.AddFlags(c, &fs) + // Not ideal. https://github.com/spf13/pflag/pull/330 would be better. + fs.VisitAll(func(f *pflag.Flag) { + if flag.CommandLine.Lookup(f.Name) == nil { + flag.CommandLine.Var(f.Value, f.Name, f.Usage) + } + }) + flag.Parse() + if err := logsapi.ValidateAndApply(c, nil /* no feature gates */); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + framework.EtcdMain(m.Run) } From 01b8046c58e288a0a9d171235fb37d4abe0acc3e Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 27 Jan 2022 11:33:11 +0100 Subject: [PATCH 3/5] scheduler: use WithName and WithValues This adds a prefix to each log entry that shows which operation and/or component produced the log entry (WithName) and ensures that relevant additional values, in particular the pod that gets scheduled, are always included (WithValues). This makes log entries easier to understand, in particular when multiple pods get scheduled in parallel. The downside is the constant overhead for these additional calls: they need to do some work, whether log entries will be printed or not. --- pkg/scheduler/framework/runtime/framework.go | 50 +++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 7a73bd162ebb..8d5164ba8d47 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -613,7 +613,10 @@ func (f *frameworkImpl) RunPreFilterPlugins(ctx context.Context, state *framewor }() var result *framework.PreFilterResult var pluginsWithNodes []string + logger := klog.FromContext(ctx).WithName("PreFilter").WithValues("pod", klog.KObj(pod)) for _, pl := range f.preFilterPlugins { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) r, s := f.runPreFilterPlugin(ctx, pl, state, pod) if !s.IsSuccess() { s.SetFailedPlugin(pl.Name()) @@ -658,11 +661,13 @@ func (f *frameworkImpl) RunPreFilterExtensionAddPod( podInfoToAdd *framework.PodInfo, nodeInfo *framework.NodeInfo, ) (status *framework.Status) { - logger := klog.FromContext(ctx) + logger := klog.FromContext(ctx).WithName("AddPod").WithValues("pod", klog.KObj(podToSchedule), "node", klog.KObj(nodeInfo.Node())) for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue } + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) status = f.runPreFilterExtensionAddPod(ctx, pl, state, podToSchedule, podInfoToAdd, nodeInfo) if !status.IsSuccess() { err := status.AsError() @@ -694,11 +699,13 @@ func (f *frameworkImpl) RunPreFilterExtensionRemovePod( podInfoToRemove *framework.PodInfo, nodeInfo *framework.NodeInfo, ) (status *framework.Status) { - logger := klog.FromContext(ctx) + logger := klog.FromContext(ctx).WithName("RemovePod").WithValues("pod", klog.KObj(podToSchedule), "node", klog.KObj(nodeInfo.Node())) for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue } + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) status = f.runPreFilterExtensionRemovePod(ctx, pl, state, podToSchedule, podInfoToRemove, nodeInfo) if !status.IsSuccess() { err := status.AsError() @@ -731,7 +738,10 @@ func (f *frameworkImpl) RunFilterPlugins( nodeInfo *framework.NodeInfo, ) framework.PluginToStatus { statuses := make(framework.PluginToStatus) + logger := klog.FromContext(ctx).WithName("Filter").WithValues("pod", klog.KObj(pod), "node", klog.KObj(nodeInfo.Node())) for _, pl := range f.filterPlugins { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) pluginStatus := f.runFilterPlugin(ctx, pl, state, pod, nodeInfo) if !pluginStatus.IsSuccess() { if !pluginStatus.IsUnschedulable() { @@ -766,10 +776,13 @@ func (f *frameworkImpl) RunPostFilterPlugins(ctx context.Context, state *framewo metrics.FrameworkExtensionPointDuration.WithLabelValues(postFilter, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() + logger := klog.FromContext(ctx).WithName("PostFilter").WithValues("pod", klog.KObj(pod)) statuses := make(framework.PluginToStatus) // `result` records the last meaningful(non-noop) PostFilterResult. var result *framework.PostFilterResult for _, pl := range f.postFilterPlugins { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) r, s := f.runPostFilterPlugin(ctx, pl, state, pod, filteredNodeStatusMap) if s.IsSuccess() { return r, s @@ -827,6 +840,8 @@ func (f *frameworkImpl) RunFilterPluginsWithNominatedPods(ctx context.Context, s // the nominated pods are treated as not running. We can't just assume the // nominated pods are running because they are not running right now and in fact, // they may end up getting scheduled to a different node. + logger := klog.FromContext(ctx).WithName("NominatedPods") + ctx = klog.NewContext(ctx, logger) for i := 0; i < 2; i++ { stateToUse := state nodeInfoToUse := info @@ -891,7 +906,10 @@ func (f *frameworkImpl) RunPreScorePlugins( defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(preScore, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() + logger := klog.FromContext(ctx).WithName("PreScore").WithValues("pod", klog.KObj(pod)) for _, pl := range f.preScorePlugins { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) status = f.runPreScorePlugin(ctx, pl, state, pod, nodes) if !status.IsSuccess() { return framework.AsStatus(fmt.Errorf("running PreScore plugin %q: %w", pl.Name(), status.AsError())) @@ -928,9 +946,13 @@ func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.Cy errCh := parallelize.NewErrorChannel() // Run Score method for each node in parallel. + logger := klog.FromContext(ctx).WithName("Score").WithValues("pod", klog.KObj(pod)) f.Parallelizer().Until(ctx, len(nodes), func(index int) { + nodeName := nodes[index].Name + logger := logger.WithValues("node", klog.ObjectRef{Name: nodeName}) for _, pl := range f.scorePlugins { - nodeName := nodes[index].Name + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) s, status := f.runScorePlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) @@ -1017,8 +1039,10 @@ func (f *frameworkImpl) RunPreBindPlugins(ctx context.Context, state *framework. defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(preBind, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx) + logger := klog.FromContext(ctx).WithName("PreBind").WithValues("pod", klog.KObj(pod)).WithValues("node", klog.ObjectRef{Name: nodeName}) for _, pl := range f.preBindPlugins { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) status = f.runPreBindPlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := status.AsError() @@ -1048,8 +1072,10 @@ func (f *frameworkImpl) RunBindPlugins(ctx context.Context, state *framework.Cyc if len(f.bindPlugins) == 0 { return framework.NewStatus(framework.Skip, "") } - logger := klog.FromContext(ctx) + logger := klog.FromContext(ctx).WithName("Bind").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) for _, bp := range f.bindPlugins { + logger := logger.WithName(bp.Name()) + ctx := klog.NewContext(ctx, logger) status = f.runBindPlugin(ctx, bp, state, pod, nodeName) if status.IsSkip() { continue @@ -1080,7 +1106,10 @@ func (f *frameworkImpl) RunPostBindPlugins(ctx context.Context, state *framework defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(postBind, framework.Success.String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() + logger := klog.FromContext(ctx).WithName("PostBind").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) for _, pl := range f.postBindPlugins { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) f.runPostBindPlugin(ctx, pl, state, pod, nodeName) } } @@ -1105,8 +1134,10 @@ func (f *frameworkImpl) RunReservePluginsReserve(ctx context.Context, state *fra defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(reserve, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx) + logger := klog.FromContext(ctx).WithName("Reserve").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) for _, pl := range f.reservePlugins { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) status = f.runReservePluginReserve(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := status.AsError() @@ -1136,7 +1167,10 @@ func (f *frameworkImpl) RunReservePluginsUnreserve(ctx context.Context, state *f }() // Execute the Unreserve operation of each reserve plugin in the // *reverse* order in which the Reserve operation was executed. + logger := klog.FromContext(ctx).WithName("Unreserve").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) for i := len(f.reservePlugins) - 1; i >= 0; i-- { + logger := logger.WithName(f.reservePlugins[i].Name()) + ctx := klog.NewContext(ctx, logger) f.runReservePluginUnreserve(ctx, f.reservePlugins[i], state, pod, nodeName) } } @@ -1164,8 +1198,10 @@ func (f *frameworkImpl) RunPermitPlugins(ctx context.Context, state *framework.C }() pluginsWaitTime := make(map[string]time.Duration) statusCode := framework.Success - logger := klog.FromContext(ctx) + logger := klog.FromContext(ctx).WithName("Permit").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) for _, pl := range f.permitPlugins { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) status, timeout := f.runPermitPlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { if status.IsUnschedulable() { From 32596ed3ef9f75fec5573c7a0c1dab5949a7d16e Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 27 Jan 2022 13:32:14 +0100 Subject: [PATCH 4/5] scheduler: only modify loggers at v>=3 --- pkg/scheduler/framework/runtime/framework.go | 228 ++++++++++++++----- 1 file changed, 172 insertions(+), 56 deletions(-) diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 8d5164ba8d47..553e0c94772f 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -613,11 +613,21 @@ func (f *frameworkImpl) RunPreFilterPlugins(ctx context.Context, state *framewor }() var result *framework.PreFilterResult var pluginsWithNodes []string - logger := klog.FromContext(ctx).WithName("PreFilter").WithValues("pod", klog.KObj(pod)) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("PreFilter").WithValues("pod", klog.KObj(pod)) + } for _, pl := range f.preFilterPlugins { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - r, s := f.runPreFilterPlugin(ctx, pl, state, pod) + var r *framework.PreFilterResult + var s *framework.Status + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + r, s = f.runPreFilterPlugin(ctx, pl, state, pod) + } else { + r, s = f.runPreFilterPlugin(ctx, pl, state, pod) + } if !s.IsSuccess() { s.SetFailedPlugin(pl.Name()) if s.IsUnschedulable() { @@ -661,17 +671,25 @@ func (f *frameworkImpl) RunPreFilterExtensionAddPod( podInfoToAdd *framework.PodInfo, nodeInfo *framework.NodeInfo, ) (status *framework.Status) { - logger := klog.FromContext(ctx).WithName("AddPod").WithValues("pod", klog.KObj(podToSchedule), "node", klog.KObj(nodeInfo.Node())) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("AddPod").WithValues("pod", klog.KObj(podToSchedule), "node", klog.KObj(nodeInfo.Node())) + } for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue } - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runPreFilterExtensionAddPod(ctx, pl, state, podToSchedule, podInfoToAdd, nodeInfo) + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + status = f.runPreFilterExtensionAddPod(ctx, pl, state, podToSchedule, podInfoToAdd, nodeInfo) + } else { + status = f.runPreFilterExtensionAddPod(ctx, pl, state, podToSchedule, podInfoToAdd, nodeInfo) + } if !status.IsSuccess() { err := status.AsError() - logger.Error(err, "Plugin failed") + logger.Error(err, "Plugin failed", "plugin", pl.Name()) return framework.AsStatus(fmt.Errorf("running AddPod on PreFilter plugin %q: %w", pl.Name(), err)) } } @@ -699,14 +717,22 @@ func (f *frameworkImpl) RunPreFilterExtensionRemovePod( podInfoToRemove *framework.PodInfo, nodeInfo *framework.NodeInfo, ) (status *framework.Status) { - logger := klog.FromContext(ctx).WithName("RemovePod").WithValues("pod", klog.KObj(podToSchedule), "node", klog.KObj(nodeInfo.Node())) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("RemovePod").WithValues("pod", klog.KObj(podToSchedule), "node", klog.KObj(nodeInfo.Node())) + } for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue } - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runPreFilterExtensionRemovePod(ctx, pl, state, podToSchedule, podInfoToRemove, nodeInfo) + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + status = f.runPreFilterExtensionRemovePod(ctx, pl, state, podToSchedule, podInfoToRemove, nodeInfo) + } else { + status = f.runPreFilterExtensionRemovePod(ctx, pl, state, podToSchedule, podInfoToRemove, nodeInfo) + } if !status.IsSuccess() { err := status.AsError() logger.Error(err, "Plugin failed") @@ -738,11 +764,20 @@ func (f *frameworkImpl) RunFilterPlugins( nodeInfo *framework.NodeInfo, ) framework.PluginToStatus { statuses := make(framework.PluginToStatus) - logger := klog.FromContext(ctx).WithName("Filter").WithValues("pod", klog.KObj(pod), "node", klog.KObj(nodeInfo.Node())) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("Filter").WithValues("pod", klog.KObj(pod), "node", klog.KObj(nodeInfo.Node())) + } for _, pl := range f.filterPlugins { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - pluginStatus := f.runFilterPlugin(ctx, pl, state, pod, nodeInfo) + var pluginStatus *framework.Status + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + pluginStatus = f.runFilterPlugin(ctx, pl, state, pod, nodeInfo) + } else { + pluginStatus = f.runFilterPlugin(ctx, pl, state, pod, nodeInfo) + } if !pluginStatus.IsSuccess() { if !pluginStatus.IsUnschedulable() { // Filter plugins are not supposed to return any status other than @@ -776,14 +811,24 @@ func (f *frameworkImpl) RunPostFilterPlugins(ctx context.Context, state *framewo metrics.FrameworkExtensionPointDuration.WithLabelValues(postFilter, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx).WithName("PostFilter").WithValues("pod", klog.KObj(pod)) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("PostFilter").WithValues("pod", klog.KObj(pod)) + } statuses := make(framework.PluginToStatus) // `result` records the last meaningful(non-noop) PostFilterResult. var result *framework.PostFilterResult for _, pl := range f.postFilterPlugins { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - r, s := f.runPostFilterPlugin(ctx, pl, state, pod, filteredNodeStatusMap) + var r *framework.PostFilterResult + var s *framework.Status + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + r, s = f.runPostFilterPlugin(ctx, pl, state, pod, filteredNodeStatusMap) + } else { + r, s = f.runPostFilterPlugin(ctx, pl, state, pod, filteredNodeStatusMap) + } if s.IsSuccess() { return r, s } else if !s.IsUnschedulable() { @@ -840,8 +885,12 @@ func (f *frameworkImpl) RunFilterPluginsWithNominatedPods(ctx context.Context, s // the nominated pods are treated as not running. We can't just assume the // nominated pods are running because they are not running right now and in fact, // they may end up getting scheduled to a different node. - logger := klog.FromContext(ctx).WithName("NominatedPods") - ctx = klog.NewContext(ctx, logger) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("NominatedPods") + ctx = klog.NewContext(ctx, logger) + } for i := 0; i < 2; i++ { stateToUse := state nodeInfoToUse := info @@ -906,11 +955,19 @@ func (f *frameworkImpl) RunPreScorePlugins( defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(preScore, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx).WithName("PreScore").WithValues("pod", klog.KObj(pod)) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("PreScore").WithValues("pod", klog.KObj(pod)) + } for _, pl := range f.preScorePlugins { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runPreScorePlugin(ctx, pl, state, pod, nodes) + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + status = f.runPreScorePlugin(ctx, pl, state, pod, nodes) + } else { + status = f.runPreScorePlugin(ctx, pl, state, pod, nodes) + } if !status.IsSuccess() { return framework.AsStatus(fmt.Errorf("running PreScore plugin %q: %w", pl.Name(), status.AsError())) } @@ -946,14 +1003,23 @@ func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.Cy errCh := parallelize.NewErrorChannel() // Run Score method for each node in parallel. - logger := klog.FromContext(ctx).WithName("Score").WithValues("pod", klog.KObj(pod)) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("Score").WithValues("pod", klog.KObj(pod)) + } f.Parallelizer().Until(ctx, len(nodes), func(index int) { nodeName := nodes[index].Name - logger := logger.WithValues("node", klog.ObjectRef{Name: nodeName}) for _, pl := range f.scorePlugins { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - s, status := f.runScorePlugin(ctx, pl, state, pod, nodeName) + var s int64 + var status *framework.Status + if verboseLogging { + logger := logger.WithName(pl.Name()).WithValues("node", klog.ObjectRef{Name: nodeName}) + ctx := klog.NewContext(ctx, logger) + s, status = f.runScorePlugin(ctx, pl, state, pod, nodeName) + } else { + s, status = f.runScorePlugin(ctx, pl, state, pod, nodeName) + } if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendErrorWithCancel(err, cancel) @@ -1039,11 +1105,19 @@ func (f *frameworkImpl) RunPreBindPlugins(ctx context.Context, state *framework. defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(preBind, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx).WithName("PreBind").WithValues("pod", klog.KObj(pod)).WithValues("node", klog.ObjectRef{Name: nodeName}) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("PreBind").WithValues("pod", klog.KObj(pod)).WithValues("node", klog.ObjectRef{Name: nodeName}) + } for _, pl := range f.preBindPlugins { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runPreBindPlugin(ctx, pl, state, pod, nodeName) + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + status = f.runPreBindPlugin(ctx, pl, state, pod, nodeName) + } else { + status = f.runPreBindPlugin(ctx, pl, state, pod, nodeName) + } if !status.IsSuccess() { err := status.AsError() logger.Error(err, "Plugin failed") @@ -1072,11 +1146,19 @@ func (f *frameworkImpl) RunBindPlugins(ctx context.Context, state *framework.Cyc if len(f.bindPlugins) == 0 { return framework.NewStatus(framework.Skip, "") } - logger := klog.FromContext(ctx).WithName("Bind").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("Bind").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + } for _, bp := range f.bindPlugins { - logger := logger.WithName(bp.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runBindPlugin(ctx, bp, state, pod, nodeName) + if verboseLogging { + logger := logger.WithName(bp.Name()) + ctx := klog.NewContext(ctx, logger) + status = f.runBindPlugin(ctx, bp, state, pod, nodeName) + } else { + status = f.runBindPlugin(ctx, bp, state, pod, nodeName) + } if status.IsSkip() { continue } @@ -1106,11 +1188,19 @@ func (f *frameworkImpl) RunPostBindPlugins(ctx context.Context, state *framework defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(postBind, framework.Success.String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx).WithName("PostBind").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("PostBind").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + } for _, pl := range f.postBindPlugins { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - f.runPostBindPlugin(ctx, pl, state, pod, nodeName) + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + f.runPostBindPlugin(ctx, pl, state, pod, nodeName) + } else { + f.runPostBindPlugin(ctx, pl, state, pod, nodeName) + } } } @@ -1134,11 +1224,19 @@ func (f *frameworkImpl) RunReservePluginsReserve(ctx context.Context, state *fra defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(reserve, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx).WithName("Reserve").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("Reserve").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + } for _, pl := range f.reservePlugins { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runReservePluginReserve(ctx, pl, state, pod, nodeName) + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + status = f.runReservePluginReserve(ctx, pl, state, pod, nodeName) + } else { + status = f.runReservePluginReserve(ctx, pl, state, pod, nodeName) + } if !status.IsSuccess() { err := status.AsError() logger.Error(err, "Plugin failed") @@ -1167,11 +1265,19 @@ func (f *frameworkImpl) RunReservePluginsUnreserve(ctx context.Context, state *f }() // Execute the Unreserve operation of each reserve plugin in the // *reverse* order in which the Reserve operation was executed. - logger := klog.FromContext(ctx).WithName("Unreserve").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("Unreserve").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + } for i := len(f.reservePlugins) - 1; i >= 0; i-- { - logger := logger.WithName(f.reservePlugins[i].Name()) - ctx := klog.NewContext(ctx, logger) - f.runReservePluginUnreserve(ctx, f.reservePlugins[i], state, pod, nodeName) + if verboseLogging { + logger := logger.WithName(f.reservePlugins[i].Name()) + ctx := klog.NewContext(ctx, logger) + f.runReservePluginUnreserve(ctx, f.reservePlugins[i], state, pod, nodeName) + } else { + f.runReservePluginUnreserve(ctx, f.reservePlugins[i], state, pod, nodeName) + } } } @@ -1198,11 +1304,21 @@ func (f *frameworkImpl) RunPermitPlugins(ctx context.Context, state *framework.C }() pluginsWaitTime := make(map[string]time.Duration) statusCode := framework.Success - logger := klog.FromContext(ctx).WithName("Permit").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + logger := klog.FromContext(ctx) + verboseLogging := logger.V(3).Enabled() + if verboseLogging { + logger = logger.WithName("Permit").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) + } for _, pl := range f.permitPlugins { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status, timeout := f.runPermitPlugin(ctx, pl, state, pod, nodeName) + var status *framework.Status + var timeout time.Duration + if verboseLogging { + logger := logger.WithName(pl.Name()) + ctx := klog.NewContext(ctx, logger) + status, timeout = f.runPermitPlugin(ctx, pl, state, pod, nodeName) + } else { + status, timeout = f.runPermitPlugin(ctx, pl, state, pod, nodeName) + } if !status.IsSuccess() { if status.IsUnschedulable() { logger.V(4).Info("Pod rejected by plugin", "status", status.Message()) From f7ac1e31be7fe27730b6a7dace2fdb35490e5f5c Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 31 Jan 2022 15:33:50 +0100 Subject: [PATCH 5/5] scheduler: only add pod and node, no WithName --- pkg/scheduler/framework/runtime/framework.go | 182 ++----------------- 1 file changed, 15 insertions(+), 167 deletions(-) diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 553e0c94772f..7a73bd162ebb 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -613,21 +613,8 @@ func (f *frameworkImpl) RunPreFilterPlugins(ctx context.Context, state *framewor }() var result *framework.PreFilterResult var pluginsWithNodes []string - logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("PreFilter").WithValues("pod", klog.KObj(pod)) - } for _, pl := range f.preFilterPlugins { - var r *framework.PreFilterResult - var s *framework.Status - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - r, s = f.runPreFilterPlugin(ctx, pl, state, pod) - } else { - r, s = f.runPreFilterPlugin(ctx, pl, state, pod) - } + r, s := f.runPreFilterPlugin(ctx, pl, state, pod) if !s.IsSuccess() { s.SetFailedPlugin(pl.Name()) if s.IsUnschedulable() { @@ -672,24 +659,14 @@ func (f *frameworkImpl) RunPreFilterExtensionAddPod( nodeInfo *framework.NodeInfo, ) (status *framework.Status) { logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("AddPod").WithValues("pod", klog.KObj(podToSchedule), "node", klog.KObj(nodeInfo.Node())) - } for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue } - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runPreFilterExtensionAddPod(ctx, pl, state, podToSchedule, podInfoToAdd, nodeInfo) - } else { - status = f.runPreFilterExtensionAddPod(ctx, pl, state, podToSchedule, podInfoToAdd, nodeInfo) - } + status = f.runPreFilterExtensionAddPod(ctx, pl, state, podToSchedule, podInfoToAdd, nodeInfo) if !status.IsSuccess() { err := status.AsError() - logger.Error(err, "Plugin failed", "plugin", pl.Name()) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running AddPod on PreFilter plugin %q: %w", pl.Name(), err)) } } @@ -718,21 +695,11 @@ func (f *frameworkImpl) RunPreFilterExtensionRemovePod( nodeInfo *framework.NodeInfo, ) (status *framework.Status) { logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("RemovePod").WithValues("pod", klog.KObj(podToSchedule), "node", klog.KObj(nodeInfo.Node())) - } for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue } - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runPreFilterExtensionRemovePod(ctx, pl, state, podToSchedule, podInfoToRemove, nodeInfo) - } else { - status = f.runPreFilterExtensionRemovePod(ctx, pl, state, podToSchedule, podInfoToRemove, nodeInfo) - } + status = f.runPreFilterExtensionRemovePod(ctx, pl, state, podToSchedule, podInfoToRemove, nodeInfo) if !status.IsSuccess() { err := status.AsError() logger.Error(err, "Plugin failed") @@ -764,20 +731,8 @@ func (f *frameworkImpl) RunFilterPlugins( nodeInfo *framework.NodeInfo, ) framework.PluginToStatus { statuses := make(framework.PluginToStatus) - logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("Filter").WithValues("pod", klog.KObj(pod), "node", klog.KObj(nodeInfo.Node())) - } for _, pl := range f.filterPlugins { - var pluginStatus *framework.Status - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - pluginStatus = f.runFilterPlugin(ctx, pl, state, pod, nodeInfo) - } else { - pluginStatus = f.runFilterPlugin(ctx, pl, state, pod, nodeInfo) - } + pluginStatus := f.runFilterPlugin(ctx, pl, state, pod, nodeInfo) if !pluginStatus.IsSuccess() { if !pluginStatus.IsUnschedulable() { // Filter plugins are not supposed to return any status other than @@ -811,24 +766,11 @@ func (f *frameworkImpl) RunPostFilterPlugins(ctx context.Context, state *framewo metrics.FrameworkExtensionPointDuration.WithLabelValues(postFilter, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("PostFilter").WithValues("pod", klog.KObj(pod)) - } statuses := make(framework.PluginToStatus) // `result` records the last meaningful(non-noop) PostFilterResult. var result *framework.PostFilterResult for _, pl := range f.postFilterPlugins { - var r *framework.PostFilterResult - var s *framework.Status - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - r, s = f.runPostFilterPlugin(ctx, pl, state, pod, filteredNodeStatusMap) - } else { - r, s = f.runPostFilterPlugin(ctx, pl, state, pod, filteredNodeStatusMap) - } + r, s := f.runPostFilterPlugin(ctx, pl, state, pod, filteredNodeStatusMap) if s.IsSuccess() { return r, s } else if !s.IsUnschedulable() { @@ -885,12 +827,6 @@ func (f *frameworkImpl) RunFilterPluginsWithNominatedPods(ctx context.Context, s // the nominated pods are treated as not running. We can't just assume the // nominated pods are running because they are not running right now and in fact, // they may end up getting scheduled to a different node. - logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("NominatedPods") - ctx = klog.NewContext(ctx, logger) - } for i := 0; i < 2; i++ { stateToUse := state nodeInfoToUse := info @@ -955,19 +891,8 @@ func (f *frameworkImpl) RunPreScorePlugins( defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(preScore, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("PreScore").WithValues("pod", klog.KObj(pod)) - } for _, pl := range f.preScorePlugins { - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runPreScorePlugin(ctx, pl, state, pod, nodes) - } else { - status = f.runPreScorePlugin(ctx, pl, state, pod, nodes) - } + status = f.runPreScorePlugin(ctx, pl, state, pod, nodes) if !status.IsSuccess() { return framework.AsStatus(fmt.Errorf("running PreScore plugin %q: %w", pl.Name(), status.AsError())) } @@ -1003,23 +928,10 @@ func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.Cy errCh := parallelize.NewErrorChannel() // Run Score method for each node in parallel. - logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("Score").WithValues("pod", klog.KObj(pod)) - } f.Parallelizer().Until(ctx, len(nodes), func(index int) { - nodeName := nodes[index].Name for _, pl := range f.scorePlugins { - var s int64 - var status *framework.Status - if verboseLogging { - logger := logger.WithName(pl.Name()).WithValues("node", klog.ObjectRef{Name: nodeName}) - ctx := klog.NewContext(ctx, logger) - s, status = f.runScorePlugin(ctx, pl, state, pod, nodeName) - } else { - s, status = f.runScorePlugin(ctx, pl, state, pod, nodeName) - } + nodeName := nodes[index].Name + s, status := f.runScorePlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendErrorWithCancel(err, cancel) @@ -1106,18 +1018,8 @@ func (f *frameworkImpl) RunPreBindPlugins(ctx context.Context, state *framework. metrics.FrameworkExtensionPointDuration.WithLabelValues(preBind, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("PreBind").WithValues("pod", klog.KObj(pod)).WithValues("node", klog.ObjectRef{Name: nodeName}) - } for _, pl := range f.preBindPlugins { - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runPreBindPlugin(ctx, pl, state, pod, nodeName) - } else { - status = f.runPreBindPlugin(ctx, pl, state, pod, nodeName) - } + status = f.runPreBindPlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := status.AsError() logger.Error(err, "Plugin failed") @@ -1147,18 +1049,8 @@ func (f *frameworkImpl) RunBindPlugins(ctx context.Context, state *framework.Cyc return framework.NewStatus(framework.Skip, "") } logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("Bind").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) - } for _, bp := range f.bindPlugins { - if verboseLogging { - logger := logger.WithName(bp.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runBindPlugin(ctx, bp, state, pod, nodeName) - } else { - status = f.runBindPlugin(ctx, bp, state, pod, nodeName) - } + status = f.runBindPlugin(ctx, bp, state, pod, nodeName) if status.IsSkip() { continue } @@ -1188,19 +1080,8 @@ func (f *frameworkImpl) RunPostBindPlugins(ctx context.Context, state *framework defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(postBind, framework.Success.String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("PostBind").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) - } for _, pl := range f.postBindPlugins { - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - f.runPostBindPlugin(ctx, pl, state, pod, nodeName) - } else { - f.runPostBindPlugin(ctx, pl, state, pod, nodeName) - } + f.runPostBindPlugin(ctx, pl, state, pod, nodeName) } } @@ -1225,18 +1106,8 @@ func (f *frameworkImpl) RunReservePluginsReserve(ctx context.Context, state *fra metrics.FrameworkExtensionPointDuration.WithLabelValues(reserve, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("Reserve").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) - } for _, pl := range f.reservePlugins { - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status = f.runReservePluginReserve(ctx, pl, state, pod, nodeName) - } else { - status = f.runReservePluginReserve(ctx, pl, state, pod, nodeName) - } + status = f.runReservePluginReserve(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := status.AsError() logger.Error(err, "Plugin failed") @@ -1265,19 +1136,8 @@ func (f *frameworkImpl) RunReservePluginsUnreserve(ctx context.Context, state *f }() // Execute the Unreserve operation of each reserve plugin in the // *reverse* order in which the Reserve operation was executed. - logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("Unreserve").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) - } for i := len(f.reservePlugins) - 1; i >= 0; i-- { - if verboseLogging { - logger := logger.WithName(f.reservePlugins[i].Name()) - ctx := klog.NewContext(ctx, logger) - f.runReservePluginUnreserve(ctx, f.reservePlugins[i], state, pod, nodeName) - } else { - f.runReservePluginUnreserve(ctx, f.reservePlugins[i], state, pod, nodeName) - } + f.runReservePluginUnreserve(ctx, f.reservePlugins[i], state, pod, nodeName) } } @@ -1305,20 +1165,8 @@ func (f *frameworkImpl) RunPermitPlugins(ctx context.Context, state *framework.C pluginsWaitTime := make(map[string]time.Duration) statusCode := framework.Success logger := klog.FromContext(ctx) - verboseLogging := logger.V(3).Enabled() - if verboseLogging { - logger = logger.WithName("Permit").WithValues("pod", klog.KObj(pod), "node", klog.ObjectRef{Name: nodeName}) - } for _, pl := range f.permitPlugins { - var status *framework.Status - var timeout time.Duration - if verboseLogging { - logger := logger.WithName(pl.Name()) - ctx := klog.NewContext(ctx, logger) - status, timeout = f.runPermitPlugin(ctx, pl, state, pod, nodeName) - } else { - status, timeout = f.runPermitPlugin(ctx, pl, state, pod, nodeName) - } + status, timeout := f.runPermitPlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { if status.IsUnschedulable() { logger.V(4).Info("Pod rejected by plugin", "status", status.Message())