diff --git a/cmd/kube-controller-manager/app/testing/testserver.go b/cmd/kube-controller-manager/app/testing/testserver.go index f2ce36dcf08c..3f5513178456 100644 --- a/cmd/kube-controller-manager/app/testing/testserver.go +++ b/cmd/kube-controller-manager/app/testing/testserver.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" + "k8s.io/klog/v2" "k8s.io/kubernetes/cmd/kube-controller-manager/app" kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" @@ -45,20 +46,14 @@ type TestServer struct { TmpDir string // Temp Dir used, by the apiserver } -// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie -type Logger interface { - Errorf(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) - Logf(format string, args ...interface{}) -} - // StartTestServer starts a kube-controller-manager. A rest client config and a tear-down func, // and location of the tmpdir are returned. // // Note: we return a tear-down func instead of a stop channel because the later will leak temporary // files that because Golang testing's call to os.Exit will not give a stop channel go routine // enough time to remove temporary files. -func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { +func StartTestServer(ctx context.Context, customFlags []string) (result TestServer, err error) { + logger := klog.FromContext(ctx) stopCh := make(chan struct{}) tearDown := func() { close(stopCh) @@ -97,7 +92,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } s.SecureServing.ServerCert.CertDirectory = result.TmpDir - t.Logf("kube-controller-manager will listen securely on port %d...", s.SecureServing.BindPort) + logger.Info("kube-controller-manager will listen securely", "port", s.SecureServing.BindPort) } config, err := s.Config(all, disabled) @@ -112,7 +107,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } }(stopCh) - t.Logf("Waiting for /healthz to be ok...") + logger.Info("Waiting for /healthz to be ok...") client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) @@ -146,14 +141,13 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t Logger, flags []string) *TestServer { - result, err := StartTestServer(t, flags) +func StartTestServerOrDie(ctx context.Context, flags []string) *TestServer { + result, err := StartTestServer(ctx, flags) if err == nil { return &result } - t.Fatalf("failed to launch server: %v", err) - return nil + panic(fmt.Errorf("failed to launch server: %v", err)) } func createListenerOnFreePort() (net.Listener, int, error) { diff --git a/cmd/kube-scheduler/app/options/configfile.go b/cmd/kube-scheduler/app/options/configfile.go index 84802da767fa..507b00cfb2fa 100644 --- a/cmd/kube-scheduler/app/options/configfile.go +++ b/cmd/kube-scheduler/app/options/configfile.go @@ -80,9 +80,9 @@ func encodeConfig(cfg *config.KubeSchedulerConfiguration) (*bytes.Buffer, error) } // LogOrWriteConfig logs the completed component config and writes it into the given file name as YAML, if either is enabled -func LogOrWriteConfig(fileName string, cfg *config.KubeSchedulerConfiguration, completedProfiles []config.KubeSchedulerProfile) error { - klogV := klog.V(2) - if !klogV.Enabled() && len(fileName) == 0 { +func LogOrWriteConfig(logger klog.Logger, fileName string, cfg *config.KubeSchedulerConfiguration, completedProfiles []config.KubeSchedulerProfile) error { + loggerV := logger.V(2) + if !loggerV.Enabled() && len(fileName) == 0 { return nil } cfg.Profiles = completedProfiles @@ -92,8 +92,8 @@ func LogOrWriteConfig(fileName string, cfg *config.KubeSchedulerConfiguration, c return err } - if klogV.Enabled() { - klogV.InfoS("Using component config", "config", buf.String()) + if loggerV.Enabled() { + loggerV.Info("Using component config", "config", buf.String()) } if len(fileName) > 0 { @@ -105,7 +105,7 @@ func LogOrWriteConfig(fileName string, cfg *config.KubeSchedulerConfiguration, c if _, err := io.Copy(configFile, buf); err != nil { return err } - klog.InfoS("Wrote configuration", "file", fileName) + logger.Info("Wrote configuration", "file", fileName) os.Exit(0) } return nil diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index c352ba664d0e..6e4c6411c245 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -108,6 +108,7 @@ for more information about scheduling and the kube-scheduler component.`, cliflag.SetUsageAndHelpFunc(cmd, *nfs, cols) if err := cmd.MarkFlagFilename("config", "yaml", "yml", "json"); err != nil { + // nolint:logcheck // TODO (?): return error instead of logging it here klog.ErrorS(err, "Failed to mark flag filename") } @@ -144,10 +145,12 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op // Run executes the scheduler based on the given configuration. It only returns on error or when context is done. func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched *scheduler.Scheduler) error { + logger := klog.FromContext(ctx) + // To help debugging, immediately log version - klog.InfoS("Starting Kubernetes Scheduler", "version", version.Get()) + logger.Info("Starting Kubernetes Scheduler", "version", version.Get()) - klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) + logger.Info("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) // Configz registration. if cz, err := configz.New("componentconfig"); err == nil { @@ -213,11 +216,11 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched * select { case <-ctx.Done(): // We were asked to terminate. Exit 0. - klog.InfoS("Requested to terminate, exiting") + logger.Info("Requested to terminate, exiting") os.Exit(0) default: // We lost the lock. - klog.ErrorS(nil, "Leaderelection lost") + logger.Error(nil, "Leaderelection lost") klog.FlushAndExit(klog.ExitFlushTimeout, 1) } }, @@ -326,11 +329,12 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions recorderFactory := getRecorderFactory(&cc) completedProfiles := make([]kubeschedulerconfig.KubeSchedulerProfile, 0) // Create the scheduler. - sched, err := scheduler.New(cc.Client, + sched, err := scheduler.New( + ctx, + cc.Client, cc.InformerFactory, cc.DynInformerFactory, recorderFactory, - ctx.Done(), scheduler.WithComponentConfigVersion(cc.ComponentConfig.TypeMeta.APIVersion), scheduler.WithKubeConfig(cc.KubeConfig), scheduler.WithProfiles(cc.ComponentConfig.Profiles...), @@ -349,7 +353,7 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions if err != nil { return nil, nil, err } - if err := options.LogOrWriteConfig(opts.WriteConfigTo, &cc.ComponentConfig, completedProfiles); err != nil { + if err := options.LogOrWriteConfig(klog.FromContext(ctx), opts.WriteConfigTo, &cc.ComponentConfig, completedProfiles); err != nil { return nil, nil, err } diff --git a/cmd/kube-scheduler/app/testing/testserver.go b/cmd/kube-scheduler/app/testing/testserver.go index e76b2204b23b..ed88a921926c 100644 --- a/cmd/kube-scheduler/app/testing/testserver.go +++ b/cmd/kube-scheduler/app/testing/testserver.go @@ -48,21 +48,15 @@ type TestServer struct { TmpDir string // Temp Dir used, by the apiserver } -// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie -type Logger interface { - Errorf(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) - Logf(format string, args ...interface{}) -} - // StartTestServer starts a kube-scheduler. A rest client config and a tear-down func, // and location of the tmpdir are returned. // // Note: we return a tear-down func instead of a stop channel because the later will leak temporary // files that because Golang testing's call to os.Exit will not give a stop channel go routine // enough time to remove temporary files. -func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { - ctx, cancel := context.WithCancel(context.Background()) +func StartTestServer(ctx context.Context, customFlags []string) (result TestServer, err error) { + logger := klog.FromContext(ctx) + ctx, cancel := context.WithCancel(ctx) var errCh chan error tearDown := func() { @@ -73,7 +67,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err if errCh != nil { err, ok := <-errCh if ok && err != nil { - klog.ErrorS(err, "Failed to shutdown test server clearly") + logger.Error(err, "Failed to shutdown test server clearly") } } if len(result.TmpDir) != 0 { @@ -108,7 +102,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } opts.SecureServing.ServerCert.CertDirectory = result.TmpDir - t.Logf("kube-scheduler will listen securely on port %d...", opts.SecureServing.BindPort) + logger.Info("kube-scheduler will listen securely", "port", opts.SecureServing.BindPort) } cc, sched, err := app.Setup(ctx, opts) @@ -124,7 +118,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } }(ctx) - t.Logf("Waiting for /healthz to be ok...") + logger.Info("Waiting for /healthz to be ok...") client, err := kubernetes.NewForConfig(cc.LoopbackClientConfig) if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) @@ -158,14 +152,13 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t Logger, flags []string) *TestServer { - result, err := StartTestServer(t, flags) +func StartTestServerOrDie(ctx context.Context, flags []string) *TestServer { + result, err := StartTestServer(ctx, flags) if err == nil { return &result } - t.Fatalf("failed to launch server: %v", err) - return nil + panic(fmt.Errorf("failed to launch server: %v", err)) } func createListenerOnFreePort() (net.Listener, int, error) { diff --git a/hack/logcheck.conf b/hack/logcheck.conf index 3ec1cf4f9a57..e792731e54e7 100644 --- a/hack/logcheck.conf +++ b/hack/logcheck.conf @@ -15,16 +15,15 @@ # Now enable it again for migrated packages. structured k8s.io/kubernetes/cmd/kube-proxy/.* -structured k8s.io/kubernetes/cmd/kube-scheduler/.* structured k8s.io/kubernetes/cmd/kubelet/.* structured k8s.io/kubernetes/pkg/kubelet/.* structured k8s.io/kubernetes/pkg/proxy/.* -structured k8s.io/kubernetes/pkg/scheduler/.* # The following packages have been migrated to contextual logging. # Packages matched here do not have to be listed above because # "contextual" implies "structured". -# TODO next: contextual k8s.io/kubernetes/pkg/scheduler/.* +contextual k8s.io/kubernetes/cmd/kube-scheduler/.* +contextual k8s.io/kubernetes/pkg/scheduler/.* # As long as contextual logging is alpha or beta, all WithName, WithValues, # NewContext calls have to go through klog. Once it is GA, we can lift diff --git a/pkg/scheduler/apis/config/v1beta2/default_plugins.go b/pkg/scheduler/apis/config/v1beta2/default_plugins.go index 0eb8af06e46d..b541b9699350 100644 --- a/pkg/scheduler/apis/config/v1beta2/default_plugins.go +++ b/pkg/scheduler/apis/config/v1beta2/default_plugins.go @@ -27,7 +27,7 @@ import ( ) // getDefaultPlugins returns the default set of plugins. -func getDefaultPlugins() *v1beta2.Plugins { +func getDefaultPlugins(logger klog.Logger) *v1beta2.Plugins { plugins := &v1beta2.Plugins{ QueueSort: v1beta2.PluginSet{ Enabled: []v1beta2.Plugin{ @@ -107,34 +107,34 @@ func getDefaultPlugins() *v1beta2.Plugins { }, }, } - applyFeatureGates(plugins) + applyFeatureGates(logger, plugins) return plugins } -func applyFeatureGates(config *v1beta2.Plugins) { +func applyFeatureGates(logger klog.Logger, config *v1beta2.Plugins) { if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) { config.Score.Enabled = append(config.Score.Enabled, v1beta2.Plugin{Name: names.VolumeBinding, Weight: pointer.Int32Ptr(1)}) } } // mergePlugins merges the custom set into the given default one, handling disabled sets. -func mergePlugins(defaultPlugins, customPlugins *v1beta2.Plugins) *v1beta2.Plugins { +func mergePlugins(logger klog.Logger, defaultPlugins, customPlugins *v1beta2.Plugins) *v1beta2.Plugins { if customPlugins == nil { return defaultPlugins } - defaultPlugins.QueueSort = mergePluginSet(defaultPlugins.QueueSort, customPlugins.QueueSort) - defaultPlugins.PreFilter = mergePluginSet(defaultPlugins.PreFilter, customPlugins.PreFilter) - defaultPlugins.Filter = mergePluginSet(defaultPlugins.Filter, customPlugins.Filter) - defaultPlugins.PostFilter = mergePluginSet(defaultPlugins.PostFilter, customPlugins.PostFilter) - defaultPlugins.PreScore = mergePluginSet(defaultPlugins.PreScore, customPlugins.PreScore) - defaultPlugins.Score = mergePluginSet(defaultPlugins.Score, customPlugins.Score) - defaultPlugins.Reserve = mergePluginSet(defaultPlugins.Reserve, customPlugins.Reserve) - defaultPlugins.Permit = mergePluginSet(defaultPlugins.Permit, customPlugins.Permit) - defaultPlugins.PreBind = mergePluginSet(defaultPlugins.PreBind, customPlugins.PreBind) - defaultPlugins.Bind = mergePluginSet(defaultPlugins.Bind, customPlugins.Bind) - defaultPlugins.PostBind = mergePluginSet(defaultPlugins.PostBind, customPlugins.PostBind) + defaultPlugins.QueueSort = mergePluginSet(logger, defaultPlugins.QueueSort, customPlugins.QueueSort) + defaultPlugins.PreFilter = mergePluginSet(logger, defaultPlugins.PreFilter, customPlugins.PreFilter) + defaultPlugins.Filter = mergePluginSet(logger, defaultPlugins.Filter, customPlugins.Filter) + defaultPlugins.PostFilter = mergePluginSet(logger, defaultPlugins.PostFilter, customPlugins.PostFilter) + defaultPlugins.PreScore = mergePluginSet(logger, defaultPlugins.PreScore, customPlugins.PreScore) + defaultPlugins.Score = mergePluginSet(logger, defaultPlugins.Score, customPlugins.Score) + defaultPlugins.Reserve = mergePluginSet(logger, defaultPlugins.Reserve, customPlugins.Reserve) + defaultPlugins.Permit = mergePluginSet(logger, defaultPlugins.Permit, customPlugins.Permit) + defaultPlugins.PreBind = mergePluginSet(logger, defaultPlugins.PreBind, customPlugins.PreBind) + defaultPlugins.Bind = mergePluginSet(logger, defaultPlugins.Bind, customPlugins.Bind) + defaultPlugins.PostBind = mergePluginSet(logger, defaultPlugins.PostBind, customPlugins.PostBind) return defaultPlugins } @@ -143,7 +143,7 @@ type pluginIndex struct { plugin v1beta2.Plugin } -func mergePluginSet(defaultPluginSet, customPluginSet v1beta2.PluginSet) v1beta2.PluginSet { +func mergePluginSet(logger klog.Logger, defaultPluginSet, customPluginSet v1beta2.PluginSet) v1beta2.PluginSet { disabledPlugins := sets.NewString() enabledCustomPlugins := make(map[string]pluginIndex) // replacedPluginIndex is a set of index of plugins, which have replaced the default plugins. @@ -162,7 +162,7 @@ func mergePluginSet(defaultPluginSet, customPluginSet v1beta2.PluginSet) v1beta2 } // The default plugin is explicitly re-configured, update the default plugin accordingly. if customPlugin, ok := enabledCustomPlugins[defaultEnabledPlugin.Name]; ok { - klog.InfoS("Default plugin is explicitly re-configured; overriding", "plugin", defaultEnabledPlugin.Name) + logger.Info("Default plugin is explicitly re-configured; overriding", "plugin", defaultEnabledPlugin.Name) // Update the default plugin in place to preserve order. defaultEnabledPlugin = customPlugin.plugin replacedPluginIndex.Insert(customPlugin.index) diff --git a/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go b/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go index 6909fc740416..f9069010acd5 100644 --- a/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go +++ b/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta2" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" "k8s.io/utils/pointer" @@ -117,11 +118,12 @@ func TestApplyFeatureGates(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) for k, v := range test.features { defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)() } - gotConfig := getDefaultPlugins() + gotConfig := getDefaultPlugins(logger) if diff := cmp.Diff(test.wantConfig, gotConfig); diff != "" { t.Errorf("unexpected config diff (-want, +got): %s", diff) } @@ -370,7 +372,8 @@ func TestMergePlugins(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.defaultPlugins = mergePlugins(test.defaultPlugins, test.customPlugins) + logger, _ := ktesting.NewTestContext(t) + test.defaultPlugins = mergePlugins(logger, test.defaultPlugins, test.customPlugins) if d := cmp.Diff(test.expectedPlugins, test.defaultPlugins); d != "" { t.Fatalf("plugins mismatch (-want +got):\n%s", d) } diff --git a/pkg/scheduler/apis/config/v1beta2/defaults.go b/pkg/scheduler/apis/config/v1beta2/defaults.go index 60ef8f065a84..f76d5e964e5b 100644 --- a/pkg/scheduler/apis/config/v1beta2/defaults.go +++ b/pkg/scheduler/apis/config/v1beta2/defaults.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/klog/v2" "k8s.io/kube-scheduler/config/v1beta2" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -63,9 +64,9 @@ func pluginsNames(p *v1beta2.Plugins) []string { return n.List() } -func setDefaults_KubeSchedulerProfile(prof *v1beta2.KubeSchedulerProfile) { +func setDefaults_KubeSchedulerProfile(logger klog.Logger, prof *v1beta2.KubeSchedulerProfile) { // Set default plugins. - prof.Plugins = mergePlugins(getDefaultPlugins(), prof.Plugins) + prof.Plugins = mergePlugins(logger, getDefaultPlugins(logger), prof.Plugins) // Set default plugin configs. scheme := GetPluginArgConversionScheme() @@ -101,6 +102,7 @@ func setDefaults_KubeSchedulerProfile(prof *v1beta2.KubeSchedulerProfile) { // SetDefaults_KubeSchedulerConfiguration sets additional defaults func SetDefaults_KubeSchedulerConfiguration(obj *v1beta2.KubeSchedulerConfiguration) { + logger := klog.TODO() // called by generated code that doesn't pass a logger if obj.Parallelism == nil { obj.Parallelism = pointer.Int32Ptr(16) } @@ -117,7 +119,7 @@ func SetDefaults_KubeSchedulerConfiguration(obj *v1beta2.KubeSchedulerConfigurat // Add the default set of plugins and apply the configuration. for i := range obj.Profiles { prof := &obj.Profiles[i] - setDefaults_KubeSchedulerProfile(prof) + setDefaults_KubeSchedulerProfile(logger, prof) } if obj.PercentageOfNodesToScore == nil { diff --git a/pkg/scheduler/apis/config/v1beta2/defaults_test.go b/pkg/scheduler/apis/config/v1beta2/defaults_test.go index d25b5309365c..0687cd248293 100644 --- a/pkg/scheduler/apis/config/v1beta2/defaults_test.go +++ b/pkg/scheduler/apis/config/v1beta2/defaults_test.go @@ -30,6 +30,7 @@ import ( componentbaseconfig "k8s.io/component-base/config/v1alpha1" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta2" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" @@ -115,6 +116,7 @@ var pluginConfigs = []v1beta2.PluginConfig{ } func TestSchedulerDefaults(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) enable := true tests := []struct { name string @@ -149,7 +151,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta2.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, @@ -187,7 +189,7 @@ func TestSchedulerDefaults(t *testing.T) { Profiles: []v1beta2.KubeSchedulerProfile{ { SchedulerName: pointer.StringPtr("default-scheduler"), - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs}, }, }, @@ -242,7 +244,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta2.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: []v1beta2.PluginConfig{ {Name: "FooPlugin"}, { @@ -435,7 +437,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta2.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, diff --git a/pkg/scheduler/apis/config/v1beta3/default_plugins.go b/pkg/scheduler/apis/config/v1beta3/default_plugins.go index ca360b78260f..fc245f77b758 100644 --- a/pkg/scheduler/apis/config/v1beta3/default_plugins.go +++ b/pkg/scheduler/apis/config/v1beta3/default_plugins.go @@ -25,7 +25,7 @@ import ( ) // getDefaultPlugins returns the default set of plugins. -func getDefaultPlugins() *v1beta3.Plugins { +func getDefaultPlugins(logger klog.Logger) *v1beta3.Plugins { plugins := &v1beta3.Plugins{ MultiPoint: v1beta3.PluginSet{ Enabled: []v1beta3.Plugin{ @@ -57,23 +57,23 @@ func getDefaultPlugins() *v1beta3.Plugins { } // mergePlugins merges the custom set into the given default one, handling disabled sets. -func mergePlugins(defaultPlugins, customPlugins *v1beta3.Plugins) *v1beta3.Plugins { +func mergePlugins(logger klog.Logger, defaultPlugins, customPlugins *v1beta3.Plugins) *v1beta3.Plugins { if customPlugins == nil { return defaultPlugins } - defaultPlugins.MultiPoint = mergePluginSet(defaultPlugins.MultiPoint, customPlugins.MultiPoint) - defaultPlugins.QueueSort = mergePluginSet(defaultPlugins.QueueSort, customPlugins.QueueSort) - defaultPlugins.PreFilter = mergePluginSet(defaultPlugins.PreFilter, customPlugins.PreFilter) - defaultPlugins.Filter = mergePluginSet(defaultPlugins.Filter, customPlugins.Filter) - defaultPlugins.PostFilter = mergePluginSet(defaultPlugins.PostFilter, customPlugins.PostFilter) - defaultPlugins.PreScore = mergePluginSet(defaultPlugins.PreScore, customPlugins.PreScore) - defaultPlugins.Score = mergePluginSet(defaultPlugins.Score, customPlugins.Score) - defaultPlugins.Reserve = mergePluginSet(defaultPlugins.Reserve, customPlugins.Reserve) - defaultPlugins.Permit = mergePluginSet(defaultPlugins.Permit, customPlugins.Permit) - defaultPlugins.PreBind = mergePluginSet(defaultPlugins.PreBind, customPlugins.PreBind) - defaultPlugins.Bind = mergePluginSet(defaultPlugins.Bind, customPlugins.Bind) - defaultPlugins.PostBind = mergePluginSet(defaultPlugins.PostBind, customPlugins.PostBind) + defaultPlugins.MultiPoint = mergePluginSet(logger, defaultPlugins.MultiPoint, customPlugins.MultiPoint) + defaultPlugins.QueueSort = mergePluginSet(logger, defaultPlugins.QueueSort, customPlugins.QueueSort) + defaultPlugins.PreFilter = mergePluginSet(logger, defaultPlugins.PreFilter, customPlugins.PreFilter) + defaultPlugins.Filter = mergePluginSet(logger, defaultPlugins.Filter, customPlugins.Filter) + defaultPlugins.PostFilter = mergePluginSet(logger, defaultPlugins.PostFilter, customPlugins.PostFilter) + defaultPlugins.PreScore = mergePluginSet(logger, defaultPlugins.PreScore, customPlugins.PreScore) + defaultPlugins.Score = mergePluginSet(logger, defaultPlugins.Score, customPlugins.Score) + defaultPlugins.Reserve = mergePluginSet(logger, defaultPlugins.Reserve, customPlugins.Reserve) + defaultPlugins.Permit = mergePluginSet(logger, defaultPlugins.Permit, customPlugins.Permit) + defaultPlugins.PreBind = mergePluginSet(logger, defaultPlugins.PreBind, customPlugins.PreBind) + defaultPlugins.Bind = mergePluginSet(logger, defaultPlugins.Bind, customPlugins.Bind) + defaultPlugins.PostBind = mergePluginSet(logger, defaultPlugins.PostBind, customPlugins.PostBind) return defaultPlugins } @@ -82,7 +82,7 @@ type pluginIndex struct { plugin v1beta3.Plugin } -func mergePluginSet(defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3.PluginSet { +func mergePluginSet(logger klog.Logger, defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3.PluginSet { disabledPlugins := sets.NewString() enabledCustomPlugins := make(map[string]pluginIndex) // replacedPluginIndex is a set of index of plugins, which have replaced the default plugins. @@ -114,7 +114,7 @@ func mergePluginSet(defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3 } // The default plugin is explicitly re-configured, update the default plugin accordingly. if customPlugin, ok := enabledCustomPlugins[defaultEnabledPlugin.Name]; ok { - klog.InfoS("Default plugin is explicitly re-configured; overriding", "plugin", defaultEnabledPlugin.Name) + logger.Info("Default plugin is explicitly re-configured; overriding", "plugin", defaultEnabledPlugin.Name) // Update the default plugin in place to preserve order. defaultEnabledPlugin = customPlugin.plugin replacedPluginIndex.Insert(customPlugin.index) diff --git a/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go b/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go index 05d0a33d5d3b..b0ebaca375d2 100644 --- a/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go +++ b/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go @@ -19,11 +19,13 @@ package v1beta3 import ( "testing" + "k8s.io/kube-scheduler/config/v1beta3" + "github.com/google/go-cmp/cmp" "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kube-scheduler/config/v1beta3" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" "k8s.io/utils/pointer" ) @@ -67,11 +69,12 @@ func TestApplyFeatureGates(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) for k, v := range test.features { defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)() } - gotConfig := getDefaultPlugins() + gotConfig := getDefaultPlugins(logger) if diff := cmp.Diff(test.wantConfig, gotConfig); diff != "" { t.Errorf("unexpected config diff (-want, +got): %s", diff) } @@ -514,7 +517,8 @@ func TestMergePlugins(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - gotPlugins := mergePlugins(test.defaultPlugins, test.customPlugins) + logger, _ := ktesting.NewTestContext(t) + gotPlugins := mergePlugins(logger, test.defaultPlugins, test.customPlugins) if d := cmp.Diff(test.expectedPlugins, gotPlugins); d != "" { t.Fatalf("plugins mismatch (-want +got):\n%s", d) } diff --git a/pkg/scheduler/apis/config/v1beta3/defaults.go b/pkg/scheduler/apis/config/v1beta3/defaults.go index 0ff3781a4211..710e57eea3ca 100644 --- a/pkg/scheduler/apis/config/v1beta3/defaults.go +++ b/pkg/scheduler/apis/config/v1beta3/defaults.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/klog/v2" "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -64,9 +65,9 @@ func pluginsNames(p *v1beta3.Plugins) []string { return n.List() } -func setDefaults_KubeSchedulerProfile(prof *v1beta3.KubeSchedulerProfile) { +func setDefaults_KubeSchedulerProfile(logger klog.Logger, prof *v1beta3.KubeSchedulerProfile) { // Set default plugins. - prof.Plugins = mergePlugins(getDefaultPlugins(), prof.Plugins) + prof.Plugins = mergePlugins(logger, getDefaultPlugins(logger), prof.Plugins) // Set default plugin configs. scheme := GetPluginArgConversionScheme() existingConfigs := sets.NewString() @@ -101,6 +102,7 @@ func setDefaults_KubeSchedulerProfile(prof *v1beta3.KubeSchedulerProfile) { // SetDefaults_KubeSchedulerConfiguration sets additional defaults func SetDefaults_KubeSchedulerConfiguration(obj *v1beta3.KubeSchedulerConfiguration) { + logger := klog.TODO() // called by generated code which doesn't pass a logger if obj.Parallelism == nil { obj.Parallelism = pointer.Int32Ptr(16) } @@ -117,7 +119,7 @@ func SetDefaults_KubeSchedulerConfiguration(obj *v1beta3.KubeSchedulerConfigurat // Add the default set of plugins and apply the configuration. for i := range obj.Profiles { prof := &obj.Profiles[i] - setDefaults_KubeSchedulerProfile(prof) + setDefaults_KubeSchedulerProfile(logger, prof) } if obj.PercentageOfNodesToScore == nil { diff --git a/pkg/scheduler/apis/config/v1beta3/defaults_test.go b/pkg/scheduler/apis/config/v1beta3/defaults_test.go index f1c2a0fa27a4..b5a8ea447297 100644 --- a/pkg/scheduler/apis/config/v1beta3/defaults_test.go +++ b/pkg/scheduler/apis/config/v1beta3/defaults_test.go @@ -30,6 +30,7 @@ import ( componentbaseconfig "k8s.io/component-base/config/v1alpha1" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" @@ -115,6 +116,7 @@ var pluginConfigs = []v1beta3.PluginConfig{ } func TestSchedulerDefaults(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) enable := true tests := []struct { name string @@ -149,7 +151,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta3.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, @@ -187,7 +189,7 @@ func TestSchedulerDefaults(t *testing.T) { Profiles: []v1beta3.KubeSchedulerProfile{ { SchedulerName: pointer.StringPtr("default-scheduler"), - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs}, }, }, @@ -242,7 +244,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta3.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: []v1beta3.PluginConfig{ {Name: "FooPlugin"}, { @@ -393,7 +395,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta3.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, @@ -430,7 +432,7 @@ func TestSchedulerDefaults(t *testing.T) { PodMaxBackoffSeconds: pointer.Int64Ptr(10), Profiles: []v1beta3.KubeSchedulerProfile{ { - Plugins: getDefaultPlugins(), + Plugins: getDefaultPlugins(logger), PluginConfig: pluginConfigs, SchedulerName: pointer.StringPtr("default-scheduler"), }, diff --git a/pkg/scheduler/eventhandlers.go b/pkg/scheduler/eventhandlers.go index 0c39dcde2775..c7fc3eae75c9 100644 --- a/pkg/scheduler/eventhandlers.go +++ b/pkg/scheduler/eventhandlers.go @@ -41,9 +41,10 @@ import ( ) func (sched *Scheduler) onStorageClassAdd(obj interface{}) { + logger := sched.logger sc, ok := obj.(*storagev1.StorageClass) if !ok { - klog.ErrorS(nil, "Cannot convert to *storagev1.StorageClass", "obj", obj) + logger.Error(nil, "Cannot convert to *storagev1.StorageClass", "obj", obj) return } @@ -54,42 +55,45 @@ func (sched *Scheduler) onStorageClassAdd(obj interface{}) { // We don't need to invalidate cached results because results will not be // cached for pod that has unbound immediate PVCs. if sc.VolumeBindingMode != nil && *sc.VolumeBindingMode == storagev1.VolumeBindingWaitForFirstConsumer { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.StorageClassAdd, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.StorageClassAdd, nil) } } func (sched *Scheduler) addNodeToCache(obj interface{}) { + logger := sched.logger node, ok := obj.(*v1.Node) if !ok { - klog.ErrorS(nil, "Cannot convert to *v1.Node", "obj", obj) + logger.Error(nil, "Cannot convert to *v1.Node", "obj", obj) return } - nodeInfo := sched.Cache.AddNode(node) - klog.V(3).InfoS("Add event for node", "node", klog.KObj(node)) - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.NodeAdd, preCheckForNode(nodeInfo)) + nodeInfo := sched.Cache.AddNode(logger, node) + logger.V(3).Info("Add event for node", "node", klog.KObj(node)) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.NodeAdd, preCheckForNode(nodeInfo)) } func (sched *Scheduler) updateNodeInCache(oldObj, newObj interface{}) { + logger := sched.logger oldNode, ok := oldObj.(*v1.Node) if !ok { - klog.ErrorS(nil, "Cannot convert oldObj to *v1.Node", "oldObj", oldObj) + logger.Error(nil, "Cannot convert oldObj to *v1.Node", "oldObj", oldObj) return } newNode, ok := newObj.(*v1.Node) if !ok { - klog.ErrorS(nil, "Cannot convert newObj to *v1.Node", "newObj", newObj) + logger.Error(nil, "Cannot convert newObj to *v1.Node", "newObj", newObj) return } - nodeInfo := sched.Cache.UpdateNode(oldNode, newNode) + nodeInfo := sched.Cache.UpdateNode(logger, oldNode, newNode) // Only requeue unschedulable pods if the node became more schedulable. if event := nodeSchedulingPropertiesChange(newNode, oldNode); event != nil { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(*event, preCheckForNode(nodeInfo)) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, *event, preCheckForNode(nodeInfo)) } } func (sched *Scheduler) deleteNodeFromCache(obj interface{}) { + logger := sched.logger var node *v1.Node switch t := obj.(type) { case *v1.Node: @@ -98,23 +102,24 @@ func (sched *Scheduler) deleteNodeFromCache(obj interface{}) { var ok bool node, ok = t.Obj.(*v1.Node) if !ok { - klog.ErrorS(nil, "Cannot convert to *v1.Node", "obj", t.Obj) + logger.Error(nil, "Cannot convert to *v1.Node", "obj", t.Obj) return } default: - klog.ErrorS(nil, "Cannot convert to *v1.Node", "obj", t) + logger.Error(nil, "Cannot convert to *v1.Node", "obj", t) return } - klog.V(3).InfoS("Delete event for node", "node", klog.KObj(node)) - if err := sched.Cache.RemoveNode(node); err != nil { - klog.ErrorS(err, "Scheduler cache RemoveNode failed") + logger.V(3).Info("Delete event for node", "node", klog.KObj(node)) + if err := sched.Cache.RemoveNode(logger, node); err != nil { + logger.Error(err, "Scheduler cache RemoveNode failed") } } func (sched *Scheduler) addPodToSchedulingQueue(obj interface{}) { + logger := sched.logger pod := obj.(*v1.Pod) - klog.V(3).InfoS("Add event for unscheduled pod", "pod", klog.KObj(pod)) - if err := sched.SchedulingQueue.Add(pod); err != nil { + logger.V(3).Info("Add event for unscheduled pod", "pod", klog.KObj(pod)) + if err := sched.SchedulingQueue.Add(logger, pod); err != nil { utilruntime.HandleError(fmt.Errorf("unable to queue %T: %v", obj, err)) } } @@ -135,7 +140,8 @@ func (sched *Scheduler) updatePodInSchedulingQueue(oldObj, newObj interface{}) { return } - if err := sched.SchedulingQueue.Update(oldPod, newPod); err != nil { + logger := sched.logger + if err := sched.SchedulingQueue.Update(logger, oldPod, newPod); err != nil { utilruntime.HandleError(fmt.Errorf("unable to update %T: %v", newObj, err)) } } @@ -156,61 +162,65 @@ func (sched *Scheduler) deletePodFromSchedulingQueue(obj interface{}) { utilruntime.HandleError(fmt.Errorf("unable to handle object in %T: %T", sched, obj)) return } - klog.V(3).InfoS("Delete event for unscheduled pod", "pod", klog.KObj(pod)) - if err := sched.SchedulingQueue.Delete(pod); err != nil { + logger := sched.logger + logger.V(3).Info("Delete event for unscheduled pod", "pod", klog.KObj(pod)) + if err := sched.SchedulingQueue.Delete(logger, pod); err != nil { utilruntime.HandleError(fmt.Errorf("unable to dequeue %T: %v", obj, err)) } fwk, err := sched.frameworkForPod(pod) if err != nil { // This shouldn't happen, because we only accept for scheduling the pods // which specify a scheduler name that matches one of the profiles. - klog.ErrorS(err, "Unable to get profile", "pod", klog.KObj(pod)) + logger.Error(err, "Unable to get profile", "pod", klog.KObj(pod)) return } // If a waiting pod is rejected, it indicates it's previously assumed and we're // removing it from the scheduler cache. In this case, signal a AssignedPodDelete // event to immediately retry some unscheduled Pods. if fwk.RejectWaitingPod(pod.UID) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.AssignedPodDelete, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.AssignedPodDelete, nil) } } func (sched *Scheduler) addPodToCache(obj interface{}) { + logger := sched.logger pod, ok := obj.(*v1.Pod) if !ok { - klog.ErrorS(nil, "Cannot convert to *v1.Pod", "obj", obj) + logger.Error(nil, "Cannot convert to *v1.Pod", "obj", obj) return } - klog.V(3).InfoS("Add event for scheduled pod", "pod", klog.KObj(pod)) + logger.V(3).Info("Add event for scheduled pod", "pod", klog.KObj(pod)) - if err := sched.Cache.AddPod(pod); err != nil { - klog.ErrorS(err, "Scheduler cache AddPod failed", "pod", klog.KObj(pod)) + if err := sched.Cache.AddPod(logger, pod); err != nil { + logger.Error(err, "Scheduler cache AddPod failed", "pod", klog.KObj(pod)) } - sched.SchedulingQueue.AssignedPodAdded(pod) + sched.SchedulingQueue.AssignedPodAdded(logger, pod) } func (sched *Scheduler) updatePodInCache(oldObj, newObj interface{}) { + logger := sched.logger oldPod, ok := oldObj.(*v1.Pod) if !ok { - klog.ErrorS(nil, "Cannot convert oldObj to *v1.Pod", "oldObj", oldObj) + logger.Error(nil, "Cannot convert oldObj to *v1.Pod", "oldObj", oldObj) return } newPod, ok := newObj.(*v1.Pod) if !ok { - klog.ErrorS(nil, "Cannot convert newObj to *v1.Pod", "newObj", newObj) + logger.Error(nil, "Cannot convert newObj to *v1.Pod", "newObj", newObj) return } - klog.V(4).InfoS("Update event for scheduled pod", "pod", klog.KObj(oldPod)) + logger.V(4).Info("Update event for scheduled pod", "pod", klog.KObj(oldPod)) - if err := sched.Cache.UpdatePod(oldPod, newPod); err != nil { - klog.ErrorS(err, "Scheduler cache UpdatePod failed", "pod", klog.KObj(oldPod)) + if err := sched.Cache.UpdatePod(logger, oldPod, newPod); err != nil { + logger.Error(err, "Scheduler cache UpdatePod failed", "pod", klog.KObj(oldPod)) } - sched.SchedulingQueue.AssignedPodUpdated(newPod) + sched.SchedulingQueue.AssignedPodUpdated(logger, newPod) } func (sched *Scheduler) deletePodFromCache(obj interface{}) { + logger := sched.logger var pod *v1.Pod switch t := obj.(type) { case *v1.Pod: @@ -219,19 +229,19 @@ func (sched *Scheduler) deletePodFromCache(obj interface{}) { var ok bool pod, ok = t.Obj.(*v1.Pod) if !ok { - klog.ErrorS(nil, "Cannot convert to *v1.Pod", "obj", t.Obj) + logger.Error(nil, "Cannot convert to *v1.Pod", "obj", t.Obj) return } default: - klog.ErrorS(nil, "Cannot convert to *v1.Pod", "obj", t) + logger.Error(nil, "Cannot convert to *v1.Pod", "obj", t) return } - klog.V(3).InfoS("Delete event for scheduled pod", "pod", klog.KObj(pod)) - if err := sched.Cache.RemovePod(pod); err != nil { - klog.ErrorS(err, "Scheduler cache RemovePod failed", "pod", klog.KObj(pod)) + logger.V(3).Info("Delete event for scheduled pod", "pod", klog.KObj(pod)) + if err := sched.Cache.RemovePod(logger, pod); err != nil { + logger.Error(err, "Scheduler cache RemovePod failed", "pod", klog.KObj(pod)) } - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.AssignedPodDelete, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.AssignedPodDelete, nil) } // assignedPod selects pods that are assigned (scheduled and running). @@ -252,6 +262,8 @@ func addAllEventHandlers( dynInformerFactory dynamicinformer.DynamicSharedInformerFactory, gvkMap map[framework.GVK]framework.ActionType, ) { + logger := sched.logger + // scheduled pod cache informerFactory.Core().V1().Pods().Informer().AddEventHandler( cache.FilteringResourceEventHandler{ @@ -320,19 +332,19 @@ func addAllEventHandlers( if at&framework.Add != 0 { evt := framework.ClusterEvent{Resource: gvk, ActionType: framework.Add, Label: fmt.Sprintf("%vAdd", shortGVK)} funcs.AddFunc = func(_ interface{}) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(evt, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, evt, nil) } } if at&framework.Update != 0 { evt := framework.ClusterEvent{Resource: gvk, ActionType: framework.Update, Label: fmt.Sprintf("%vUpdate", shortGVK)} funcs.UpdateFunc = func(_, _ interface{}) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(evt, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, evt, nil) } } if at&framework.Delete != 0 { evt := framework.ClusterEvent{Resource: gvk, ActionType: framework.Delete, Label: fmt.Sprintf("%vDelete", shortGVK)} funcs.DeleteFunc = func(_ interface{}) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(evt, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, evt, nil) } } return funcs @@ -388,7 +400,7 @@ func addAllEventHandlers( informerFactory.Storage().V1().StorageClasses().Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(_, _ interface{}) { - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.StorageClassUpdate, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, queue.StorageClassUpdate, nil) }, }, ) @@ -407,7 +419,7 @@ func addAllEventHandlers( // - foos.v1 (2 sections) // - foo.v1.example.com (the first section should be plural) if strings.Count(string(gvk), ".") < 2 { - klog.ErrorS(nil, "incorrect event registration", "gvk", gvk) + logger.Error(nil, "incorrect event registration", "gvk", gvk) continue } // Fall back to try dynamic informers. diff --git a/pkg/scheduler/eventhandlers_test.go b/pkg/scheduler/eventhandlers_test.go index 856d2302652d..841b5b6a2447 100644 --- a/pkg/scheduler/eventhandlers_test.go +++ b/pkg/scheduler/eventhandlers_test.go @@ -29,6 +29,7 @@ import ( storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2/ktesting" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -36,7 +37,6 @@ import ( dyfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" - "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename" @@ -220,11 +220,13 @@ func TestUpdatePodInCache(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() sched := &Scheduler{ - Cache: cache.New(ttl, ctx.Done()), + Cache: cache.New(ctx, ttl), SchedulingQueue: queue.NewTestQueue(ctx, nil), + logger: logger, } sched.addPodToCache(tt.oldObj) sched.updatePodInCache(tt.oldObj, tt.newObj) @@ -430,7 +432,8 @@ func TestAddAllEventHandlers(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() informerFactory := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 0) @@ -438,6 +441,7 @@ func TestAddAllEventHandlers(t *testing.T) { testSched := Scheduler{ StopEverything: ctx.Done(), SchedulingQueue: schedulingQueue, + logger: logger, } dynclient := dyfake.NewSimpleDynamicClient(scheme) diff --git a/pkg/scheduler/extender.go b/pkg/scheduler/extender.go index e29026ad1552..c6c6428f44f9 100644 --- a/pkg/scheduler/extender.go +++ b/pkg/scheduler/extender.go @@ -28,6 +28,7 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" restclient "k8s.io/client-go/rest" + "k8s.io/klog/v2" extenderv1 "k8s.io/kube-scheduler/extender/v1" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" @@ -163,6 +164,7 @@ func (h *HTTPExtender) SupportsPreemption() bool { // ProcessPreemption returns filtered candidate nodes and victims after running preemption logic in extender. func (h *HTTPExtender) ProcessPreemption( + logger klog.Logger, pod *v1.Pod, nodeNameToVictims map[string]*extenderv1.Victims, nodeInfos framework.NodeInfoLister, diff --git a/pkg/scheduler/extender_test.go b/pkg/scheduler/extender_test.go index 113f8c4906f8..63fd9e33f4ce 100644 --- a/pkg/scheduler/extender_test.go +++ b/pkg/scheduler/extender_test.go @@ -25,9 +25,9 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" clientsetfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/klog/v2/ktesting" extenderv1 "k8s.io/kube-scheduler/extender/v1" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" @@ -272,6 +272,9 @@ func TestSchedulerWithExtenders(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) @@ -279,21 +282,23 @@ func TestSchedulerWithExtenders(t *testing.T) { for ii := range test.extenders { extenders = append(extenders, &test.extenders[ii]) } - cache := internalcache.New(time.Duration(0), wait.NeverStop) + cache := internalcache.New(ctx, time.Duration(0)) for _, name := range test.nodes { - cache.AddNode(createNode(name)) + cache.AddNode(logger, createNode(name)) } fwk, err := st.NewFramework( test.registerPlugins, "", runtime.WithClientSet(client), runtime.WithInformerFactory(informerFactory), runtime.WithPodNominator(internalqueue.NewPodNominator(informerFactory.Core().V1().Pods().Lister())), + runtime.WithLogger(logger), ) if err != nil { t.Fatal(err) } scheduler := newScheduler( + logger, cache, extenders, nil, @@ -305,7 +310,7 @@ func TestSchedulerWithExtenders(t *testing.T) { emptySnapshot, schedulerapi.DefaultPercentageOfNodesToScore) podIgnored := &v1.Pod{} - result, err := scheduler.SchedulePod(context.Background(), fwk, framework.NewCycleState(), podIgnored) + result, err := scheduler.SchedulePod(ctx, fwk, framework.NewCycleState(), podIgnored) if test.expectsErr { if err == nil { t.Errorf("Unexpected non-error, result %+v", result) diff --git a/pkg/scheduler/framework/extender.go b/pkg/scheduler/framework/extender.go index d47c89bf1e40..4b1ca9297adb 100644 --- a/pkg/scheduler/framework/extender.go +++ b/pkg/scheduler/framework/extender.go @@ -18,6 +18,7 @@ package framework import ( v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" extenderv1 "k8s.io/kube-scheduler/extender/v1" ) @@ -58,6 +59,7 @@ type Extender interface { // 1. Subset of given candidate nodes after preemption phase of extender. // 2. A different set of victim pod for every given candidate node after preemption phase of extender. ProcessPreemption( + logger klog.Logger, pod *v1.Pod, nodeNameToVictims map[string]*extenderv1.Victims, nodeInfos NodeInfoLister, diff --git a/pkg/scheduler/framework/interface.go b/pkg/scheduler/framework/interface.go index feadee9af8dc..b07fa1246f72 100644 --- a/pkg/scheduler/framework/interface.go +++ b/pkg/scheduler/framework/interface.go @@ -35,6 +35,7 @@ import ( clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/events" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework/parallelize" ) @@ -621,6 +622,10 @@ type Handle interface { // Parallelizer returns a parallelizer holding parallelism for scheduler. Parallelizer() parallelize.Parallelizer + + // Logger returns a logger that can be used for background activities when + // the caller does not provide a context-specific logger. + Logger() klog.Logger } // PreFilterResult wraps needed info for scheduler framework to act upon PreFilter phase. @@ -690,13 +695,13 @@ func (ni *NominatingInfo) Mode() NominatingMode { type PodNominator interface { // AddNominatedPod adds the given pod to the nominator or // updates it if it already exists. - AddNominatedPod(pod *PodInfo, nominatingInfo *NominatingInfo) + AddNominatedPod(logger klog.Logger, pod *PodInfo, nominatingInfo *NominatingInfo) // DeleteNominatedPodIfExists deletes nominatedPod from internal cache. It's a no-op if it doesn't exist. - DeleteNominatedPodIfExists(pod *v1.Pod) + DeleteNominatedPodIfExists(logger klog.Logger, pod *v1.Pod) // UpdateNominatedPod updates the with . - UpdateNominatedPod(oldPod *v1.Pod, newPodInfo *PodInfo) + UpdateNominatedPod(logger klog.Logger, oldPod *v1.Pod, newPodInfo *PodInfo) // NominatedPodsForNode returns nominatedPods on the given node. - NominatedPodsForNode(nodeName string) []*PodInfo + NominatedPodsForNode(logger klog.Logger, nodeName string) []*PodInfo } // PluginsRunner abstracts operations to run some plugins. diff --git a/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go b/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go index e695e320dfaf..ead881874819 100644 --- a/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go +++ b/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go @@ -49,7 +49,8 @@ func (b DefaultBinder) Name() string { // Bind binds pods to nodes using the k8s client. func (b DefaultBinder) Bind(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) *framework.Status { - klog.V(3).InfoS("Attempting to bind pod to node", "pod", klog.KObj(p), "node", klog.KRef("", nodeName)) + logger := klog.FromContext(ctx) + logger.V(3).Info("Attempting to bind pod to node", "pod", klog.KObj(p), "node", klog.KRef("", nodeName)) binding := &v1.Binding{ ObjectMeta: metav1.ObjectMeta{Namespace: p.Namespace, Name: p.Name, UID: p.UID}, Target: v1.ObjectReference{Kind: "Node", Name: nodeName}, diff --git a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption.go b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption.go index 91fc9ef9ed00..acf6887242ce 100644 --- a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption.go +++ b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption.go @@ -141,8 +141,9 @@ func (pl *DefaultPreemption) SelectVictimsOnNode( nodeInfo *framework.NodeInfo, pdbs []*policy.PodDisruptionBudget) ([]*v1.Pod, int, *framework.Status) { var potentialVictims []*framework.PodInfo + logger := klog.FromContext(ctx) removePod := func(rpi *framework.PodInfo) error { - if err := nodeInfo.RemovePod(rpi.Pod); err != nil { + if err := nodeInfo.RemovePod(logger, rpi.Pod); err != nil { return err } status := pl.fh.RunPreFilterExtensionRemovePod(ctx, state, pod, rpi, nodeInfo) @@ -205,7 +206,7 @@ func (pl *DefaultPreemption) SelectVictimsOnNode( } rpi := pi.Pod victims = append(victims, rpi) - klog.V(5).InfoS("Pod is a potential preemption victim on node", "pod", klog.KObj(rpi), "node", klog.KObj(nodeInfo.Node())) + logger.V(5).Info("Pod is a potential preemption victim on node", "pod", klog.KObj(rpi), "node", klog.KObj(nodeInfo.Node())) } return fits, nil } diff --git a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go index ebbbcaf7291e..52118b6809d4 100644 --- a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go +++ b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go @@ -36,6 +36,7 @@ import ( clientsetfake "k8s.io/client-go/kubernetes/fake" clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/events" + "k8s.io/klog/v2/ktesting" kubeschedulerconfigv1beta2 "k8s.io/kube-scheduler/config/v1beta2" extenderv1 "k8s.io/kube-scheduler/extender/v1" "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -328,6 +329,7 @@ func TestPostFilter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) cs := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(cs, 0) podInformer := informerFactory.Core().V1().Pods().Informer() @@ -358,6 +360,7 @@ func TestPostFilter(t *testing.T) { frameworkruntime.WithPodNominator(internalqueue.NewPodNominator(informerFactory.Core().V1().Pods().Lister())), frameworkruntime.WithExtenders(extenders), frameworkruntime.WithSnapshotSharedLister(internalcache.NewSnapshot(tt.pods, tt.nodes)), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) @@ -1037,6 +1040,7 @@ func TestDryRunPreemption(t *testing.T) { labelKeys := []string{"hostname", "zone", "region"} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) nodes := make([]*v1.Node, len(tt.nodeNames)) fakeFilterRCMap := make(map[string]framework.Code, len(tt.nodeNames)) for i, nodeName := range tt.nodeNames { @@ -1089,6 +1093,7 @@ func TestDryRunPreemption(t *testing.T) { frameworkruntime.WithSnapshotSharedLister(snapshot), frameworkruntime.WithInformerFactory(informerFactory), frameworkruntime.WithParallelism(parallelism), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) @@ -1320,6 +1325,7 @@ func TestSelectBestCandidate(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) rand.Seed(4) nodes := make([]*v1.Node, len(tt.nodeNames)) for i, nodeName := range tt.nodeNames { @@ -1343,6 +1349,7 @@ func TestSelectBestCandidate(t *testing.T) { "", frameworkruntime.WithPodNominator(internalqueue.NewPodNominator(informerFactory.Core().V1().Pods().Lister())), frameworkruntime.WithSnapshotSharedLister(snapshot), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) @@ -1374,7 +1381,7 @@ func TestSelectBestCandidate(t *testing.T) { } offset, numCandidates := pl.GetOffsetAndNumCandidates(int32(len(nodeInfos))) candidates, _, _ := pe.DryRunPreemption(context.Background(), tt.pod, nodeInfos, nil, offset, numCandidates) - s := pe.SelectCandidate(candidates) + s := pe.SelectCandidate(logger, candidates) if s == nil || len(s.Name()) == 0 { return } @@ -1437,6 +1444,7 @@ func TestPodEligibleToPreemptOthers(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) var nodes []*v1.Node for _, n := range test.nodes { nodes = append(nodes, st.MakeNode().Name(n).Obj()) @@ -1447,6 +1455,7 @@ func TestPodEligibleToPreemptOthers(t *testing.T) { } f, err := st.NewFramework(registeredPlugins, "", frameworkruntime.WithSnapshotSharedLister(internalcache.NewSnapshot(test.pods, nodes)), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) @@ -1625,6 +1634,9 @@ func TestPreempt(t *testing.T) { labelKeys := []string{"hostname", "zone", "region"} for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) podInformer := informerFactory.Core().V1().Pods().Informer() @@ -1639,12 +1651,9 @@ func TestPreempt(t *testing.T) { return true, nil, nil }) - stop := make(chan struct{}) - defer close(stop) - - cache := internalcache.New(time.Duration(0), stop) + cache := internalcache.New(ctx, time.Duration(0)) for _, pod := range test.pods { - cache.AddPod(pod) + cache.AddPod(logger, pod) } cachedNodeInfoMap := map[string]*framework.NodeInfo{} nodes := make([]*v1.Node, len(test.nodeNames)) @@ -1657,7 +1666,7 @@ func TestPreempt(t *testing.T) { node.ObjectMeta.Labels[labelKeys[i]] = label } node.Name = node.ObjectMeta.Labels["hostname"] - cache.AddNode(node) + cache.AddNode(logger, node) nodes[i] = node // Set nodeInfo to extenders to mock extenders' cache for preemption. @@ -1684,6 +1693,7 @@ func TestPreempt(t *testing.T) { frameworkruntime.WithPodNominator(internalqueue.NewPodNominator(informerFactory.Core().V1().Pods().Lister())), frameworkruntime.WithSnapshotSharedLister(internalcache.NewSnapshot(test.pods, nodes)), frameworkruntime.WithInformerFactory(informerFactory), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) diff --git a/pkg/scheduler/framework/plugins/examples/stateful/stateful.go b/pkg/scheduler/framework/plugins/examples/stateful/stateful.go index 2eb22468da82..ecd7eb68a550 100644 --- a/pkg/scheduler/framework/plugins/examples/stateful/stateful.go +++ b/pkg/scheduler/framework/plugins/examples/stateful/stateful.go @@ -23,7 +23,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -82,9 +81,9 @@ func (mp *MultipointExample) PreBind(ctx context.Context, state *framework.Cycle } // New initializes a new plugin and returns it. -func New(config *runtime.Unknown, _ framework.Handle) (framework.Plugin, error) { +func New(config *runtime.Unknown, fh framework.Handle) (framework.Plugin, error) { if config == nil { - klog.ErrorS(nil, "MultipointExample configuration cannot be empty") + fh.Logger().Error(nil, "MultipointExample configuration cannot be empty") return nil, fmt.Errorf("MultipointExample configuration cannot be empty") } mp := MultipointExample{} diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/filtering.go b/pkg/scheduler/framework/plugins/interpodaffinity/filtering.go index 778a70f2c7bc..249bd11390ce 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/filtering.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/filtering.go @@ -159,7 +159,7 @@ func (pl *InterPodAffinity) getExistingAntiAffinityCounts(ctx context.Context, p nodeInfo := nodes[i] node := nodeInfo.Node() if node == nil { - klog.ErrorS(nil, "Node not found") + klog.FromContext(ctx).Error(nil, "Node not found") return } topoMap := make(topologyToMatchedTermCount) @@ -198,7 +198,7 @@ func (pl *InterPodAffinity) getIncomingAffinityAntiAffinityCounts(ctx context.Co nodeInfo := allNodes[i] node := nodeInfo.Node() if node == nil { - klog.ErrorS(nil, "Node not found") + klog.FromContext(ctx).Error(nil, "Node not found") return } affinity := make(topologyToMatchedTermCount) @@ -255,7 +255,8 @@ func (pl *InterPodAffinity) PreFilter(ctx context.Context, cycleState *framework return nil, framework.AsStatus(err) } } - s.namespaceLabels = GetNamespaceLabelsSnapshot(pod.Namespace, pl.nsLister) + logger := klog.FromContext(ctx) + s.namespaceLabels = GetNamespaceLabelsSnapshot(logger, pod.Namespace, pl.nsLister) s.existingAntiAffinityCounts = pl.getExistingAntiAffinityCounts(ctx, pod, s.namespaceLabels, nodesWithRequiredAntiAffinityPods) s.affinityCounts, s.antiAffinityCounts = pl.getIncomingAffinityAntiAffinityCounts(ctx, s.podInfo, allNodes) diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go b/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go index ce0aa6b6beda..8081c39b2f7b 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/filtering_test.go @@ -26,6 +26,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" plugintesting "k8s.io/kubernetes/pkg/scheduler/framework/plugins/testing" @@ -1434,10 +1435,11 @@ func TestGetTPMapMatchingIncomingAffinityAntiAffinity(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() snapshot := cache.NewSnapshot(tt.existingPods, tt.nodes) l, _ := snapshot.NodeInfos().List() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() p := plugintesting.SetupPluginWithInformers(ctx, t, New, &config.InterPodAffinityArgs{}, snapshot, nil) gotAffinityPodsMap, gotAntiAffinityPodsMap := p.(*InterPodAffinity).getIncomingAffinityAntiAffinityCounts(ctx, framework.NewPodInfo(tt.pod), l) if !reflect.DeepEqual(gotAffinityPodsMap, tt.wantAffinityPodsMap) { diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/plugin.go b/pkg/scheduler/framework/plugins/interpodaffinity/plugin.go index f7e84ce09d96..2ca14362f7b0 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/plugin.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/plugin.go @@ -122,12 +122,12 @@ func (pl *InterPodAffinity) mergeAffinityTermNamespacesIfNotEmpty(at *framework. // GetNamespaceLabelsSnapshot returns a snapshot of the labels associated with // the namespace. -func GetNamespaceLabelsSnapshot(ns string, nsLister listersv1.NamespaceLister) (nsLabels labels.Set) { +func GetNamespaceLabelsSnapshot(logger klog.Logger, ns string, nsLister listersv1.NamespaceLister) (nsLabels labels.Set) { podNS, err := nsLister.Get(ns) if err == nil { // Create and return snapshot of the labels. return labels.Merge(podNS.Labels, nil) } - klog.V(3).InfoS("getting namespace, assuming empty set of namespace labels", "namespace", ns, "err", err) + logger.V(3).Info("getting namespace, assuming empty set of namespace labels", "namespace", ns, "err", err) return } diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/scoring.go b/pkg/scheduler/framework/plugins/interpodaffinity/scoring.go index f333bb6e3ca8..058dac274916 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/scoring.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/scoring.go @@ -22,8 +22,9 @@ import ( "math" "sync/atomic" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -178,7 +179,8 @@ func (pl *InterPodAffinity) PreScore( return framework.AsStatus(fmt.Errorf("updating PreferredAntiAffinityTerms: %w", err)) } } - state.namespaceLabels = GetNamespaceLabelsSnapshot(pod.Namespace, pl.nsLister) + logger := klog.FromContext(pCtx) + state.namespaceLabels = GetNamespaceLabelsSnapshot(logger, pod.Namespace, pl.nsLister) topoScores := make([]scoreMap, len(allNodes)) index := int32(-1) diff --git a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go index 4f91937ba542..3904ea277013 100644 --- a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go @@ -23,6 +23,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" "k8s.io/kubernetes/pkg/scheduler/framework" @@ -54,12 +55,13 @@ func (ba *BalancedAllocation) Score(ctx context.Context, state *framework.CycleS return 0, framework.AsStatus(fmt.Errorf("getting node %q from Snapshot: %w", nodeName, err)) } + logger := klog.FromContext(ctx) // ba.score favors nodes with balanced resource usage rate. // It calculates the standard deviation for those resources and prioritizes the node based on how close the usage of those resources is to each other. // Detail: score = (1 - std) * MaxNodeScore, where std is calculated by the root square of Σ((fraction(i)-mean)^2)/len(resources) // The algorithm is partly inspired by: // "Wei Huang et al. An Energy Efficient Virtual Machine Placement Algorithm with Balanced Resource Utilization" - return ba.score(pod, nodeInfo) + return ba.score(logger, pod, nodeInfo) } // ScoreExtensions of the Score plugin. diff --git a/pkg/scheduler/framework/plugins/noderesources/fit.go b/pkg/scheduler/framework/plugins/noderesources/fit.go index e4b2a5044dd5..a91d97ae5aef 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" @@ -331,5 +332,6 @@ func (f *Fit) Score(ctx context.Context, state *framework.CycleState, pod *v1.Po return 0, framework.AsStatus(fmt.Errorf("getting node %q from Snapshot: %w", nodeName, err)) } - return f.score(pod, nodeInfo) + logger := klog.FromContext(ctx) + return f.score(logger, pod, nodeInfo) } diff --git a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go index 48eadc88ff09..50617fff1bde 100644 --- a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go @@ -45,6 +45,7 @@ type resourceToValueMap map[v1.ResourceName]int64 // score will use `scorer` function to calculate the score. func (r *resourceAllocationScorer) score( + logger klog.Logger, pod *v1.Pod, nodeInfo *framework.NodeInfo) (int64, *framework.Status) { node := nodeInfo.Node() @@ -58,7 +59,7 @@ func (r *resourceAllocationScorer) score( requested := make(resourceToValueMap) allocatable := make(resourceToValueMap) for resource := range r.resourceToWeightMap { - alloc, req := r.calculateResourceAllocatableRequest(nodeInfo, pod, resource) + alloc, req := r.calculateResourceAllocatableRequest(logger, nodeInfo, pod, resource) if alloc != 0 { // Only fill the extended resource entry when it's non-zero. allocatable[resource], requested[resource] = alloc, req @@ -67,7 +68,7 @@ func (r *resourceAllocationScorer) score( score := r.scorer(requested, allocatable) - klog.V(10).InfoS("Listing internal info for allocatable resources, requested resources and score", "pod", + logger.V(10).Info("Listing internal info for allocatable resources, requested resources and score", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceAllocationScorer", r.Name, "allocatableResource", allocatable, "requestedResource", requested, "resourceScore", score, ) @@ -79,7 +80,7 @@ func (r *resourceAllocationScorer) score( // - 1st param: quantity of allocatable resource on the node. // - 2nd param: aggregated quantity of requested resource on the node. // Note: if it's an extended resource, and the pod doesn't request it, (0, 0) is returned. -func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(nodeInfo *framework.NodeInfo, pod *v1.Pod, resource v1.ResourceName) (int64, int64) { +func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(logger klog.Logger, nodeInfo *framework.NodeInfo, pod *v1.Pod, resource v1.ResourceName) (int64, int64) { requested := nodeInfo.NonZeroRequested if r.useRequested { requested = nodeInfo.Requested @@ -103,7 +104,7 @@ func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(nodeInfo return nodeInfo.Allocatable.ScalarResources[resource], (nodeInfo.Requested.ScalarResources[resource] + podRequest) } } - klog.V(10).InfoS("Requested resource is omitted for node score calculation", "resourceName", resource) + logger.V(10).Info("Requested resource is omitted for node score calculation", "resourceName", resource) return 0, 0 } diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go index 8c33826d488e..03f55f98fb89 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go @@ -92,15 +92,17 @@ func (pl *CSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod *v return framework.NewStatus(framework.Error, "node not found") } + logger := klog.FromContext(ctx) + // If CSINode doesn't exist, the predicate may read the limits from Node object csiNode, err := pl.csiNodeLister.Get(node.Name) if err != nil { // TODO: return the error once CSINode is created by default (2 releases) - klog.V(5).InfoS("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) + logger.V(5).Info("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) } newVolumes := make(map[string]string) - if err := pl.filterAttachableVolumes(pod, csiNode, true /* new pod */, newVolumes); err != nil { + if err := pl.filterAttachableVolumes(logger, pod, csiNode, true /* new pod */, newVolumes); err != nil { return framework.AsStatus(err) } @@ -117,7 +119,7 @@ func (pl *CSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod *v attachedVolumes := make(map[string]string) for _, existingPod := range nodeInfo.Pods { - if err := pl.filterAttachableVolumes(existingPod.Pod, csiNode, false /* existing pod */, attachedVolumes); err != nil { + if err := pl.filterAttachableVolumes(logger, existingPod.Pod, csiNode, false /* existing pod */, attachedVolumes); err != nil { return framework.AsStatus(err) } } @@ -138,7 +140,7 @@ func (pl *CSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod *v maxVolumeLimit, ok := nodeVolumeLimits[v1.ResourceName(volumeLimitKey)] if ok { currentVolumeCount := attachedVolumeCount[volumeLimitKey] - klog.V(5).InfoS("Found plugin volume limits", "node", node.Name, "volumeLimitKey", volumeLimitKey, + logger.V(5).Info("Found plugin volume limits", "node", node.Name, "volumeLimitKey", volumeLimitKey, "maxLimits", maxVolumeLimit, "currentVolumeCount", currentVolumeCount, "newVolumeCount", count, "pod", klog.KObj(pod)) if currentVolumeCount+count > int(maxVolumeLimit) { @@ -151,7 +153,7 @@ func (pl *CSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod *v } func (pl *CSILimits) filterAttachableVolumes( - pod *v1.Pod, csiNode *storagev1.CSINode, newPod bool, result map[string]string) error { + logger klog.Logger, pod *v1.Pod, csiNode *storagev1.CSINode, newPod bool, result map[string]string) error { for _, vol := range pod.Spec.Volumes { pvcName := "" isEphemeral := false @@ -172,7 +174,7 @@ func (pl *CSILimits) filterAttachableVolumes( // - If the volume is migratable and CSI migration is enabled, need to count it // as well. // - If the volume is not migratable, it will be count in non_csi filter. - if err := pl.checkAttachableInlineVolume(vol, csiNode, pod, result); err != nil { + if err := pl.checkAttachableInlineVolume(logger, vol, csiNode, pod, result); err != nil { return err } @@ -194,7 +196,7 @@ func (pl *CSILimits) filterAttachableVolumes( } // If the PVC is invalid, we don't count the volume because // there's no guarantee that it belongs to the running predicate. - klog.V(5).InfoS("Unable to look up PVC info", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName)) + logger.V(5).Info("Unable to look up PVC info", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName)) continue } @@ -205,9 +207,9 @@ func (pl *CSILimits) filterAttachableVolumes( } } - driverName, volumeHandle := pl.getCSIDriverInfo(csiNode, pvc) + driverName, volumeHandle := pl.getCSIDriverInfo(logger, csiNode, pvc) if driverName == "" || volumeHandle == "" { - klog.V(5).InfoS("Could not find a CSI driver name or volume handle, not counting volume") + logger.V(5).Info("Could not find a CSI driver name or volume handle, not counting volume") continue } @@ -220,7 +222,7 @@ func (pl *CSILimits) filterAttachableVolumes( // checkAttachableInlineVolume takes an inline volume and add to the result map if the // volume is migratable and CSI migration for this plugin has been enabled. -func (pl *CSILimits) checkAttachableInlineVolume(vol v1.Volume, csiNode *storagev1.CSINode, +func (pl *CSILimits) checkAttachableInlineVolume(logger klog.Logger, vol v1.Volume, csiNode *storagev1.CSINode, pod *v1.Pod, result map[string]string) error { if !pl.translator.IsInlineMigratable(&vol) { return nil @@ -231,7 +233,7 @@ func (pl *CSILimits) checkAttachableInlineVolume(vol v1.Volume, csiNode *storage return fmt.Errorf("looking up provisioner name for volume %v: %w", vol, err) } if !isCSIMigrationOn(csiNode, inTreeProvisionerName) { - klog.V(5).InfoS("CSI Migration is not enabled for provisioner", "provisioner", inTreeProvisionerName, + logger.V(5).Info("CSI Migration is not enabled for provisioner", "provisioner", inTreeProvisionerName, "pod", klog.KObj(pod), "csiNode", csiNode.Name) return nil } @@ -258,21 +260,21 @@ func (pl *CSILimits) checkAttachableInlineVolume(vol v1.Volume, csiNode *storage // getCSIDriverInfo returns the CSI driver name and volume ID of a given PVC. // If the PVC is from a migrated in-tree plugin, this function will return // the information of the CSI driver that the plugin has been migrated to. -func (pl *CSILimits) getCSIDriverInfo(csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { +func (pl *CSILimits) getCSIDriverInfo(logger klog.Logger, csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { pvName := pvc.Spec.VolumeName if pvName == "" { - klog.V(5).InfoS("Persistent volume had no name for claim", "PVC", klog.KObj(pvc)) - return pl.getCSIDriverInfoFromSC(csiNode, pvc) + logger.V(5).Info("Persistent volume had no name for claim", "PVC", klog.KObj(pvc)) + return pl.getCSIDriverInfoFromSC(logger, csiNode, pvc) } pv, err := pl.pvLister.Get(pvName) if err != nil { - klog.V(5).InfoS("Unable to look up PV info for PVC and PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvName)) + logger.V(5).Info("Unable to look up PV info for PVC and PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvName)) // If we can't fetch PV associated with PVC, may be it got deleted // or PVC was prebound to a PVC that hasn't been created yet. // fallback to using StorageClass for volume counting - return pl.getCSIDriverInfoFromSC(csiNode, pvc) + return pl.getCSIDriverInfoFromSC(logger, csiNode, pvc) } csiSource := pv.Spec.PersistentVolumeSource.CSI @@ -284,23 +286,23 @@ func (pl *CSILimits) getCSIDriverInfo(csiNode *storagev1.CSINode, pvc *v1.Persis pluginName, err := pl.translator.GetInTreePluginNameFromSpec(pv, nil) if err != nil { - klog.V(5).InfoS("Unable to look up plugin name from PV spec", "err", err) + logger.V(5).Info("Unable to look up plugin name from PV spec", "err", err) return "", "" } if !isCSIMigrationOn(csiNode, pluginName) { - klog.V(5).InfoS("CSI Migration of plugin is not enabled", "plugin", pluginName) + logger.V(5).Info("CSI Migration of plugin is not enabled", "plugin", pluginName) return "", "" } csiPV, err := pl.translator.TranslateInTreePVToCSI(pv) if err != nil { - klog.V(5).InfoS("Unable to translate in-tree volume to CSI", "err", err) + logger.V(5).Info("Unable to translate in-tree volume to CSI", "err", err) return "", "" } if csiPV.Spec.PersistentVolumeSource.CSI == nil { - klog.V(5).InfoS("Unable to get a valid volume source for translated PV", "PV", pvName) + logger.V(5).Info("Unable to get a valid volume source for translated PV", "PV", pvName) return "", "" } @@ -311,7 +313,7 @@ func (pl *CSILimits) getCSIDriverInfo(csiNode *storagev1.CSINode, pvc *v1.Persis } // getCSIDriverInfoFromSC returns the CSI driver name and a random volume ID of a given PVC's StorageClass. -func (pl *CSILimits) getCSIDriverInfoFromSC(csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { +func (pl *CSILimits) getCSIDriverInfoFromSC(logger klog.Logger, csiNode *storagev1.CSINode, pvc *v1.PersistentVolumeClaim) (string, string) { namespace := pvc.Namespace pvcName := pvc.Name scName := storagehelpers.GetPersistentVolumeClaimClass(pvc) @@ -319,13 +321,13 @@ func (pl *CSILimits) getCSIDriverInfoFromSC(csiNode *storagev1.CSINode, pvc *v1. // If StorageClass is not set or not found, then PVC must be using immediate binding mode // and hence it must be bound before scheduling. So it is safe to not count it. if scName == "" { - klog.V(5).InfoS("PVC has no StorageClass", "PVC", klog.KObj(pvc)) + logger.V(5).Info("PVC has no StorageClass", "PVC", klog.KObj(pvc)) return "", "" } storageClass, err := pl.scLister.Get(scName) if err != nil { - klog.V(5).InfoS("Could not get StorageClass for PVC", "PVC", klog.KObj(pvc), "err", err) + logger.V(5).Info("Could not get StorageClass for PVC", "PVC", klog.KObj(pvc), "err", err) return "", "" } @@ -337,13 +339,13 @@ func (pl *CSILimits) getCSIDriverInfoFromSC(csiNode *storagev1.CSINode, pvc *v1. provisioner := storageClass.Provisioner if pl.translator.IsMigratableIntreePluginByName(provisioner) { if !isCSIMigrationOn(csiNode, provisioner) { - klog.V(5).InfoS("CSI Migration of provisioner is not enabled", "provisioner", provisioner) + logger.V(5).Info("CSI Migration of provisioner is not enabled", "provisioner", provisioner) return "", "" } driverName, err := pl.translator.GetCSINameFromInTreeName(provisioner) if err != nil { - klog.V(5).InfoS("Unable to look up driver name from provisioner name", "provisioner", provisioner, "err", err) + logger.V(5).Info("Unable to look up driver name from provisioner name", "provisioner", provisioner, "err", err) return "", "" } return driverName, volumeHandle diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi.go b/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi.go index 4a2535e5f26b..eb4cfbfb6ebe 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi.go @@ -72,7 +72,7 @@ const AzureDiskName = names.AzureDiskLimits // NewAzureDisk returns function that initializes a new plugin and returns it. func NewAzureDisk(_ runtime.Object, handle framework.Handle, fts feature.Features) (framework.Plugin, error) { informerFactory := handle.SharedInformerFactory() - return newNonCSILimitsWithInformerFactory(azureDiskVolumeFilterType, informerFactory, fts), nil + return newNonCSILimitsWithInformerFactory(handle.Logger(), azureDiskVolumeFilterType, informerFactory, fts), nil } // CinderName is the name of the plugin used in the plugin registry and configurations. @@ -81,7 +81,7 @@ const CinderName = names.CinderLimits // NewCinder returns function that initializes a new plugin and returns it. func NewCinder(_ runtime.Object, handle framework.Handle, fts feature.Features) (framework.Plugin, error) { informerFactory := handle.SharedInformerFactory() - return newNonCSILimitsWithInformerFactory(cinderVolumeFilterType, informerFactory, fts), nil + return newNonCSILimitsWithInformerFactory(handle.Logger(), cinderVolumeFilterType, informerFactory, fts), nil } // EBSName is the name of the plugin used in the plugin registry and configurations. @@ -90,7 +90,7 @@ const EBSName = names.EBSLimits // NewEBS returns function that initializes a new plugin and returns it. func NewEBS(_ runtime.Object, handle framework.Handle, fts feature.Features) (framework.Plugin, error) { informerFactory := handle.SharedInformerFactory() - return newNonCSILimitsWithInformerFactory(ebsVolumeFilterType, informerFactory, fts), nil + return newNonCSILimitsWithInformerFactory(handle.Logger(), ebsVolumeFilterType, informerFactory, fts), nil } // GCEPDName is the name of the plugin used in the plugin registry and configurations. @@ -99,7 +99,7 @@ const GCEPDName = names.GCEPDLimits // NewGCEPD returns function that initializes a new plugin and returns it. func NewGCEPD(_ runtime.Object, handle framework.Handle, fts feature.Features) (framework.Plugin, error) { informerFactory := handle.SharedInformerFactory() - return newNonCSILimitsWithInformerFactory(gcePDVolumeFilterType, informerFactory, fts), nil + return newNonCSILimitsWithInformerFactory(handle.Logger(), gcePDVolumeFilterType, informerFactory, fts), nil } // nonCSILimits contains information to check the max number of volumes for a plugin. @@ -124,6 +124,7 @@ var _ framework.EnqueueExtensions = &nonCSILimits{} // newNonCSILimitsWithInformerFactory returns a plugin with filter name and informer factory. func newNonCSILimitsWithInformerFactory( + logger klog.Logger, filterName string, informerFactory informers.SharedInformerFactory, fts feature.Features, @@ -133,7 +134,7 @@ func newNonCSILimitsWithInformerFactory( csiNodesLister := informerFactory.Storage().V1().CSINodes().Lister() scLister := informerFactory.Storage().V1().StorageClasses().Lister() - return newNonCSILimits(filterName, csiNodesLister, scLister, pvLister, pvcLister, fts) + return newNonCSILimits(logger, filterName, csiNodesLister, scLister, pvLister, pvcLister, fts) } // newNonCSILimits creates a plugin which evaluates whether a pod can fit based on the @@ -147,6 +148,7 @@ func newNonCSILimitsWithInformerFactory( // types, counts the number of unique volumes, and rejects the new pod if it would place the total count over // the maximum. func newNonCSILimits( + logger klog.Logger, filterName string, csiNodeLister storagelisters.CSINodeLister, scLister storagelisters.StorageClassLister, @@ -176,14 +178,14 @@ func newNonCSILimits( filter = cinderVolumeFilter volumeLimitKey = v1.ResourceName(volumeutil.CinderVolumeLimitKey) default: - klog.ErrorS(errors.New("wrong filterName"), "Cannot create nonCSILimits plugin") + logger.Error(errors.New("wrong filterName"), "Cannot create nonCSILimits plugin") return nil } pl := &nonCSILimits{ name: name, filter: filter, volumeLimitKey: volumeLimitKey, - maxVolumeFunc: getMaxVolumeFunc(filterName), + maxVolumeFunc: getMaxVolumeFunc(logger, filterName), csiNodeLister: csiNodeLister, pvLister: pvLister, pvcLister: pvcLister, @@ -216,8 +218,9 @@ func (pl *nonCSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod return nil } + logger := klog.FromContext(ctx) newVolumes := make(sets.String) - if err := pl.filterVolumes(pod, true /* new pod */, newVolumes); err != nil { + if err := pl.filterVolumes(logger, pod, true /* new pod */, newVolumes); err != nil { return framework.AsStatus(err) } @@ -238,7 +241,7 @@ func (pl *nonCSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod if err != nil { // we don't fail here because the CSINode object is only necessary // for determining whether the migration is enabled or not - klog.V(5).InfoS("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) + logger.V(5).Info("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) } } @@ -250,7 +253,7 @@ func (pl *nonCSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod // count unique volumes existingVolumes := make(sets.String) for _, existingPod := range nodeInfo.Pods { - if err := pl.filterVolumes(existingPod.Pod, false /* existing pod */, existingVolumes); err != nil { + if err := pl.filterVolumes(logger, existingPod.Pod, false /* existing pod */, existingVolumes); err != nil { return framework.AsStatus(err) } } @@ -274,7 +277,7 @@ func (pl *nonCSILimits) Filter(ctx context.Context, _ *framework.CycleState, pod return nil } -func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes sets.String) error { +func (pl *nonCSILimits) filterVolumes(logger klog.Logger, pod *v1.Pod, newPod bool, filteredVolumes sets.String) error { volumes := pod.Spec.Volumes for i := range volumes { vol := &volumes[i] @@ -317,7 +320,7 @@ func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes } // If the PVC is invalid, we don't count the volume because // there's no guarantee that it belongs to the running predicate. - klog.V(4).InfoS("Unable to look up PVC info, assuming PVC doesn't match predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName), "err", err) + logger.V(4).Info("Unable to look up PVC info, assuming PVC doesn't match predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName), "err", err) continue } @@ -335,7 +338,7 @@ func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes // original PV where it was bound to, so we count the volume if // it belongs to the running predicate. if pl.matchProvisioner(pvc) { - klog.V(4).InfoS("PVC is not bound, assuming PVC matches predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName)) + logger.V(4).Info("PVC is not bound, assuming PVC matches predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName)) filteredVolumes.Insert(pvID) } continue @@ -346,7 +349,7 @@ func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes // If the PV is invalid and PVC belongs to the running predicate, // log the error and count the PV towards the PV limit. if pl.matchProvisioner(pvc) { - klog.V(4).InfoS("Unable to look up PV, assuming PV matches predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName), "PV", klog.KRef("", pvName), "err", err) + logger.V(4).Info("Unable to look up PV, assuming PV matches predicate when counting limits", "pod", klog.KObj(pod), "PVC", klog.KRef(pod.Namespace, pvcName), "PV", klog.KRef("", pvName), "err", err) filteredVolumes.Insert(pvID) } continue @@ -375,12 +378,12 @@ func (pl *nonCSILimits) matchProvisioner(pvc *v1.PersistentVolumeClaim) bool { } // getMaxVolLimitFromEnv checks the max PD volumes environment variable, otherwise returning a default value. -func getMaxVolLimitFromEnv() int { +func getMaxVolLimitFromEnv(logger klog.Logger) int { if rawMaxVols := os.Getenv(KubeMaxPDVols); rawMaxVols != "" { if parsedMaxVols, err := strconv.Atoi(rawMaxVols); err != nil { - klog.ErrorS(err, "Unable to parse maximum PD volumes value, using default") + logger.Error(err, "Unable to parse maximum PD volumes value, using default") } else if parsedMaxVols <= 0 { - klog.ErrorS(errors.New("maximum PD volumes is negative"), "Unable to parse maximum PD volumes value, using default") + logger.Error(errors.New("maximum PD volumes is negative"), "Unable to parse maximum PD volumes value, using default") } else { return parsedMaxVols } @@ -501,9 +504,9 @@ var cinderVolumeFilter = VolumeFilter{ }, } -func getMaxVolumeFunc(filterName string) func(node *v1.Node) int { +func getMaxVolumeFunc(logger klog.Logger, filterName string) func(node *v1.Node) int { return func(node *v1.Node) int { - maxVolumesFromEnv := getMaxVolLimitFromEnv() + maxVolumesFromEnv := getMaxVolLimitFromEnv(logger) if maxVolumesFromEnv > 0 { return maxVolumesFromEnv } diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi_test.go index 922abd071dbb..f22e401aa6a3 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/non_csi_test.go @@ -27,6 +27,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" csilibplugins "k8s.io/csi-translation-lib/plugins" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/framework" fakeframework "k8s.io/kubernetes/pkg/scheduler/framework/fake" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" @@ -154,9 +155,10 @@ func TestEphemeralLimits(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) fts := feature.Features{} node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), filterName) - p := newNonCSILimits(filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(filterName, driverName), getFakePVLister(filterName), append(getFakePVCLister(filterName), test.extraClaims...), fts).(framework.FilterPlugin) + p := newNonCSILimits(logger, filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(filterName, driverName), getFakePVLister(filterName), append(getFakePVCLister(filterName), test.extraClaims...), fts).(framework.FilterPlugin) gotStatus := p.Filter(context.Background(), nil, test.newPod, node) if !reflect.DeepEqual(gotStatus, test.wantStatus) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) @@ -319,8 +321,9 @@ func TestAzureDiskLimits(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) - p := newNonCSILimits(test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) + p := newNonCSILimits(logger, test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) gotStatus := p.Filter(context.Background(), nil, test.newPod, node) if !reflect.DeepEqual(gotStatus, test.wantStatus) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) @@ -529,8 +532,9 @@ func TestEBSLimits(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) - p := newNonCSILimits(test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) + p := newNonCSILimits(logger, test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) gotStatus := p.Filter(context.Background(), nil, test.newPod, node) if !reflect.DeepEqual(gotStatus, test.wantStatus) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) @@ -693,8 +697,9 @@ func TestGCEPDLimits(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), test.filterName) - p := newNonCSILimits(test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) + p := newNonCSILimits(logger, test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin) gotStatus := p.Filter(context.Background(), nil, test.newPod, node) if !reflect.DeepEqual(gotStatus, test.wantStatus) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) @@ -730,8 +735,9 @@ func TestGetMaxVols(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) os.Setenv(KubeMaxPDVols, test.rawMaxVols) - result := getMaxVolLimitFromEnv() + result := getMaxVolLimitFromEnv(logger) if result != test.expected { t.Errorf("expected %v got %v", test.expected, result) } diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go index 888750d03a0e..67dfe799cd11 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go @@ -262,7 +262,7 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod) nodeInfo := allNodes[i] node := nodeInfo.Node() if node == nil { - klog.ErrorS(nil, "Node not found") + klog.FromContext(ctx).Error(nil, "Node not found") return } @@ -335,12 +335,13 @@ func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.C return nil } + logger := klog.FromContext(ctx) podLabelSet := labels.Set(pod.Labels) for _, c := range s.Constraints { tpKey := c.TopologyKey tpVal, ok := node.Labels[c.TopologyKey] if !ok { - klog.V(5).InfoS("Node doesn't have required label", "node", klog.KObj(node), "label", tpKey) + logger.V(5).Info("Node doesn't have required label", "node", klog.KObj(node), "label", tpKey) return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonNodeLabelNotMatch) } @@ -348,7 +349,7 @@ func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.C // 'existing matching num' + 'if self-match (1 or 0)' - 'global minimum' <= 'maxSkew' minMatchNum, err := s.minMatchNum(tpKey, c.MinDomains, pl.enableMinDomainsInPodTopologySpread) if err != nil { - klog.ErrorS(err, "Internal error occurred while retrieving value precalculated in PreFilter", "topologyKey", tpKey, "paths", s.TpKeyToCriticalPaths) + logger.Error(err, "Internal error occurred while retrieving value precalculated in PreFilter", "topologyKey", tpKey, "paths", s.TpKeyToCriticalPaths) continue } @@ -364,7 +365,7 @@ func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.C } skew := matchNum + selfMatchNum - minMatchNum if skew > int(c.MaxSkew) { - klog.V(5).InfoS("Node failed spreadConstraint: matchNum + selfMatchNum - minMatchNum > maxSkew", "node", klog.KObj(node), "topologyKey", tpKey, "matchNum", matchNum, "selfMatchNum", selfMatchNum, "minMatchNum", minMatchNum, "maxSkew", c.MaxSkew) + logger.V(5).Info("Node failed spreadConstraint: matchNum + selfMatchNum - minMatchNum > maxSkew", "node", klog.KObj(node), "topologyKey", tpKey, "matchNum", matchNum, "selfMatchNum", selfMatchNum, "minMatchNum", minMatchNum, "maxSkew", c.MaxSkew) return framework.NewStatus(framework.Unschedulable, ErrReasonConstraintsNotMatch) } } diff --git a/pkg/scheduler/framework/plugins/volumebinding/assume_cache.go b/pkg/scheduler/framework/plugins/volumebinding/assume_cache.go index 1ee97a3eb24b..e06194086aac 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/assume_cache.go +++ b/pkg/scheduler/framework/plugins/volumebinding/assume_cache.go @@ -87,6 +87,10 @@ func (e *errObjectName) Error() string { // Restore() sets the latest object pointer back to the informer object. // Get/List() always returns the latest object pointer. type assumeCache struct { + // The logger that was chosen when setting up the cache. + // Will be used for all operations. + logger klog.Logger + // Synchronizes updates to store rwMutex sync.RWMutex @@ -129,8 +133,9 @@ func (c *assumeCache) objInfoIndexFunc(obj interface{}) ([]string, error) { } // NewAssumeCache creates an assume cache for general objects. -func NewAssumeCache(informer cache.SharedIndexInformer, description, indexName string, indexFunc cache.IndexFunc) AssumeCache { +func NewAssumeCache(logger klog.Logger, informer cache.SharedIndexInformer, description, indexName string, indexFunc cache.IndexFunc) AssumeCache { c := &assumeCache{ + logger: logger, description: description, indexFunc: indexFunc, indexName: indexName, @@ -161,7 +166,7 @@ func (c *assumeCache) add(obj interface{}) { name, err := cache.MetaNamespaceKeyFunc(obj) if err != nil { - klog.ErrorS(&errObjectName{err}, "Add failed") + c.logger.Error(&errObjectName{err}, "Add failed") return } @@ -171,29 +176,29 @@ func (c *assumeCache) add(obj interface{}) { if objInfo, _ := c.getObjInfo(name); objInfo != nil { newVersion, err := c.getObjVersion(name, obj) if err != nil { - klog.ErrorS(err, "Add failed: couldn't get object version") + c.logger.Error(err, "Add failed: couldn't get object version") return } storedVersion, err := c.getObjVersion(name, objInfo.latestObj) if err != nil { - klog.ErrorS(err, "Add failed: couldn't get stored object version") + c.logger.Error(err, "Add failed: couldn't get stored object version") return } // Only update object if version is newer. // This is so we don't override assumed objects due to informer resync. if newVersion <= storedVersion { - klog.V(10).InfoS("Skip adding object to assume cache because version is not newer than storedVersion", "description", c.description, "cacheKey", name, "newVersion", newVersion, "storedVersion", storedVersion) + c.logger.V(10).Info("Skip adding object to assume cache because version is not newer than storedVersion", "description", c.description, "cacheKey", name, "newVersion", newVersion, "storedVersion", storedVersion) return } } objInfo := &objInfo{name: name, latestObj: obj, apiObj: obj} if err = c.store.Update(objInfo); err != nil { - klog.InfoS("Error occurred while updating stored object", "err", err) + c.logger.Info("Error occurred while updating stored object", "err", err) } else { - klog.V(10).InfoS("Adding object to assume cache", "description", c.description, "cacheKey", name, "assumeCache", obj) + c.logger.V(10).Info("Adding object to assume cache", "description", c.description, "cacheKey", name, "assumeCache", obj) } } @@ -208,7 +213,7 @@ func (c *assumeCache) delete(obj interface{}) { name, err := cache.MetaNamespaceKeyFunc(obj) if err != nil { - klog.ErrorS(&errObjectName{err}, "Failed to delete") + c.logger.Error(&errObjectName{err}, "Failed to delete") return } @@ -218,7 +223,7 @@ func (c *assumeCache) delete(obj interface{}) { objInfo := &objInfo{name: name} err = c.store.Delete(objInfo) if err != nil { - klog.ErrorS(err, "Failed to delete", "description", c.description, "cacheKey", name) + c.logger.Error(err, "Failed to delete", "description", c.description, "cacheKey", name) } } @@ -280,14 +285,14 @@ func (c *assumeCache) List(indexObj interface{}) []interface{} { allObjs := []interface{}{} objs, err := c.store.Index(c.indexName, &objInfo{latestObj: indexObj}) if err != nil { - klog.ErrorS(err, "List index error") + c.logger.Error(err, "List index error") return nil } for _, obj := range objs { objInfo, ok := obj.(*objInfo) if !ok { - klog.ErrorS(&errWrongType{"objInfo", obj}, "List error") + c.logger.Error(&errWrongType{"objInfo", obj}, "List error") continue } allObjs = append(allObjs, objInfo.latestObj) @@ -325,7 +330,7 @@ func (c *assumeCache) Assume(obj interface{}) error { // Only update the cached object objInfo.latestObj = obj - klog.V(4).InfoS("Assumed object", "description", c.description, "cacheKey", name, "version", newVersion) + c.logger.V(4).Info("Assumed object", "description", c.description, "cacheKey", name, "version", newVersion) return nil } @@ -336,10 +341,10 @@ func (c *assumeCache) Restore(objName string) { objInfo, err := c.getObjInfo(objName) if err != nil { // This could be expected if object got deleted - klog.V(5).InfoS("Restore object", "description", c.description, "cacheKey", objName, "err", err) + c.logger.V(5).Info("Restore object", "description", c.description, "cacheKey", objName, "err", err) } else { objInfo.latestObj = objInfo.apiObj - klog.V(4).InfoS("Restored object", "description", c.description, "cacheKey", objName) + c.logger.V(4).Info("Restored object", "description", c.description, "cacheKey", objName) } } @@ -354,6 +359,7 @@ type PVAssumeCache interface { type pvAssumeCache struct { AssumeCache + logger klog.Logger } func pvStorageClassIndexFunc(obj interface{}) ([]string, error) { @@ -364,8 +370,12 @@ func pvStorageClassIndexFunc(obj interface{}) ([]string, error) { } // NewPVAssumeCache creates a PV assume cache. -func NewPVAssumeCache(informer cache.SharedIndexInformer) PVAssumeCache { - return &pvAssumeCache{NewAssumeCache(informer, "v1.PersistentVolume", "storageclass", pvStorageClassIndexFunc)} +func NewPVAssumeCache(logger klog.Logger, informer cache.SharedIndexInformer) PVAssumeCache { + logger = klog.LoggerWithName(logger, "PV Cache") + return &pvAssumeCache{ + AssumeCache: NewAssumeCache(logger, informer, "v1.PersistentVolume", "storageclass", pvStorageClassIndexFunc), + logger: logger, + } } func (c *pvAssumeCache) GetPV(pvName string) (*v1.PersistentVolume, error) { @@ -403,7 +413,7 @@ func (c *pvAssumeCache) ListPVs(storageClassName string) []*v1.PersistentVolume for _, obj := range objs { pv, ok := obj.(*v1.PersistentVolume) if !ok { - klog.ErrorS(&errWrongType{"v1.PersistentVolume", obj}, "ListPVs") + c.logger.Error(&errWrongType{"v1.PersistentVolume", obj}, "ListPVs") continue } pvs = append(pvs, pv) @@ -423,11 +433,16 @@ type PVCAssumeCache interface { type pvcAssumeCache struct { AssumeCache + logger klog.Logger } // NewPVCAssumeCache creates a PVC assume cache. -func NewPVCAssumeCache(informer cache.SharedIndexInformer) PVCAssumeCache { - return &pvcAssumeCache{NewAssumeCache(informer, "v1.PersistentVolumeClaim", "", nil)} +func NewPVCAssumeCache(logger klog.Logger, informer cache.SharedIndexInformer) PVCAssumeCache { + logger = klog.LoggerWithName(logger, "PVC Cache") + return &pvcAssumeCache{ + AssumeCache: NewAssumeCache(logger, informer, "v1.PersistentVolumeClaim", "", nil), + logger: logger, + } } func (c *pvcAssumeCache) GetPVC(pvcKey string) (*v1.PersistentVolumeClaim, error) { diff --git a/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go b/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go index 49fcb1cee852..6dcb0bff74ba 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/assume_cache_test.go @@ -23,6 +23,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-helpers/storage/volume" + "k8s.io/klog/v2/ktesting" ) func verifyListPVs(t *testing.T, cache PVAssumeCache, expectedPVs map[string]*v1.PersistentVolume, storageClassName string) { @@ -53,6 +54,7 @@ func verifyPV(cache PVAssumeCache, name string, expectedPV *v1.PersistentVolume) } func TestAssumePV(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) scenarios := map[string]struct { oldPV *v1.PersistentVolume newPV *v1.PersistentVolume @@ -96,7 +98,7 @@ func TestAssumePV(t *testing.T) { } for name, scenario := range scenarios { - cache := NewPVAssumeCache(nil) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -130,7 +132,8 @@ func TestAssumePV(t *testing.T) { } func TestRestorePV(t *testing.T) { - cache := NewPVAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -170,7 +173,8 @@ func TestRestorePV(t *testing.T) { } func TestBasicPVCache(t *testing.T) { - cache := NewPVAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -214,7 +218,8 @@ func TestBasicPVCache(t *testing.T) { } func TestPVCacheWithStorageClasses(t *testing.T) { - cache := NewPVAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -260,7 +265,8 @@ func TestPVCacheWithStorageClasses(t *testing.T) { } func TestAssumeUpdatePVCache(t *testing.T) { - cache := NewPVAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVAssumeCache(logger, nil) internalCache, ok := cache.(*pvAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -353,7 +359,8 @@ func TestAssumePVC(t *testing.T) { } for name, scenario := range scenarios { - cache := NewPVCAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVCAssumeCache(logger, nil) internalCache, ok := cache.(*pvcAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -387,7 +394,8 @@ func TestAssumePVC(t *testing.T) { } func TestRestorePVC(t *testing.T) { - cache := NewPVCAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVCAssumeCache(logger, nil) internalCache, ok := cache.(*pvcAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") @@ -427,7 +435,8 @@ func TestRestorePVC(t *testing.T) { } func TestAssumeUpdatePVCCache(t *testing.T) { - cache := NewPVCAssumeCache(nil) + logger, _ := ktesting.NewTestContext(t) + cache := NewPVCAssumeCache(logger, nil) internalCache, ok := cache.(*pvcAssumeCache).AssumeCache.(*assumeCache) if !ok { t.Fatalf("Failed to get internal cache") diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder.go b/pkg/scheduler/framework/plugins/volumebinding/binder.go index f715699e52dc..020a31b00c87 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder.go @@ -146,7 +146,7 @@ type InTreeToCSITranslator interface { type SchedulerVolumeBinder interface { // GetPodVolumes returns a pod's PVCs separated into bound, unbound with delayed binding (including provisioning) // and unbound with immediate binding (including prebound) - GetPodVolumes(pod *v1.Pod) (boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) + GetPodVolumes(logger klog.Logger, pod *v1.Pod) (boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) // FindPodVolumes checks if all of a Pod's PVCs can be satisfied by the // node and returns pod's volumes information. @@ -161,7 +161,7 @@ type SchedulerVolumeBinder interface { // for volumes that still need to be created. // // This function is called by the scheduler VolumeBinding plugin and can be called in parallel - FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) + FindPodVolumes(logger klog.Logger, pod *v1.Pod, boundClaims, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) // AssumePodVolumes will: // 1. Take the PV matches for unbound PVCs and update the PV cache assuming @@ -172,10 +172,10 @@ type SchedulerVolumeBinder interface { // It returns true if all volumes are fully bound // // This function is called serially. - AssumePodVolumes(assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (allFullyBound bool, err error) + AssumePodVolumes(logger klog.Logger, assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (allFullyBound bool, err error) // RevertAssumedPodVolumes will revert assumed PV and PVC cache. - RevertAssumedPodVolumes(podVolumes *PodVolumes) + RevertAssumedPodVolumes(logger klog.Logger, podVolumes *PodVolumes) // BindPodVolumes will: // 1. Initiate the volume binding by making the API call to prebind the PV @@ -185,7 +185,7 @@ type SchedulerVolumeBinder interface { // 3. Wait for PVCs to be completely bound by the PV controller // // This function can be called in parallel. - BindPodVolumes(assumedPod *v1.Pod, podVolumes *PodVolumes) error + BindPodVolumes(logger klog.Logger, assumedPod *v1.Pod, podVolumes *PodVolumes) error } type volumeBinder struct { @@ -220,6 +220,7 @@ type CapacityCheck struct { // // capacityCheck determines how storage capacity is checked (CSIStorageCapacity feature). func NewVolumeBinder( + logger klog.Logger, kubeClient clientset.Interface, podInformer coreinformers.PodInformer, nodeInformer coreinformers.NodeInformer, @@ -235,8 +236,8 @@ func NewVolumeBinder( classLister: storageClassInformer.Lister(), nodeLister: nodeInformer.Lister(), csiNodeLister: csiNodeInformer.Lister(), - pvcCache: NewPVCAssumeCache(pvcInformer.Informer()), - pvCache: NewPVAssumeCache(pvInformer.Informer()), + pvcCache: NewPVCAssumeCache(logger, pvcInformer.Informer()), + pvCache: NewPVAssumeCache(logger, pvInformer.Informer()), bindTimeout: bindTimeout, translator: csitrans.New(), } @@ -250,11 +251,11 @@ func NewVolumeBinder( // FindPodVolumes finds the matching PVs for PVCs and nodes to provision PVs // for the given pod and node. If the node does not fit, confilict reasons are // returned. -func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) { +func (b *volumeBinder) FindPodVolumes(logger klog.Logger, pod *v1.Pod, boundClaims, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) { podVolumes = &PodVolumes{} // Warning: Below log needs high verbosity as it can be printed several times (#60933). - klog.V(5).InfoS("FindPodVolumes", "pod", klog.KObj(pod), "node", klog.KObj(node)) + logger.V(5).Info("FindPodVolumes starts") // Initialize to true for pods that don't have volumes. These // booleans get translated into reason strings when the function @@ -306,7 +307,7 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []* // Check PV node affinity on bound volumes if len(boundClaims) > 0 { - boundVolumesSatisfied, boundPVsFound, err = b.checkBoundClaims(boundClaims, node, pod) + boundVolumesSatisfied, boundPVsFound, err = b.checkBoundClaims(logger, boundClaims, node) if err != nil { return } @@ -336,7 +337,7 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []* // Find matching volumes if len(claimsToFindMatching) > 0 { var unboundClaims []*v1.PersistentVolumeClaim - unboundVolumesSatisfied, staticBindings, unboundClaims, err = b.findMatchingVolumes(pod, claimsToFindMatching, node) + unboundVolumesSatisfied, staticBindings, unboundClaims, err = b.findMatchingVolumes(logger, pod, claimsToFindMatching, node) if err != nil { return } @@ -346,7 +347,7 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []* // Check for claims to provision. This is the first time where we potentially // find out that storage is not sufficient for the node. if len(claimsToProvision) > 0 { - unboundVolumesSatisfied, sufficientStorage, dynamicProvisions, err = b.checkVolumeProvisions(pod, claimsToProvision, node) + unboundVolumesSatisfied, sufficientStorage, dynamicProvisions, err = b.checkVolumeProvisions(logger, pod, claimsToProvision, node) if err != nil { return } @@ -361,16 +362,15 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, boundClaims, claimsToBind []* // 1. Update the pvCache with the new prebound PV. // 2. Update the pvcCache with the new PVCs with annotations set // 3. Update PodVolumes again with cached API updates for PVs and PVCs. -func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (allFullyBound bool, err error) { - klog.V(4).InfoS("AssumePodVolumes", "pod", klog.KObj(assumedPod), "node", klog.KRef("", nodeName)) +func (b *volumeBinder) AssumePodVolumes(logger klog.Logger, assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (allFullyBound bool, err error) { defer func() { if err != nil { metrics.VolumeSchedulingStageFailed.WithLabelValues("assume").Inc() } }() - if allBound := b.arePodVolumesBound(assumedPod); allBound { - klog.V(4).InfoS("AssumePodVolumes: all PVCs bound and nothing to do", "pod", klog.KObj(assumedPod), "node", klog.KRef("", nodeName)) + if allBound := b.arePodVolumesBound(logger, assumedPod); allBound { + logger.V(4).Info("All PVCs bound and nothing to do") return true, nil } @@ -378,7 +378,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, pod newBindings := []*BindingInfo{} for _, binding := range podVolumes.StaticBindings { newPV, dirty, err := volume.GetBindVolumeToClaim(binding.pv, binding.pvc) - klog.V(5).InfoS("AssumePodVolumes: GetBindVolumeToClaim", + logger.V(5).Info("AssumePodVolumes: GetBindVolumeToClaim", "pod", klog.KObj(assumedPod), "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc), @@ -386,7 +386,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, pod "dirty", dirty, ) if err != nil { - klog.ErrorS(err, "AssumePodVolumes: fail to GetBindVolumeToClaim") + logger.Error(err, "AssumePodVolumes: fail to GetBindVolumeToClaim") b.revertAssumedPVs(newBindings) return false, err } @@ -424,7 +424,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, pod } // RevertAssumedPodVolumes will revert assumed PV and PVC cache. -func (b *volumeBinder) RevertAssumedPodVolumes(podVolumes *PodVolumes) { +func (b *volumeBinder) RevertAssumedPodVolumes(logger klog.Logger, podVolumes *PodVolumes) { b.revertAssumedPVs(podVolumes.StaticBindings) b.revertAssumedPVCs(podVolumes.DynamicProvisions) } @@ -432,9 +432,8 @@ func (b *volumeBinder) RevertAssumedPodVolumes(podVolumes *PodVolumes) { // BindPodVolumes gets the cached bindings and PVCs to provision in pod's volumes information, // makes the API update for those PVs/PVCs, and waits for the PVCs to be completely bound // by the PV controller. -func (b *volumeBinder) BindPodVolumes(assumedPod *v1.Pod, podVolumes *PodVolumes) (err error) { - klog.V(4).InfoS("BindPodVolumes", "pod", klog.KObj(assumedPod), "node", klog.KRef("", assumedPod.Spec.NodeName)) - +func (b *volumeBinder) BindPodVolumes(logger klog.Logger, assumedPod *v1.Pod, podVolumes *PodVolumes) (err error) { + podName := getPodName(assumedPod) defer func() { if err != nil { metrics.VolumeSchedulingStageFailed.WithLabelValues("bind").Inc() @@ -445,13 +444,13 @@ func (b *volumeBinder) BindPodVolumes(assumedPod *v1.Pod, podVolumes *PodVolumes claimsToProvision := podVolumes.DynamicProvisions // Start API operations - err = b.bindAPIUpdate(assumedPod, bindings, claimsToProvision) + err = b.bindAPIUpdate(logger, podName, bindings, claimsToProvision) if err != nil { return err } err = wait.Poll(time.Second, b.bindTimeout, func() (bool, error) { - b, err := b.checkBindings(assumedPod, bindings, claimsToProvision) + b, err := b.checkBindings(logger, assumedPod, bindings, claimsToProvision) return b, err }) if err != nil { @@ -469,8 +468,7 @@ func getPVCName(pvc *v1.PersistentVolumeClaim) string { } // bindAPIUpdate makes the API update for those PVs/PVCs. -func (b *volumeBinder) bindAPIUpdate(pod *v1.Pod, bindings []*BindingInfo, claimsToProvision []*v1.PersistentVolumeClaim) error { - podName := getPodName(pod) +func (b *volumeBinder) bindAPIUpdate(logger klog.Logger, podName string, bindings []*BindingInfo, claimsToProvision []*v1.PersistentVolumeClaim) error { if bindings == nil { return fmt.Errorf("failed to get cached bindings for pod %q", podName) } @@ -500,15 +498,15 @@ func (b *volumeBinder) bindAPIUpdate(pod *v1.Pod, bindings []*BindingInfo, claim // Do the actual prebinding. Let the PV controller take care of the rest // There is no API rollback if the actual binding fails for _, binding = range bindings { - klog.V(5).InfoS("bindAPIUpdate: binding PV to PVC", "pod", klog.KObj(pod), "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc)) + logger.V(5).Info("Binding", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc)) // TODO: does it hurt if we make an api call and nothing needs to be updated? - klog.V(2).InfoS("Claim bound to volume", "PVC", klog.KObj(binding.pvc), "PV", klog.KObj(binding.pv)) + logger.V(2).Info("Claim bound to volume", "PVC", klog.KObj(binding.pvc), "PV", klog.KObj(binding.pv)) newPV, err := b.kubeClient.CoreV1().PersistentVolumes().Update(context.TODO(), binding.pv, metav1.UpdateOptions{}) if err != nil { - klog.V(4).InfoS("Updating PersistentVolume: binding to claim failed", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc), "err", err) + logger.V(4).Info("Updating PersistentVolume: binding to claim failed", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc), "err", err) return err } - klog.V(4).InfoS("Updating PersistentVolume: bound to claim", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc)) + logger.V(4).Info("Updating PersistentVolume: bound to claim", "PV", klog.KObj(binding.pv), "PVC", klog.KObj(binding.pvc)) // Save updated object from apiserver for later checking. binding.pv = newPV lastProcessedBinding++ @@ -517,7 +515,7 @@ func (b *volumeBinder) bindAPIUpdate(pod *v1.Pod, bindings []*BindingInfo, claim // Update claims objects to trigger volume provisioning. Let the PV controller take care of the rest // PV controller is expected to signal back by removing related annotations if actual provisioning fails for i, claim = range claimsToProvision { - klog.V(5).InfoS("Updating claims objects to trigger volume provisioning", "pod", klog.KObj(pod), "PVC", klog.KObj(claim)) + logger.V(5).Info("Updating claims objects to trigger volume provisioning", "PVC", klog.KObj(claim)) newClaim, err := b.kubeClient.CoreV1().PersistentVolumeClaims(claim.Namespace).Update(context.TODO(), claim, metav1.UpdateOptions{}) if err != nil { return err @@ -544,7 +542,7 @@ var ( // PV/PVC cache can be assumed again in main scheduler loop, we must check // latest state in API server which are shared with PV controller and // provisioners -func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claimsToProvision []*v1.PersistentVolumeClaim) (bool, error) { +func (b *volumeBinder) checkBindings(logger klog.Logger, pod *v1.Pod, bindings []*BindingInfo, claimsToProvision []*v1.PersistentVolumeClaim) (bool, error) { podName := getPodName(pod) if bindings == nil { return false, fmt.Errorf("failed to get cached bindings for pod %q", podName) @@ -561,7 +559,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim csiNode, err := b.csiNodeLister.Get(node.Name) if err != nil { // TODO: return the error once CSINode is created by default - klog.V(4).InfoS("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) + logger.V(4).Info("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) } // Check for any conditions that might require scheduling retry @@ -573,7 +571,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim if apierrors.IsNotFound(err) { return false, fmt.Errorf("pod does not exist any more: %w", err) } - klog.ErrorS(err, "Failed to get pod from the lister", "pod", klog.KObj(pod)) + logger.Error(err, "Failed to get pod from the lister") } for _, binding := range bindings { @@ -669,11 +667,11 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim } // All pvs and pvcs that we operated on are bound - klog.V(4).InfoS("All PVCs for pod are bound", "pod", klog.KObj(pod)) + logger.V(4).Info("All PVCs for pod are bound") return true, nil } -func (b *volumeBinder) isVolumeBound(pod *v1.Pod, vol *v1.Volume) (bound bool, pvc *v1.PersistentVolumeClaim, err error) { +func (b *volumeBinder) isVolumeBound(logger klog.Logger, pod *v1.Pod, vol *v1.Volume) (bound bool, pvc *v1.PersistentVolumeClaim, err error) { pvcName := "" isEphemeral := false switch { @@ -688,7 +686,7 @@ func (b *volumeBinder) isVolumeBound(pod *v1.Pod, vol *v1.Volume) (bound bool, p return true, nil, nil } - bound, pvc, err = b.isPVCBound(pod.Namespace, pvcName) + bound, pvc, err = b.isPVCBound(logger, pod.Namespace, pvcName) // ... the PVC must be owned by the pod. if isEphemeral && err == nil && pvc != nil { if err := ephemeral.VolumeIsForPod(pod, pvc); err != nil { @@ -698,7 +696,7 @@ func (b *volumeBinder) isVolumeBound(pod *v1.Pod, vol *v1.Volume) (bound bool, p return } -func (b *volumeBinder) isPVCBound(namespace, pvcName string) (bool, *v1.PersistentVolumeClaim, error) { +func (b *volumeBinder) isPVCBound(logger klog.Logger, namespace, pvcName string) (bool, *v1.PersistentVolumeClaim, error) { claim := &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, @@ -713,12 +711,12 @@ func (b *volumeBinder) isPVCBound(namespace, pvcName string) (bool, *v1.Persiste fullyBound := b.isPVCFullyBound(pvc) if fullyBound { - klog.V(5).InfoS("PVC is fully bound to PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvc.Spec.VolumeName)) + logger.V(5).Info("PVC is fully bound to PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvc.Spec.VolumeName)) } else { if pvc.Spec.VolumeName != "" { - klog.V(5).InfoS("PVC is not fully bound to PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvc.Spec.VolumeName)) + logger.V(5).Info("PVC is not fully bound to PV", "PVC", klog.KObj(pvc), "PV", klog.KRef("", pvc.Spec.VolumeName)) } else { - klog.V(5).InfoS("PVC is not bound", "PVC", klog.KObj(pvc)) + logger.V(5).Info("PVC is not bound", "PVC", klog.KObj(pvc)) } } return fullyBound, pvc, nil @@ -729,9 +727,9 @@ func (b *volumeBinder) isPVCFullyBound(pvc *v1.PersistentVolumeClaim) bool { } // arePodVolumesBound returns true if all volumes are fully bound -func (b *volumeBinder) arePodVolumesBound(pod *v1.Pod) bool { +func (b *volumeBinder) arePodVolumesBound(logger klog.Logger, pod *v1.Pod) bool { for _, vol := range pod.Spec.Volumes { - if isBound, _, _ := b.isVolumeBound(pod, &vol); !isBound { + if isBound, _, _ := b.isVolumeBound(logger, pod, &vol); !isBound { // Pod has at least one PVC that needs binding return false } @@ -741,13 +739,13 @@ func (b *volumeBinder) arePodVolumesBound(pod *v1.Pod) bool { // GetPodVolumes returns a pod's PVCs separated into bound, unbound with delayed binding (including provisioning) // and unbound with immediate binding (including prebound) -func (b *volumeBinder) GetPodVolumes(pod *v1.Pod) (boundClaims []*v1.PersistentVolumeClaim, unboundClaimsDelayBinding []*v1.PersistentVolumeClaim, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) { +func (b *volumeBinder) GetPodVolumes(logger klog.Logger, pod *v1.Pod) (boundClaims []*v1.PersistentVolumeClaim, unboundClaimsDelayBinding []*v1.PersistentVolumeClaim, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) { boundClaims = []*v1.PersistentVolumeClaim{} unboundClaimsImmediate = []*v1.PersistentVolumeClaim{} unboundClaimsDelayBinding = []*v1.PersistentVolumeClaim{} for _, vol := range pod.Spec.Volumes { - volumeBound, pvc, err := b.isVolumeBound(pod, &vol) + volumeBound, pvc, err := b.isVolumeBound(logger, pod, &vol) if err != nil { return nil, nil, nil, err } @@ -775,11 +773,11 @@ func (b *volumeBinder) GetPodVolumes(pod *v1.Pod) (boundClaims []*v1.PersistentV return boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate, nil } -func (b *volumeBinder) checkBoundClaims(claims []*v1.PersistentVolumeClaim, node *v1.Node, pod *v1.Pod) (bool, bool, error) { +func (b *volumeBinder) checkBoundClaims(logger klog.Logger, claims []*v1.PersistentVolumeClaim, node *v1.Node) (bool, bool, error) { csiNode, err := b.csiNodeLister.Get(node.Name) if err != nil { // TODO: return the error once CSINode is created by default - klog.V(4).InfoS("Could not get a CSINode object for the node", "node", klog.KObj(node), "err", err) + logger.V(4).Info("Could not get a CSINode object for the node", "err", err) } for _, pvc := range claims { @@ -799,19 +797,19 @@ func (b *volumeBinder) checkBoundClaims(claims []*v1.PersistentVolumeClaim, node err = volume.CheckNodeAffinity(pv, node.Labels) if err != nil { - klog.V(4).InfoS("PersistentVolume and node mismatch for pod", "PV", klog.KRef("", pvName), "node", klog.KObj(node), "pod", klog.KObj(pod), "err", err) + logger.V(4).Info("PersistentVolume and node mismatch for pod", "PV", klog.KRef("", pvName), "err", err) return false, true, nil } - klog.V(5).InfoS("PersistentVolume and node matches for pod", "PV", klog.KRef("", pvName), "node", klog.KObj(node), "pod", klog.KObj(pod)) + logger.V(5).Info("PersistentVolume and node matches for pod", "PV", klog.KRef("", pvName)) } - klog.V(4).InfoS("All bound volumes for pod match with node", "pod", klog.KObj(pod), "node", klog.KObj(node)) + logger.V(4).Info("All bound volumes for pod match with node") return true, true, nil } // findMatchingVolumes tries to find matching volumes for given claims, // and return unbound claims for further provision. -func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (foundMatches bool, bindings []*BindingInfo, unboundClaims []*v1.PersistentVolumeClaim, err error) { +func (b *volumeBinder) findMatchingVolumes(logger klog.Logger, pod *v1.Pod, claimsToBind []*v1.PersistentVolumeClaim, node *v1.Node) (foundMatches bool, bindings []*BindingInfo, unboundClaims []*v1.PersistentVolumeClaim, err error) { // Sort all the claims by increasing size request to get the smallest fits sort.Sort(byPVCSize(claimsToBind)) @@ -830,7 +828,7 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.Persi return false, nil, nil, err } if pv == nil { - klog.V(4).InfoS("No matching volumes for pod", "pod", klog.KObj(pod), "PVC", klog.KObj(pvc), "node", klog.KObj(node)) + logger.V(4).Info("No matching volumes for pod", "PVC", klog.KObj(pvc), "node", klog.KObj(node)) unboundClaims = append(unboundClaims, pvc) foundMatches = false continue @@ -839,11 +837,11 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.Persi // matching PV needs to be excluded so we don't select it again chosenPVs[pv.Name] = pv bindings = append(bindings, &BindingInfo{pv: pv, pvc: pvc}) - klog.V(5).InfoS("Found matching PV for PVC for pod", "PV", klog.KObj(pv), "PVC", klog.KObj(pvc), "node", klog.KObj(node), "pod", klog.KObj(pod)) + logger.V(5).Info("Found matching PV for PVC for pod", "PV", klog.KObj(pv), "PVC", klog.KObj(pvc), "node", klog.KObj(node)) } if foundMatches { - klog.V(4).InfoS("Found matching volumes for pod", "pod", klog.KObj(pod), "node", klog.KObj(node)) + logger.V(4).Info("Found matching volumes for pod") } return @@ -852,7 +850,7 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*v1.Persi // checkVolumeProvisions checks given unbound claims (the claims have gone through func // findMatchingVolumes, and do not have matching volumes for binding), and return true // if all of the claims are eligible for dynamic provision. -func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v1.PersistentVolumeClaim, node *v1.Node) (provisionSatisfied, sufficientStorage bool, dynamicProvisions []*v1.PersistentVolumeClaim, err error) { +func (b *volumeBinder) checkVolumeProvisions(logger klog.Logger, pod *v1.Pod, claimsToProvision []*v1.PersistentVolumeClaim, node *v1.Node) (provisionSatisfied, sufficientStorage bool, dynamicProvisions []*v1.PersistentVolumeClaim, err error) { dynamicProvisions = []*v1.PersistentVolumeClaim{} // We return early with provisionedClaims == nil if a check @@ -870,18 +868,18 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v } provisioner := class.Provisioner if provisioner == "" || provisioner == volume.NotSupportedProvisioner { - klog.V(4).InfoS("Storage class of claim does not support dynamic provisioning", "storageClassName", className, "PVC", klog.KObj(claim)) + logger.V(4).Info("Storage class of claim does not support dynamic provisioning", "storageClassName", className, "PVC", klog.KObj(claim)) return false, true, nil, nil } // Check if the node can satisfy the topology requirement in the class if !v1helper.MatchTopologySelectorTerms(class.AllowedTopologies, labels.Set(node.Labels)) { - klog.V(4).InfoS("Node cannot satisfy provisioning topology requirements of claim", "node", klog.KObj(node), "PVC", klog.KObj(claim)) + logger.V(4).Info("Node cannot satisfy provisioning topology requirements of claim", "PVC", klog.KObj(claim)) return false, true, nil, nil } // Check storage capacity. - sufficient, err := b.hasEnoughCapacity(provisioner, claim, class, node) + sufficient, err := b.hasEnoughCapacity(logger, provisioner, claim, class, node) if err != nil { return false, false, nil, err } @@ -893,7 +891,7 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v dynamicProvisions = append(dynamicProvisions, claim) } - klog.V(4).InfoS("Provisioning for claims of pod that has no matching volumes...", "claimCount", len(claimsToProvision), "pod", klog.KObj(pod), "node", klog.KObj(node)) + logger.V(4).Info("Provisioning for claims of pod that has no matching volumes...", "claimCount", len(claimsToProvision)) return true, true, dynamicProvisions, nil } @@ -912,7 +910,7 @@ func (b *volumeBinder) revertAssumedPVCs(claims []*v1.PersistentVolumeClaim) { // hasEnoughCapacity checks whether the provisioner has enough capacity left for a new volume of the given size // that is available from the node. -func (b *volumeBinder) hasEnoughCapacity(provisioner string, claim *v1.PersistentVolumeClaim, storageClass *storagev1.StorageClass, node *v1.Node) (bool, error) { +func (b *volumeBinder) hasEnoughCapacity(logger klog.Logger, provisioner string, claim *v1.PersistentVolumeClaim, storageClass *storagev1.StorageClass, node *v1.Node) (bool, error) { quantity, ok := claim.Spec.Resources.Requests[v1.ResourceStorage] if !ok { // No capacity to check for. @@ -945,7 +943,7 @@ func (b *volumeBinder) hasEnoughCapacity(provisioner string, claim *v1.Persisten for _, capacity := range capacities { if capacity.StorageClassName == storageClass.Name && capacitySufficient(capacity, sizeInBytes) && - b.nodeHasAccess(node, capacity) { + b.nodeHasAccess(logger, node, capacity) { // Enough capacity found. return true, nil } @@ -953,8 +951,8 @@ func (b *volumeBinder) hasEnoughCapacity(provisioner string, claim *v1.Persisten // TODO (?): this doesn't give any information about which pools where considered and why // they had to be rejected. Log that above? But that might be a lot of log output... - klog.V(4).InfoS("Node has no accessible CSIStorageCapacity with enough capacity for PVC", - "node", klog.KObj(node), "PVC", klog.KObj(claim), "size", sizeInBytes, "storageClass", klog.KObj(storageClass)) + logger.V(4).Info("Node has no accessible CSIStorageCapacity with enough capacity for PVC", + "PVC", klog.KObj(claim), "size", sizeInBytes, "storageClass", klog.KObj(storageClass)) return false, nil } @@ -967,7 +965,7 @@ func capacitySufficient(capacity *storagev1.CSIStorageCapacity, sizeInBytes int6 return limit != nil && limit.Value() >= sizeInBytes } -func (b *volumeBinder) nodeHasAccess(node *v1.Node, capacity *storagev1.CSIStorageCapacity) bool { +func (b *volumeBinder) nodeHasAccess(logger klog.Logger, node *v1.Node, capacity *storagev1.CSIStorageCapacity) bool { if capacity.NodeTopology == nil { // Unavailable return false @@ -975,7 +973,7 @@ func (b *volumeBinder) nodeHasAccess(node *v1.Node, capacity *storagev1.CSIStora // Only matching by label is supported. selector, err := metav1.LabelSelectorAsSelector(capacity.NodeTopology) if err != nil { - klog.ErrorS(err, "Unexpected error converting to a label selector", "nodeTopology", capacity.NodeTopology) + logger.Error(err, "Unexpected error converting to a label selector", "nodeTopology", capacity.NodeTopology) return false } return selector.Matches(labels.Set(node.Labels)) diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go index 094144fa6b33..56520bcd9512 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go @@ -43,6 +43,8 @@ import ( featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" + _ "k8s.io/klog/v2/ktesting/init" "k8s.io/kubernetes/pkg/controller" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" "k8s.io/kubernetes/pkg/features" @@ -122,10 +124,6 @@ var ( zone1Labels = map[string]string{v1.LabelFailureDomainBetaZone: "us-east-1", v1.LabelFailureDomainBetaRegion: "us-east-1a"} ) -func init() { - klog.InitFlags(nil) -} - type testEnv struct { client clientset.Interface reactor *pvtesting.VolumeReactor @@ -142,7 +140,8 @@ type testEnv struct { internalCSIStorageCapacityInformer storageinformers.CSIStorageCapacityInformer } -func newTestBinder(t *testing.T, stopCh <-chan struct{}) *testEnv { +func newTestBinder(t *testing.T, ctx context.Context) *testEnv { + logger := klog.FromContext(ctx) client := &fake.Clientset{} reactor := pvtesting.NewVolumeReactor(client, nil, nil, nil) // TODO refactor all tests to use real watch mechanism, see #72327 @@ -169,6 +168,7 @@ func newTestBinder(t *testing.T, stopCh <-chan struct{}) *testEnv { CSIStorageCapacityInformer: csiStorageCapacityInformer, } binder := NewVolumeBinder( + logger, client, podInformer, nodeInformer, @@ -180,10 +180,10 @@ func newTestBinder(t *testing.T, stopCh <-chan struct{}) *testEnv { 10*time.Second) // Wait for informers cache sync - informerFactory.Start(stopCh) - for v, synced := range informerFactory.WaitForCacheSync(stopCh) { + informerFactory.Start(ctx.Done()) + for v, synced := range informerFactory.WaitForCacheSync(ctx.Done()) { if !synced { - klog.ErrorS(nil, "Error syncing informer", "informer", v) + logger.Error(nil, "Error syncing informer", "informer", v) os.Exit(1) } } @@ -820,15 +820,15 @@ func checkReasons(t *testing.T, actual, expected ConflictReasons) { } // findPodVolumes gets and finds volumes for given pod and node -func findPodVolumes(binder SchedulerVolumeBinder, pod *v1.Pod, node *v1.Node) (*PodVolumes, ConflictReasons, error) { - boundClaims, claimsToBind, unboundClaimsImmediate, err := binder.GetPodVolumes(pod) +func findPodVolumes(logger klog.Logger, binder SchedulerVolumeBinder, pod *v1.Pod, node *v1.Node) (*PodVolumes, ConflictReasons, error) { + boundClaims, claimsToBind, unboundClaimsImmediate, err := binder.GetPodVolumes(logger, pod) if err != nil { return nil, nil, err } if len(unboundClaimsImmediate) > 0 { return nil, nil, fmt.Errorf("pod has unbound immediate PersistentVolumeClaims") } - return binder.FindPodVolumes(pod, boundClaims, claimsToBind, node) + return binder.FindPodVolumes(logger, pod, boundClaims, claimsToBind, node) } func TestFindPodVolumesWithoutProvisioning(t *testing.T) { @@ -980,11 +980,12 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) { } run := func(t *testing.T, scenario scenarioType, csiDriver *storagev1.CSIDriver) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initVolumes(scenario.pvs, scenario.pvs) if csiDriver != nil { testEnv.addCSIDriver(csiDriver) @@ -1005,7 +1006,7 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) { } // Execute - podVolumes, reasons, err := findPodVolumes(testEnv.binder, scenario.pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, scenario.pod, testNode) // Validate if !scenario.shouldFail && err != nil { @@ -1107,11 +1108,12 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) { } run := func(t *testing.T, scenario scenarioType, csiDriver *storagev1.CSIDriver) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initVolumes(scenario.pvs, scenario.pvs) if csiDriver != nil { testEnv.addCSIDriver(csiDriver) @@ -1132,7 +1134,7 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) { } // Execute - podVolumes, reasons, err := findPodVolumes(testEnv.binder, scenario.pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, scenario.pod, testNode) // Validate if !scenario.shouldFail && err != nil { @@ -1214,13 +1216,14 @@ func TestFindPodVolumesWithCSIMigration(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigrationGCE, true)() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initVolumes(scenario.pvs, scenario.pvs) var node *v1.Node @@ -1250,7 +1253,7 @@ func TestFindPodVolumesWithCSIMigration(t *testing.T) { } // Execute - _, reasons, err := findPodVolumes(testEnv.binder, scenario.pod, node) + _, reasons, err := findPodVolumes(logger, testEnv.binder, scenario.pod, node) // Validate if !scenario.shouldFail && err != nil { @@ -1333,11 +1336,12 @@ func TestAssumePodVolumes(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initClaims(scenario.podPVCs, scenario.podPVCs) pod := makePod("test-pod"). withNamespace("testns"). @@ -1350,7 +1354,7 @@ func TestAssumePodVolumes(t *testing.T) { testEnv.initVolumes(scenario.pvs, scenario.pvs) // Execute - allBound, err := testEnv.binder.AssumePodVolumes(pod, "node1", podVolumes) + allBound, err := testEnv.binder.AssumePodVolumes(logger, pod, "node1", podVolumes) // Validate if !scenario.shouldFail && err != nil { @@ -1382,7 +1386,8 @@ func TestAssumePodVolumes(t *testing.T) { } func TestRevertAssumedPodVolumes(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() podPVCs := []*v1.PersistentVolumeClaim{unboundPVC, provisionedPVC} @@ -1393,7 +1398,7 @@ func TestRevertAssumedPodVolumes(t *testing.T) { expectedProvisionings := []*v1.PersistentVolumeClaim{selectedNodePVC} // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initClaims(podPVCs, podPVCs) pod := makePod("test-pod"). withNamespace("testns"). @@ -1405,13 +1410,13 @@ func TestRevertAssumedPodVolumes(t *testing.T) { } testEnv.initVolumes(pvs, pvs) - allbound, err := testEnv.binder.AssumePodVolumes(pod, "node1", podVolumes) + allbound, err := testEnv.binder.AssumePodVolumes(logger, pod, "node1", podVolumes) if allbound || err != nil { t.Errorf("No volumes are assumed") } testEnv.validateAssume(t, pod, expectedBindings, expectedProvisionings) - testEnv.binder.RevertAssumedPodVolumes(podVolumes) + testEnv.binder.RevertAssumedPodVolumes(logger, podVolumes) testEnv.validateCacheRestored(t, pod, bindings, provisionedPVCs) } @@ -1510,11 +1515,12 @@ func TestBindAPIUpdate(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) pod := makePod("test-pod"). withNamespace("testns"). withNodeName("node1").Pod @@ -1529,7 +1535,7 @@ func TestBindAPIUpdate(t *testing.T) { testEnv.assumeVolumes(t, "node1", pod, scenario.bindings, scenario.provisionedPVCs) // Execute - err := testEnv.internalBinder.bindAPIUpdate(pod, scenario.bindings, scenario.provisionedPVCs) + err := testEnv.internalBinder.bindAPIUpdate(logger, pod.Name, scenario.bindings, scenario.provisionedPVCs) // Validate if !scenario.shouldFail && err != nil { @@ -1708,13 +1714,14 @@ func TestCheckBindings(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup pod := makePod("test-pod"). withNamespace("testns"). withNodeName("node1").Pod - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.internalPodInformer.Informer().GetIndexer().Add(pod) testEnv.initNodes([]*v1.Node{node1}) testEnv.initVolumes(scenario.initPVs, nil) @@ -1738,7 +1745,7 @@ func TestCheckBindings(t *testing.T) { } // Execute - allBound, err := testEnv.internalBinder.checkBindings(pod, scenario.bindings, scenario.provisionedPVCs) + allBound, err := testEnv.internalBinder.checkBindings(logger, pod, scenario.bindings, scenario.provisionedPVCs) // Validate if !scenario.shouldFail && err != nil { @@ -1839,7 +1846,8 @@ func TestCheckBindingsWithCSIMigration(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigrationGCE, scenario.migrationEnabled)() @@ -1848,7 +1856,7 @@ func TestCheckBindingsWithCSIMigration(t *testing.T) { pod := makePod("test-pod"). withNamespace("testns"). withNodeName("node1").Pod - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.internalPodInformer.Informer().GetIndexer().Add(pod) testEnv.initNodes(scenario.initNodes) testEnv.initCSINodes(scenario.initCSINodes) @@ -1865,7 +1873,7 @@ func TestCheckBindingsWithCSIMigration(t *testing.T) { } // Execute - allBound, err := testEnv.internalBinder.checkBindings(pod, scenario.bindings, scenario.provisionedPVCs) + allBound, err := testEnv.internalBinder.checkBindings(logger, pod, scenario.bindings, scenario.provisionedPVCs) // Validate if !scenario.shouldFail && err != nil { @@ -2031,13 +2039,14 @@ func TestBindPodVolumes(t *testing.T) { } run := func(t *testing.T, scenario scenarioType) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup pod := makePod("test-pod"). withNamespace("testns"). withNodeName("node1").Pod - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.internalPodInformer.Informer().GetIndexer().Add(pod) if scenario.nodes == nil { scenario.nodes = []*v1.Node{node1} @@ -2075,7 +2084,7 @@ func TestBindPodVolumes(t *testing.T) { go func(scenario scenarioType) { time.Sleep(5 * time.Second) // Sleep a while to run after bindAPIUpdate in BindPodVolumes - klog.V(5).InfoS("Running delay function") + logger.V(5).Info("Running delay function") scenario.delayFunc(t, ctx, testEnv, pod, scenario.initPVs, scenario.initPVCs) }(scenario) } @@ -2085,7 +2094,7 @@ func TestBindPodVolumes(t *testing.T) { StaticBindings: bindings, DynamicProvisions: claimsToProvision, } - err := testEnv.binder.BindPodVolumes(pod, podVolumes) + err := testEnv.binder.BindPodVolumes(logger, pod, podVolumes) // Validate if !scenario.shouldFail && err != nil { @@ -2111,9 +2120,10 @@ func TestFindAssumeVolumes(t *testing.T) { pvs := []*v1.PersistentVolume{pvNode2, pvNode1a, pvNode1c} // Setup - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) testEnv.initVolumes(pvs, pvs) testEnv.initClaims(podPVCs, podPVCs) pod := makePod("test-pod"). @@ -2132,7 +2142,7 @@ func TestFindAssumeVolumes(t *testing.T) { // Execute // 1. Find matching PVs - podVolumes, reasons, err := findPodVolumes(testEnv.binder, pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, pod, testNode) if err != nil { t.Errorf("Test failed: FindPodVolumes returned error: %v", err) } @@ -2142,7 +2152,7 @@ func TestFindAssumeVolumes(t *testing.T) { expectedBindings := podVolumes.StaticBindings // 2. Assume matches - allBound, err := testEnv.binder.AssumePodVolumes(pod, testNode.Name, podVolumes) + allBound, err := testEnv.binder.AssumePodVolumes(logger, pod, testNode.Name, podVolumes) if err != nil { t.Errorf("Test failed: AssumePodVolumes returned error: %v", err) } @@ -2158,7 +2168,7 @@ func TestFindAssumeVolumes(t *testing.T) { // This should always return the original chosen pv // Run this many times in case sorting returns different orders for the two PVs. for i := 0; i < 50; i++ { - podVolumes, reasons, err := findPodVolumes(testEnv.binder, pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, pod, testNode) if err != nil { t.Errorf("Test failed: FindPodVolumes returned error: %v", err) } @@ -2267,11 +2277,13 @@ func TestCapacity(t *testing.T) { } run := func(t *testing.T, scenario scenarioType, optIn bool) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() // Setup: the driver has the feature enabled, but the scheduler might not. - testEnv := newTestBinder(t, ctx.Done()) + testEnv := newTestBinder(t, ctx) + testEnv.addCSIDriver(makeCSIDriver(provisioner, optIn)) testEnv.addCSIStorageCapacities(scenario.capacities) @@ -2285,7 +2297,7 @@ func TestCapacity(t *testing.T) { withPVCSVolume(scenario.pvcs).Pod // Execute - podVolumes, reasons, err := findPodVolumes(testEnv.binder, pod, testNode) + podVolumes, reasons, err := findPodVolumes(logger, testEnv.binder, pod, testNode) // Validate shouldFail := scenario.shouldFail diff --git a/pkg/scheduler/framework/plugins/volumebinding/fake_binder.go b/pkg/scheduler/framework/plugins/volumebinding/fake_binder.go index 4a07eff9a2fd..cce54242b247 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/fake_binder.go +++ b/pkg/scheduler/framework/plugins/volumebinding/fake_binder.go @@ -16,7 +16,10 @@ limitations under the License. package volumebinding -import v1 "k8s.io/api/core/v1" +import ( + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" +) // FakeVolumeBinderConfig holds configurations for fake volume binder. type FakeVolumeBinderConfig struct { @@ -43,26 +46,26 @@ type FakeVolumeBinder struct { } // GetPodVolumes implements SchedulerVolumeBinder.GetPodVolumes. -func (b *FakeVolumeBinder) GetPodVolumes(pod *v1.Pod) (boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) { +func (b *FakeVolumeBinder) GetPodVolumes(_ klog.Logger, pod *v1.Pod) (boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) { return nil, nil, nil, nil } // FindPodVolumes implements SchedulerVolumeBinder.FindPodVolumes. -func (b *FakeVolumeBinder) FindPodVolumes(pod *v1.Pod, _, _ []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) { +func (b *FakeVolumeBinder) FindPodVolumes(_ klog.Logger, pod *v1.Pod, _, _ []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) { return nil, b.config.FindReasons, b.config.FindErr } // AssumePodVolumes implements SchedulerVolumeBinder.AssumePodVolumes. -func (b *FakeVolumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (bool, error) { +func (b *FakeVolumeBinder) AssumePodVolumes(_ klog.Logger, assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (bool, error) { b.AssumeCalled = true return b.config.AllBound, b.config.AssumeErr } // RevertAssumedPodVolumes implements SchedulerVolumeBinder.RevertAssumedPodVolumes -func (b *FakeVolumeBinder) RevertAssumedPodVolumes(_ *PodVolumes) {} +func (b *FakeVolumeBinder) RevertAssumedPodVolumes(_ klog.Logger, _ *PodVolumes) {} // BindPodVolumes implements SchedulerVolumeBinder.BindPodVolumes. -func (b *FakeVolumeBinder) BindPodVolumes(assumedPod *v1.Pod, podVolumes *PodVolumes) error { +func (b *FakeVolumeBinder) BindPodVolumes(_ klog.Logger, assumedPod *v1.Pod, podVolumes *PodVolumes) error { b.BindCalled = true return b.config.BindErr } diff --git a/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go b/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go index 48ee01dea4eb..df4b87fbd371 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go +++ b/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go @@ -173,7 +173,8 @@ func (pl *VolumeBinding) PreFilter(ctx context.Context, state *framework.CycleSt state.Write(stateKey, &stateData{skip: true}) return nil, nil } - boundClaims, claimsToBind, unboundClaimsImmediate, err := pl.Binder.GetPodVolumes(pod) + logger := klog.FromContext(ctx) + boundClaims, claimsToBind, unboundClaimsImmediate, err := pl.Binder.GetPodVolumes(logger, pod) if err != nil { return nil, framework.AsStatus(err) } @@ -222,6 +223,7 @@ func getStateData(cs *framework.CycleState) (*stateData, error) { // The predicate returns true if all bound PVCs have compatible PVs with the node, and if all unbound // PVCs can be matched with an available and node-compatible PV. func (pl *VolumeBinding) Filter(ctx context.Context, cs *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { + logger := klog.FromContext(ctx) node := nodeInfo.Node() if node == nil { return framework.NewStatus(framework.Error, "node not found") @@ -236,7 +238,7 @@ func (pl *VolumeBinding) Filter(ctx context.Context, cs *framework.CycleState, p return nil } - podVolumes, reasons, err := pl.Binder.FindPodVolumes(pod, state.boundClaims, state.claimsToBind, node) + podVolumes, reasons, err := pl.Binder.FindPodVolumes(logger, pod, state.boundClaims, state.claimsToBind, node) if err != nil { return framework.AsStatus(err) @@ -301,7 +303,8 @@ func (pl *VolumeBinding) Reserve(ctx context.Context, cs *framework.CycleState, // we don't need to hold the lock as only one node will be reserved for the given pod podVolumes, ok := state.podVolumesByNode[nodeName] if ok { - allBound, err := pl.Binder.AssumePodVolumes(pod, nodeName, podVolumes) + logger := klog.FromContext(ctx) + allBound, err := pl.Binder.AssumePodVolumes(logger, pod, nodeName, podVolumes) if err != nil { return framework.AsStatus(err) } @@ -332,13 +335,14 @@ func (pl *VolumeBinding) PreBind(ctx context.Context, cs *framework.CycleState, if !ok { return framework.AsStatus(fmt.Errorf("no pod volumes found for node %q", nodeName)) } - klog.V(5).InfoS("Trying to bind volumes for pod", "pod", klog.KObj(pod)) - err = pl.Binder.BindPodVolumes(pod, podVolumes) + logger := klog.FromContext(ctx) + logger.V(5).Info("Trying to bind volumes for pod") + err = pl.Binder.BindPodVolumes(logger, pod, podVolumes) if err != nil { - klog.V(1).InfoS("Failed to bind volumes for pod", "pod", klog.KObj(pod), "err", err) + logger.V(1).Info("Failed to bind volumes for pod", "err", err) return framework.AsStatus(err) } - klog.V(5).InfoS("Success binding volumes for pod", "pod", klog.KObj(pod)) + logger.V(5).Info("Success binding volumes for pod") return nil } @@ -354,7 +358,8 @@ func (pl *VolumeBinding) Unreserve(ctx context.Context, cs *framework.CycleState if !ok { return } - pl.Binder.RevertAssumedPodVolumes(podVolumes) + logger := klog.FromContext(ctx) + pl.Binder.RevertAssumedPodVolumes(logger, podVolumes) return } @@ -379,7 +384,7 @@ func New(plArgs runtime.Object, fh framework.Handle, fts feature.Features) (fram CSIDriverInformer: fh.SharedInformerFactory().Storage().V1().CSIDrivers(), CSIStorageCapacityInformer: fh.SharedInformerFactory().Storage().V1().CSIStorageCapacities(), } - binder := NewVolumeBinder(fh.ClientSet(), podInformer, nodeInformer, csiNodeInformer, pvcInformer, pvInformer, storageClassInformer, capacityCheck, time.Duration(args.BindTimeoutSeconds)*time.Second) + binder := NewVolumeBinder(fh.Logger(), fh.ClientSet(), podInformer, nodeInformer, csiNodeInformer, pvcInformer, pvInformer, storageClassInformer, capacityCheck, time.Duration(args.BindTimeoutSeconds)*time.Second) // build score function var scorer volumeCapacityScorer diff --git a/pkg/scheduler/framework/plugins/volumezone/volume_zone.go b/pkg/scheduler/framework/plugins/volumezone/volume_zone.go index fb548121d452..16902a87f119 100644 --- a/pkg/scheduler/framework/plugins/volumezone/volume_zone.go +++ b/pkg/scheduler/framework/plugins/volumezone/volume_zone.go @@ -104,6 +104,7 @@ func (pl *VolumeZone) Filter(ctx context.Context, _ *framework.CycleState, pod * return nil } + logger := klog.FromContext(ctx) for i := range pod.Spec.Volumes { volume := pod.Spec.Volumes[i] if volume.PersistentVolumeClaim == nil { @@ -152,12 +153,12 @@ func (pl *VolumeZone) Filter(ctx context.Context, _ *framework.CycleState, pod * nodeV := nodeConstraints[k] volumeVSet, err := volumehelpers.LabelZonesToSet(v) if err != nil { - klog.InfoS("Failed to parse label, ignoring the label", "label", fmt.Sprintf("%s:%s", k, v), "err", err) + logger.Info("Failed to parse label, ignoring the label", "label", fmt.Sprintf("%s:%s", k, v), "err", err) continue } if !volumeVSet.Has(nodeV) { - klog.V(10).InfoS("Won't schedule pod onto node due to volume (mismatch on label key)", "pod", klog.KObj(pod), "node", klog.KObj(node), "PV", klog.KRef("", pvName), "PVLabelKey", k) + logger.V(10).Info("Won't schedule pod onto node due to volume (mismatch on label key)", "pod", klog.KObj(pod), "node", klog.KObj(node), "PV", klog.KRef("", pvName), "PVLabelKey", k) return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonConflict) } } diff --git a/pkg/scheduler/framework/preemption/preemption.go b/pkg/scheduler/framework/preemption/preemption.go index af5e46d1ae2b..d8c483ff6b00 100644 --- a/pkg/scheduler/framework/preemption/preemption.go +++ b/pkg/scheduler/framework/preemption/preemption.go @@ -136,6 +136,8 @@ type Evaluator struct { // - . It's the regular happy path // and the non-empty nominatedNodeName will be applied to the preemptor pod. func (ev *Evaluator) Preempt(ctx context.Context, pod *v1.Pod, m framework.NodeToStatusMap) (*framework.PostFilterResult, *framework.Status) { + logger := klog.FromContext(ctx) + // 0) Fetch the latest version of . // It's safe to directly fetch pod here. Because the informer cache has already been // initialized when creating the Scheduler obj, i.e., factory.go#MakeDefaultErrorFunc(). @@ -143,13 +145,13 @@ func (ev *Evaluator) Preempt(ctx context.Context, pod *v1.Pod, m framework.NodeT podNamespace, podName := pod.Namespace, pod.Name pod, err := ev.PodLister.Pods(pod.Namespace).Get(pod.Name) if err != nil { - klog.ErrorS(err, "Getting the updated preemptor pod object", "pod", klog.KRef(podNamespace, podName)) + logger.Error(err, "Getting the updated preemptor pod object", "pod", klog.KRef(podNamespace, podName)) return nil, framework.AsStatus(err) } // 1) Ensure the preemptor is eligible to preempt other pods. if ok, msg := ev.PodEligibleToPreemptOthers(pod, m[pod.Status.NominatedNodeName]); !ok { - klog.V(5).InfoS("Pod is not eligible for preemption", "pod", klog.KObj(pod), "reason", msg) + logger.V(5).Info("Pod is not eligible for preemption", "pod", klog.KObj(pod), "reason", msg) return nil, framework.NewStatus(framework.Unschedulable, msg) } @@ -174,13 +176,13 @@ func (ev *Evaluator) Preempt(ctx context.Context, pod *v1.Pod, m framework.NodeT } // 3) Interact with registered Extenders to filter out some candidates if needed. - candidates, status := ev.callExtenders(pod, candidates) + candidates, status := ev.callExtenders(logger, pod, candidates) if !status.IsSuccess() { return nil, status } // 4) Find the best candidate. - bestCandidate := ev.SelectCandidate(candidates) + bestCandidate := ev.SelectCandidate(logger, candidates) if bestCandidate == nil || len(bestCandidate.Name()) == 0 { return nil, framework.NewStatus(framework.Unschedulable, "no candidate node for preemption") } @@ -203,12 +205,13 @@ func (ev *Evaluator) findCandidates(ctx context.Context, pod *v1.Pod, m framewor if len(allNodes) == 0 { return nil, nil, errors.New("no nodes available") } + logger := klog.FromContext(ctx) potentialNodes, unschedulableNodeStatus := nodesWherePreemptionMightHelp(allNodes, m) if len(potentialNodes) == 0 { - klog.V(3).InfoS("Preemption will not help schedule pod on any node", "pod", klog.KObj(pod)) + logger.V(3).Info("Preemption will not help schedule pod on any node", "pod", klog.KObj(pod)) // In this case, we should clean-up any existing nominated node name of the pod. if err := util.ClearNominatedNodeName(ctx, ev.Handler.ClientSet(), pod); err != nil { - klog.ErrorS(err, "Cannot clear 'NominatedNodeName' field of pod", "pod", klog.KObj(pod)) + logger.Error(err, "Cannot clear 'NominatedNodeName' field of pod", "pod", klog.KObj(pod)) // We do not return as this error is not critical. } return nil, unschedulableNodeStatus, nil @@ -220,12 +223,12 @@ func (ev *Evaluator) findCandidates(ctx context.Context, pod *v1.Pod, m framewor } offset, numCandidates := ev.GetOffsetAndNumCandidates(int32(len(potentialNodes))) - if klogV := klog.V(5); klogV.Enabled() { + if logger := logger.V(5); logger.Enabled() { var sample []string for i := offset; i < offset+10 && i < int32(len(potentialNodes)); i++ { sample = append(sample, potentialNodes[i].Node().Name) } - klogV.InfoS("Selecting candidates from a pool of nodes", "potentialNodesCount", len(potentialNodes), "offset", offset, "sampleLength", len(sample), "sample", sample, "candidates", numCandidates) + logger.Info("Selecting candidates from a pool of nodes", "potentialNodesCount", len(potentialNodes), "offset", offset, "sampleLength", len(sample), "sample", sample, "candidates", numCandidates) } candidates, nodeStatuses, err := ev.DryRunPreemption(ctx, pod, potentialNodes, pdbs, offset, numCandidates) for node, nodeStatus := range unschedulableNodeStatus { @@ -238,7 +241,7 @@ func (ev *Evaluator) findCandidates(ctx context.Context, pod *v1.Pod, m framewor // We will only check with extenders that support preemption. // Extenders which do not support preemption may later prevent preemptor from being scheduled on the nominated // node. In that case, scheduler will find a different host for the preemptor in subsequent scheduling cycles. -func (ev *Evaluator) callExtenders(pod *v1.Pod, candidates []Candidate) ([]Candidate, *framework.Status) { +func (ev *Evaluator) callExtenders(logger klog.Logger, pod *v1.Pod, candidates []Candidate) ([]Candidate, *framework.Status) { extenders := ev.Handler.Extenders() nodeLister := ev.Handler.SnapshotSharedLister().NodeInfos() if len(extenders) == 0 { @@ -255,10 +258,10 @@ func (ev *Evaluator) callExtenders(pod *v1.Pod, candidates []Candidate) ([]Candi if !extender.SupportsPreemption() || !extender.IsInterested(pod) { continue } - nodeNameToVictims, err := extender.ProcessPreemption(pod, victimsMap, nodeLister) + nodeNameToVictims, err := extender.ProcessPreemption(logger, pod, victimsMap, nodeLister) if err != nil { if extender.IsIgnorable() { - klog.InfoS("Skipping extender as it returned error and has ignorable flag set", + logger.Info("Skipping extender as it returned error and has ignorable flag set", "extender", extender, "err", err) continue } @@ -269,7 +272,7 @@ func (ev *Evaluator) callExtenders(pod *v1.Pod, candidates []Candidate) ([]Candi if victims == nil || len(victims.Pods) == 0 { if extender.IsIgnorable() { delete(nodeNameToVictims, nodeName) - klog.InfoS("Ignoring node without victims", "node", klog.KRef("", nodeName)) + logger.Info("Ignoring node without victims", "node", klog.KRef("", nodeName)) continue } return nil, framework.AsStatus(fmt.Errorf("expected at least one victim pod on node %q", nodeName)) @@ -298,7 +301,7 @@ func (ev *Evaluator) callExtenders(pod *v1.Pod, candidates []Candidate) ([]Candi // SelectCandidate chooses the best-fit candidate from given and return it. // NOTE: This method is exported for easier testing in default preemption. -func (ev *Evaluator) SelectCandidate(candidates []Candidate) Candidate { +func (ev *Evaluator) SelectCandidate(logger klog.Logger, candidates []Candidate) Candidate { if len(candidates) == 0 { return nil } @@ -307,7 +310,7 @@ func (ev *Evaluator) SelectCandidate(candidates []Candidate) Candidate { } victimsMap := ev.CandidatesToVictimsMap(candidates) - candidateNode := pickOneNodeForPreemption(victimsMap) + candidateNode := pickOneNodeForPreemption(logger, victimsMap) // Same as candidatesToVictimsMap, this logic is not applicable for out-of-tree // preemption plugins that exercise different candidates on the same nominated node. @@ -319,7 +322,7 @@ func (ev *Evaluator) SelectCandidate(candidates []Candidate) Candidate { } // We shouldn't reach here. - klog.ErrorS(errors.New("no candidate selected"), "Should not reach here", "candidates", candidates) + logger.Error(errors.New("no candidate selected"), "Should not reach here", "candidates", candidates) // To not break the whole flow, return the first candidate. return candidates[0] } @@ -329,6 +332,7 @@ func (ev *Evaluator) SelectCandidate(candidates []Candidate) Candidate { // - Reject the victim pods if they are in waitingPod map // - Clear the low-priority pods' nominatedNodeName status if needed func (ev *Evaluator) prepareCandidate(ctx context.Context, c Candidate, pod *v1.Pod, pluginName string) *framework.Status { + logger := klog.FromContext(ctx) fh := ev.Handler cs := ev.Handler.ClientSet() for _, victim := range c.Victims().Pods { @@ -337,7 +341,7 @@ func (ev *Evaluator) prepareCandidate(ctx context.Context, c Candidate, pod *v1. if waitingPod := fh.GetWaitingPod(victim.UID); waitingPod != nil { waitingPod.Reject(pluginName, "preempted") } else if err := util.DeletePod(ctx, cs, victim); err != nil { - klog.ErrorS(err, "Preempting pod", "pod", klog.KObj(victim), "preemptor", klog.KObj(pod)) + logger.Error(err, "Preempting pod", "pod", klog.KObj(victim), "preemptor", klog.KObj(pod)) return framework.AsStatus(err) } fh.EventRecorder().Eventf(victim, pod, v1.EventTypeNormal, "Preempted", "Preempting", "Preempted by %v/%v on node %v", @@ -349,9 +353,9 @@ func (ev *Evaluator) prepareCandidate(ctx context.Context, c Candidate, pod *v1. // this node. So, we should remove their nomination. Removing their // nomination updates these pods and moves them to the active queue. It // lets scheduler find another place for them. - nominatedPods := getLowerPriorityNominatedPods(fh, pod, c.Name()) + nominatedPods := getLowerPriorityNominatedPods(logger, fh, pod, c.Name()) if err := util.ClearNominatedNodeName(ctx, cs, nominatedPods...); err != nil { - klog.ErrorS(err, "Cannot clear 'NominatedNodeName' field") + logger.Error(err, "Cannot clear 'NominatedNodeName' field") // We do not return as this error is not critical. } @@ -394,7 +398,7 @@ func getPodDisruptionBudgets(pdbLister policylisters.PodDisruptionBudgetLister) // 6. If there are still ties, the first such node is picked (sort of randomly). // The 'minNodes1' and 'minNodes2' are being reused here to save the memory // allocation and garbage collection time. -func pickOneNodeForPreemption(nodesToVictims map[string]*extenderv1.Victims) string { +func pickOneNodeForPreemption(logger klog.Logger, nodesToVictims map[string]*extenderv1.Victims) string { if len(nodesToVictims) == 0 { return "" } @@ -493,7 +497,7 @@ func pickOneNodeForPreemption(nodesToVictims map[string]*extenderv1.Victims) str if latestStartTime == nil { // If the earliest start time of all pods on the 1st node is nil, just return it, // which is not expected to happen. - klog.ErrorS(errors.New("earliestStartTime is nil for node"), "Should not reach here", "node", klog.KRef("", minNodes2[0])) + logger.Error(errors.New("earliestStartTime is nil for node"), "Should not reach here", "node", klog.KRef("", minNodes2[0])) return minNodes2[0] } nodeToReturn := minNodes2[0] @@ -502,7 +506,7 @@ func pickOneNodeForPreemption(nodesToVictims map[string]*extenderv1.Victims) str // Get earliest start time of all pods on the current node. earliestStartTimeOnNode := util.GetEarliestPodStartTime(nodesToVictims[node]) if earliestStartTimeOnNode == nil { - klog.ErrorS(errors.New("earliestStartTime is nil for node"), "Should not reach here", "node", klog.KRef("", node)) + logger.Error(errors.New("earliestStartTime is nil for node"), "Should not reach here", "node", klog.KRef("", node)) continue } if earliestStartTimeOnNode.After(latestStartTime.Time) { @@ -521,8 +525,8 @@ func pickOneNodeForPreemption(nodesToVictims map[string]*extenderv1.Victims) str // manipulation of NodeInfo and PreFilter state per nominated pod. It may not be // worth the complexity, especially because we generally expect to have a very // small number of nominated pods per node. -func getLowerPriorityNominatedPods(pn framework.PodNominator, pod *v1.Pod, nodeName string) []*v1.Pod { - podInfos := pn.NominatedPodsForNode(nodeName) +func getLowerPriorityNominatedPods(logger klog.Logger, pn framework.PodNominator, pod *v1.Pod, nodeName string) []*v1.Pod { + podInfos := pn.NominatedPodsForNode(logger, nodeName) if len(podInfos) == 0 { return nil diff --git a/pkg/scheduler/framework/preemption/preemption_test.go b/pkg/scheduler/framework/preemption/preemption_test.go index 14ecc9d87857..9c9561c8829e 100644 --- a/pkg/scheduler/framework/preemption/preemption_test.go +++ b/pkg/scheduler/framework/preemption/preemption_test.go @@ -29,6 +29,8 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" clientsetfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/klog/v2/ktesting" + _ "k8s.io/klog/v2/ktesting/init" extenderv1 "k8s.io/kube-scheduler/extender/v1" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/parallelize" @@ -266,6 +268,7 @@ func TestDryRunPreemption(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) registeredPlugins := append([]st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New)}, st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), @@ -285,6 +288,7 @@ func TestDryRunPreemption(t *testing.T) { frameworkruntime.WithInformerFactory(informerFactory), frameworkruntime.WithParallelism(parallelism), frameworkruntime.WithSnapshotSharedLister(internalcache.NewSnapshot(tt.testPods, tt.nodes)), + frameworkruntime.WithLogger(logger), ) if err != nil { t.Fatal(err) diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index e12ed28c875b..7a73bd162ebb 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -91,6 +91,7 @@ type frameworkImpl struct { kubeConfig *restclient.Config eventRecorder events.EventRecorder informerFactory informers.SharedInformerFactory + logger klog.Logger metricsRecorder *metricsRecorder profileName string @@ -146,6 +147,7 @@ type frameworkOptions struct { captureProfile CaptureProfile clusterEventMap map[framework.ClusterEvent]sets.String parallelizer parallelize.Parallelizer + logger *klog.Logger } // Option for the frameworkImpl. @@ -227,6 +229,13 @@ func WithCaptureProfile(c CaptureProfile) Option { } } +// WithLogger overrides the default logger from k8s.io/klog. +func WithLogger(logger klog.Logger) Option { + return func(o *frameworkOptions) { + o.logger = &logger + } +} + func defaultFrameworkOptions(stopCh <-chan struct{}) frameworkOptions { return frameworkOptions{ metricsRecorder: newMetricsRecorder(1000, time.Second, stopCh), @@ -251,6 +260,11 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, stopCh <-cha opt(&options) } + logger := klog.TODO() + if options.logger != nil { + logger = *options.logger + } + f := &frameworkImpl{ registry: r, snapshotSharedLister: options.snapshotSharedLister, @@ -264,6 +278,7 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, stopCh <-cha extenders: options.extenders, PodNominator: options.podNominator, parallelizer: options.parallelizer, + logger: logger, } if profile == nil { @@ -313,7 +328,7 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, stopCh <-cha pluginsMap[name] = p // Update ClusterEventMap in place. - fillEventToPluginMap(p, options.clusterEventMap) + fillEventToPluginMap(logger, p, options.clusterEventMap) } // initialize plugins per individual extension points @@ -325,7 +340,7 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, stopCh <-cha // initialize multiPoint plugins to their expanded extension points if len(profile.Plugins.MultiPoint.Enabled) > 0 { - if err := f.expandMultiPointPlugins(profile, pluginsMap); err != nil { + if err := f.expandMultiPointPlugins(logger, profile, pluginsMap); err != nil { return nil, err } } @@ -427,7 +442,7 @@ func (os *orderedSet) delete(s string) { } } -func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerProfile, pluginsMap map[string]framework.Plugin) error { +func (f *frameworkImpl) expandMultiPointPlugins(logger klog.Logger, profile *config.KubeSchedulerProfile, pluginsMap map[string]framework.Plugin) error { // initialize MultiPoint plugins for _, e := range f.getExtensionPoints(profile.Plugins) { plugins := reflect.ValueOf(e.slicePtr).Elem() @@ -444,7 +459,7 @@ func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerPro disabledSet.Insert(disabledPlugin.Name) } if disabledSet.Has("*") { - klog.V(4).InfoS("all plugins disabled for extension point, skipping MultiPoint expansion", "extension", pluginType) + logger.V(4).Info("all plugins disabled for extension point, skipping MultiPoint expansion", "extension", pluginType) continue } @@ -465,7 +480,7 @@ func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerPro // a plugin that's enabled via MultiPoint can still be disabled for specific extension points if disabledSet.Has(ep.Name) { - klog.V(4).InfoS("plugin disabled for extension point", "plugin", ep.Name, "extension", pluginType) + logger.V(4).Info("plugin disabled for extension point", "plugin", ep.Name, "extension", pluginType) continue } @@ -475,7 +490,7 @@ func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerPro // This maintains expected behavior for overriding default plugins (see https://github.com/kubernetes/kubernetes/pull/99582) if enabledSet.has(ep.Name) { overridePlugins.insert(ep.Name) - klog.InfoS("MultiPoint plugin is explicitly re-configured; overriding", "plugin", ep.Name) + logger.Info("MultiPoint plugin is explicitly re-configured; overriding", "plugin", ep.Name) continue } @@ -514,7 +529,7 @@ func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerPro return nil } -func fillEventToPluginMap(p framework.Plugin, eventToPlugins map[framework.ClusterEvent]sets.String) { +func fillEventToPluginMap(logger klog.Logger, p framework.Plugin, eventToPlugins map[framework.ClusterEvent]sets.String) { ext, ok := p.(framework.EnqueueExtensions) if !ok { // If interface EnqueueExtensions is not implemented, register the default events @@ -528,7 +543,7 @@ func fillEventToPluginMap(p framework.Plugin, eventToPlugins map[framework.Clust // We treat it as: the plugin is not interested in any event, and hence pod failed by that plugin // cannot be moved by any regular cluster event. if len(events) == 0 { - klog.InfoS("Plugin's EventsToRegister() returned nil", "plugin", p.Name()) + logger.Info("Plugin's EventsToRegister() returned nil") return } // The most common case: a plugin implements EnqueueExtensions and returns non-nil result. @@ -643,6 +658,7 @@ func (f *frameworkImpl) RunPreFilterExtensionAddPod( podInfoToAdd *framework.PodInfo, nodeInfo *framework.NodeInfo, ) (status *framework.Status) { + logger := klog.FromContext(ctx) for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue @@ -650,7 +666,7 @@ func (f *frameworkImpl) RunPreFilterExtensionAddPod( status = f.runPreFilterExtensionAddPod(ctx, pl, state, podToSchedule, podInfoToAdd, nodeInfo) if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running AddPod on PreFilter plugin", "plugin", pl.Name(), "pod", klog.KObj(podToSchedule)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running AddPod on PreFilter plugin %q: %w", pl.Name(), err)) } } @@ -678,6 +694,7 @@ func (f *frameworkImpl) RunPreFilterExtensionRemovePod( podInfoToRemove *framework.PodInfo, nodeInfo *framework.NodeInfo, ) (status *framework.Status) { + logger := klog.FromContext(ctx) for _, pl := range f.preFilterPlugins { if pl.PreFilterExtensions() == nil { continue @@ -685,7 +702,7 @@ func (f *frameworkImpl) RunPreFilterExtensionRemovePod( status = f.runPreFilterExtensionRemovePod(ctx, pl, state, podToSchedule, podInfoToRemove, nodeInfo) if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running RemovePod on PreFilter plugin", "plugin", pl.Name(), "pod", klog.KObj(podToSchedule)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running RemovePod on PreFilter plugin %q: %w", pl.Name(), err)) } } @@ -841,7 +858,8 @@ func addNominatedPods(ctx context.Context, fh framework.Handle, pod *v1.Pod, sta // This may happen only in tests. return false, state, nodeInfo, nil } - nominatedPodInfos := fh.NominatedPodsForNode(nodeInfo.Node().Name) + logger := klog.FromContext(ctx) + nominatedPodInfos := fh.NominatedPodsForNode(logger, nodeInfo.Node().Name) if len(nominatedPodInfos) == 0 { return false, state, nodeInfo, nil } @@ -999,11 +1017,12 @@ func (f *frameworkImpl) RunPreBindPlugins(ctx context.Context, state *framework. defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(preBind, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() + logger := klog.FromContext(ctx) for _, pl := range f.preBindPlugins { status = f.runPreBindPlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running PreBind plugin", "plugin", pl.Name(), "pod", klog.KObj(pod)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running PreBind plugin %q: %w", pl.Name(), err)) } } @@ -1029,6 +1048,7 @@ func (f *frameworkImpl) RunBindPlugins(ctx context.Context, state *framework.Cyc if len(f.bindPlugins) == 0 { return framework.NewStatus(framework.Skip, "") } + logger := klog.FromContext(ctx) for _, bp := range f.bindPlugins { status = f.runBindPlugin(ctx, bp, state, pod, nodeName) if status.IsSkip() { @@ -1036,7 +1056,7 @@ func (f *frameworkImpl) RunBindPlugins(ctx context.Context, state *framework.Cyc } if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running Bind plugin", "plugin", bp.Name(), "pod", klog.KObj(pod)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running Bind plugin %q: %w", bp.Name(), err)) } return status @@ -1085,11 +1105,12 @@ func (f *frameworkImpl) RunReservePluginsReserve(ctx context.Context, state *fra defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(reserve, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() + logger := klog.FromContext(ctx) for _, pl := range f.reservePlugins { status = f.runReservePluginReserve(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := status.AsError() - klog.ErrorS(err, "Failed running Reserve plugin", "plugin", pl.Name(), "pod", klog.KObj(pod)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running Reserve plugin %q: %w", pl.Name(), err)) } } @@ -1143,11 +1164,12 @@ func (f *frameworkImpl) RunPermitPlugins(ctx context.Context, state *framework.C }() pluginsWaitTime := make(map[string]time.Duration) statusCode := framework.Success + logger := klog.FromContext(ctx) for _, pl := range f.permitPlugins { status, timeout := f.runPermitPlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { if status.IsUnschedulable() { - klog.V(4).InfoS("Pod rejected by permit plugin", "pod", klog.KObj(pod), "plugin", pl.Name(), "status", status.Message()) + logger.V(4).Info("Pod rejected by plugin", "status", status.Message()) status.SetFailedPlugin(pl.Name()) return status } @@ -1160,7 +1182,7 @@ func (f *frameworkImpl) RunPermitPlugins(ctx context.Context, state *framework.C statusCode = framework.Wait } else { err := status.AsError() - klog.ErrorS(err, "Failed running Permit plugin", "plugin", pl.Name(), "pod", klog.KObj(pod)) + logger.Error(err, "Plugin failed") return framework.AsStatus(fmt.Errorf("running Permit plugin %q: %w", pl.Name(), err)).WithFailedPlugin(pl.Name()) } } @@ -1169,7 +1191,7 @@ func (f *frameworkImpl) RunPermitPlugins(ctx context.Context, state *framework.C waitingPod := newWaitingPod(pod, pluginsWaitTime) f.waitingPods.add(waitingPod) msg := fmt.Sprintf("one or more plugins asked to wait and no plugin rejected pod %q", pod.Name) - klog.V(4).InfoS("One or more plugins asked to wait and no plugin rejected pod", "pod", klog.KObj(pod)) + logger.V(4).Info("One or more plugins asked to wait and no plugin rejected pod") return framework.NewStatus(framework.Wait, msg) } return nil @@ -1192,7 +1214,9 @@ func (f *frameworkImpl) WaitOnPermit(ctx context.Context, pod *v1.Pod) *framewor return nil } defer f.waitingPods.remove(pod.UID) - klog.V(4).InfoS("Pod waiting on permit", "pod", klog.KObj(pod)) + + logger := klog.FromContext(ctx) + logger.V(4).Info("Pod waiting on permit", "pod", klog.KObj(pod)) startTime := time.Now() s := <-waitingPod.s @@ -1200,12 +1224,12 @@ func (f *frameworkImpl) WaitOnPermit(ctx context.Context, pod *v1.Pod) *framewor if !s.IsSuccess() { if s.IsUnschedulable() { - klog.V(4).InfoS("Pod rejected while waiting on permit", "pod", klog.KObj(pod), "status", s.Message()) + logger.V(4).Info("Pod rejected while waiting on permit", "pod", klog.KObj(pod), "status", s.Message()) s.SetFailedPlugin(s.FailedPlugin()) return s } err := s.AsError() - klog.ErrorS(err, "Failed waiting on permit for pod", "pod", klog.KObj(pod)) + logger.Error(err, "Failed waiting on permit for pod", "pod", klog.KObj(pod)) return framework.AsStatus(fmt.Errorf("waiting on permit for pod: %w", err)).WithFailedPlugin(s.FailedPlugin()) } return nil @@ -1333,3 +1357,8 @@ func (f *frameworkImpl) ProfileName() string { func (f *frameworkImpl) Parallelizer() parallelize.Parallelizer { return f.parallelizer } + +// Logger returns a logger for background activities. +func (f *frameworkImpl) Logger() klog.Logger { + return f.logger +} diff --git a/pkg/scheduler/framework/runtime/framework_test.go b/pkg/scheduler/framework/runtime/framework_test.go index a26afe96af17..67e12d144e42 100644 --- a/pkg/scheduler/framework/runtime/framework_test.go +++ b/pkg/scheduler/framework/runtime/framework_test.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/component-base/metrics/testutil" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" @@ -1590,6 +1591,7 @@ func TestFilterPluginsWithNominatedPods(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) registry := Registry{} cfgPls := &config.Plugins{} @@ -1621,6 +1623,7 @@ func TestFilterPluginsWithNominatedPods(t *testing.T) { podNominator := internalqueue.NewPodNominator(nil) if tt.nominatedPod != nil { podNominator.AddNominatedPod( + logger, framework.NewPodInfo(tt.nominatedPod), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: nodeName}) } diff --git a/pkg/scheduler/framework/types.go b/pkg/scheduler/framework/types.go index 93970327d5aa..9c8e79b1963b 100644 --- a/pkg/scheduler/framework/types.go +++ b/pkg/scheduler/framework/types.go @@ -631,11 +631,11 @@ func podWithRequiredAntiAffinity(p *v1.Pod) bool { len(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 } -func removeFromSlice(s []*PodInfo, k string) []*PodInfo { +func removeFromSlice(logger klog.Logger, s []*PodInfo, k string) []*PodInfo { for i := range s { k2, err := GetPodKey(s[i].Pod) if err != nil { - klog.ErrorS(err, "Cannot get pod key", "pod", klog.KObj(s[i].Pod)) + logger.Error(err, "Cannot get pod key", "pod", klog.KObj(s[i].Pod)) continue } if k == k2 { @@ -649,22 +649,22 @@ func removeFromSlice(s []*PodInfo, k string) []*PodInfo { } // RemovePod subtracts pod information from this NodeInfo. -func (n *NodeInfo) RemovePod(pod *v1.Pod) error { +func (n *NodeInfo) RemovePod(logger klog.Logger, pod *v1.Pod) error { k, err := GetPodKey(pod) if err != nil { return err } if podWithAffinity(pod) { - n.PodsWithAffinity = removeFromSlice(n.PodsWithAffinity, k) + n.PodsWithAffinity = removeFromSlice(logger, n.PodsWithAffinity, k) } if podWithRequiredAntiAffinity(pod) { - n.PodsWithRequiredAntiAffinity = removeFromSlice(n.PodsWithRequiredAntiAffinity, k) + n.PodsWithRequiredAntiAffinity = removeFromSlice(logger, n.PodsWithRequiredAntiAffinity, k) } for i := range n.Pods { k2, err := GetPodKey(n.Pods[i].Pod) if err != nil { - klog.ErrorS(err, "Cannot get pod key", "pod", klog.KObj(n.Pods[i].Pod)) + logger.Error(err, "Cannot get pod key", "pod", klog.KObj(n.Pods[i].Pod)) continue } if k == k2 { diff --git a/pkg/scheduler/framework/types_test.go b/pkg/scheduler/framework/types_test.go index 7b1a0b420906..31764f8f906b 100644 --- a/pkg/scheduler/framework/types_test.go +++ b/pkg/scheduler/framework/types_test.go @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2/ktesting" ) func TestNewResource(t *testing.T) { @@ -1081,10 +1082,11 @@ func TestNodeInfoRemovePod(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) ni := fakeNodeInfo(pods...) gen := ni.Generation - err := ni.RemovePod(test.pod) + err := ni.RemovePod(logger, test.pod) if err != nil { if test.errExpected { expectedErrorMsg := fmt.Errorf("no corresponding pod %s in pods of node %s", test.pod.Name, ni.Node().Name) diff --git a/pkg/scheduler/internal/cache/cache.go b/pkg/scheduler/internal/cache/cache.go index 1cd69a9704db..9c1f6a531025 100644 --- a/pkg/scheduler/internal/cache/cache.go +++ b/pkg/scheduler/internal/cache/cache.go @@ -17,6 +17,7 @@ limitations under the License. package cache import ( + "context" "fmt" "os" "sync" @@ -37,10 +38,11 @@ var ( // New returns a Cache implementation. // It automatically starts a go routine that manages expiration of assumed pods. // "ttl" is how long the assumed pod will get expired. -// "stop" is the channel that would close the background goroutine. -func New(ttl time.Duration, stop <-chan struct{}) Cache { - cache := newCache(ttl, cleanAssumedPeriod, stop) - cache.run() +// "ctx" is the context that would close the background goroutine. +func New(ctx context.Context, ttl time.Duration) Cache { + logger := klog.FromContext(ctx) + cache := newCache(ctx, ttl, cleanAssumedPeriod) + cache.run(logger) return cache } @@ -97,14 +99,15 @@ func (cache *cacheImpl) createImageStateSummary(state *imageState) *framework.Im } } -func newCache(ttl, period time.Duration, stop <-chan struct{}) *cacheImpl { +func newCache(ctx context.Context, ttl, period time.Duration) *cacheImpl { + logger := klog.FromContext(ctx) return &cacheImpl{ ttl: ttl, period: period, - stop: stop, + stop: ctx.Done(), nodes: make(map[string]*nodeInfoListItem), - nodeTree: newNodeTree(nil), + nodeTree: newNodeTree(logger, nil), assumedPods: make(sets.String), podStates: make(map[string]*podState), imageStates: make(map[string]*imageState), @@ -121,10 +124,10 @@ func newNodeInfoListItem(ni *framework.NodeInfo) *nodeInfoListItem { // moveNodeInfoToHead moves a NodeInfo to the head of "cache.nodes" doubly // linked list. The head is the most recently updated NodeInfo. // We assume cache lock is already acquired. -func (cache *cacheImpl) moveNodeInfoToHead(name string) { +func (cache *cacheImpl) moveNodeInfoToHead(logger klog.Logger, name string) { ni, ok := cache.nodes[name] if !ok { - klog.ErrorS(nil, "No node info with given name found in the cache", "node", klog.KRef("", name)) + logger.Error(nil, "No node info with given name found in the cache", "node", klog.KRef("", name)) return } // if the node info list item is already at the head, we are done. @@ -149,10 +152,10 @@ func (cache *cacheImpl) moveNodeInfoToHead(name string) { // removeNodeInfoFromList removes a NodeInfo from the "cache.nodes" doubly // linked list. // We assume cache lock is already acquired. -func (cache *cacheImpl) removeNodeInfoFromList(name string) { +func (cache *cacheImpl) removeNodeInfoFromList(logger klog.Logger, name string) { ni, ok := cache.nodes[name] if !ok { - klog.ErrorS(nil, "No node info with given name found in the cache", "node", klog.KRef("", name)) + logger.Error(nil, "No node info with given name found in the cache", "node", klog.KRef("", name)) return } @@ -194,7 +197,7 @@ func (cache *cacheImpl) Dump() *Dump { // nodeInfo.Node() is guaranteed to be not nil for all the nodes in the snapshot. // This function tracks generation number of NodeInfo and updates only the // entries of an existing snapshot that have changed after the snapshot was taken. -func (cache *cacheImpl) UpdateSnapshot(nodeSnapshot *Snapshot) error { +func (cache *cacheImpl) UpdateSnapshot(logger klog.Logger, nodeSnapshot *Snapshot) error { cache.mu.Lock() defer cache.mu.Unlock() @@ -271,7 +274,7 @@ func (cache *cacheImpl) UpdateSnapshot(nodeSnapshot *Snapshot) error { } if updateAllLists || updateNodesHavePodsWithAffinity || updateNodesHavePodsWithRequiredAntiAffinity || updateUsedPVCSet { - cache.updateNodeInfoSnapshotList(nodeSnapshot, updateAllLists) + cache.updateNodeInfoSnapshotList(logger, nodeSnapshot, updateAllLists) } if len(nodeSnapshot.nodeInfoList) != cache.nodeTree.numNodes { @@ -280,17 +283,17 @@ func (cache *cacheImpl) UpdateSnapshot(nodeSnapshot *Snapshot) error { ", trying to recover", len(nodeSnapshot.nodeInfoList), cache.nodeTree.numNodes, len(nodeSnapshot.nodeInfoMap), len(cache.nodes)) - klog.ErrorS(nil, errMsg) + logger.Error(nil, errMsg) // We will try to recover by re-creating the lists for the next scheduling cycle, but still return an // error to surface the problem, the error will likely cause a failure to the current scheduling cycle. - cache.updateNodeInfoSnapshotList(nodeSnapshot, true) + cache.updateNodeInfoSnapshotList(logger, nodeSnapshot, true) return fmt.Errorf(errMsg) } return nil } -func (cache *cacheImpl) updateNodeInfoSnapshotList(snapshot *Snapshot, updateAll bool) { +func (cache *cacheImpl) updateNodeInfoSnapshotList(logger klog.Logger, snapshot *Snapshot, updateAll bool) { snapshot.havePodsWithAffinityNodeInfoList = make([]*framework.NodeInfo, 0, cache.nodeTree.numNodes) snapshot.havePodsWithRequiredAntiAffinityNodeInfoList = make([]*framework.NodeInfo, 0, cache.nodeTree.numNodes) snapshot.usedPVCSet = sets.NewString() @@ -299,7 +302,7 @@ func (cache *cacheImpl) updateNodeInfoSnapshotList(snapshot *Snapshot, updateAll snapshot.nodeInfoList = make([]*framework.NodeInfo, 0, cache.nodeTree.numNodes) nodesList, err := cache.nodeTree.list() if err != nil { - klog.ErrorS(err, "Error occurred while retrieving the list of names of the nodes from node tree") + logger.Error(err, "Error occurred while retrieving the list of names of the nodes from node tree") } for _, nodeName := range nodesList { if nodeInfo := snapshot.nodeInfoMap[nodeName]; nodeInfo != nil { @@ -314,7 +317,7 @@ func (cache *cacheImpl) updateNodeInfoSnapshotList(snapshot *Snapshot, updateAll snapshot.usedPVCSet.Insert(key) } } else { - klog.ErrorS(nil, "Node exists in nodeTree but not in NodeInfoMap, this should not happen", "node", klog.KRef("", nodeName)) + logger.Error(nil, "Node exists in nodeTree but not in NodeInfoMap, this should not happen", "node", klog.KRef("", nodeName)) } } } else { @@ -369,7 +372,7 @@ func (cache *cacheImpl) PodCount() (int, error) { return count, nil } -func (cache *cacheImpl) AssumePod(pod *v1.Pod) error { +func (cache *cacheImpl) AssumePod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -381,15 +384,15 @@ func (cache *cacheImpl) AssumePod(pod *v1.Pod) error { return fmt.Errorf("pod %v is in the cache, so can't be assumed", key) } - return cache.addPod(pod, true) + return cache.addPod(logger, pod, true) } -func (cache *cacheImpl) FinishBinding(pod *v1.Pod) error { - return cache.finishBinding(pod, time.Now()) +func (cache *cacheImpl) FinishBinding(logger klog.Logger, pod *v1.Pod) error { + return cache.finishBinding(logger, pod, time.Now()) } // finishBinding exists to make tests deterministic by injecting now as an argument -func (cache *cacheImpl) finishBinding(pod *v1.Pod, now time.Time) error { +func (cache *cacheImpl) finishBinding(logger klog.Logger, pod *v1.Pod, now time.Time) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -398,7 +401,7 @@ func (cache *cacheImpl) finishBinding(pod *v1.Pod, now time.Time) error { cache.mu.RLock() defer cache.mu.RUnlock() - klog.V(5).InfoS("Finished binding for pod, can be expired", "pod", klog.KObj(pod)) + logger.V(5).Info("Finished binding for pod, can be expired", "pod", klog.KObj(pod)) currState, ok := cache.podStates[key] if ok && cache.assumedPods.Has(key) { dl := now.Add(cache.ttl) @@ -408,7 +411,7 @@ func (cache *cacheImpl) finishBinding(pod *v1.Pod, now time.Time) error { return nil } -func (cache *cacheImpl) ForgetPod(pod *v1.Pod) error { +func (cache *cacheImpl) ForgetPod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -424,13 +427,13 @@ func (cache *cacheImpl) ForgetPod(pod *v1.Pod) error { // Only assumed pod can be forgotten. if ok && cache.assumedPods.Has(key) { - return cache.removePod(pod) + return cache.removePod(logger, pod) } return fmt.Errorf("pod %v wasn't assumed so cannot be forgotten", key) } // Assumes that lock is already acquired. -func (cache *cacheImpl) addPod(pod *v1.Pod, assumePod bool) error { +func (cache *cacheImpl) addPod(logger klog.Logger, pod *v1.Pod, assumePod bool) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -441,7 +444,7 @@ func (cache *cacheImpl) addPod(pod *v1.Pod, assumePod bool) error { cache.nodes[pod.Spec.NodeName] = n } n.info.AddPod(pod) - cache.moveNodeInfoToHead(pod.Spec.NodeName) + cache.moveNodeInfoToHead(logger, pod.Spec.NodeName) ps := &podState{ pod: pod, } @@ -453,18 +456,18 @@ func (cache *cacheImpl) addPod(pod *v1.Pod, assumePod bool) error { } // Assumes that lock is already acquired. -func (cache *cacheImpl) updatePod(oldPod, newPod *v1.Pod) error { - if err := cache.removePod(oldPod); err != nil { +func (cache *cacheImpl) updatePod(logger klog.Logger, oldPod, newPod *v1.Pod) error { + if err := cache.removePod(logger, oldPod); err != nil { return err } - return cache.addPod(newPod, false) + return cache.addPod(logger, newPod, false) } // Assumes that lock is already acquired. // Removes a pod from the cached node info. If the node information was already // removed and there are no more pods left in the node, cleans up the node from // the cache. -func (cache *cacheImpl) removePod(pod *v1.Pod) error { +func (cache *cacheImpl) removePod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -472,15 +475,15 @@ func (cache *cacheImpl) removePod(pod *v1.Pod) error { n, ok := cache.nodes[pod.Spec.NodeName] if !ok { - klog.ErrorS(nil, "Node not found when trying to remove pod", "node", klog.KRef("", pod.Spec.NodeName), "pod", klog.KObj(pod)) + logger.Error(nil, "Node not found when trying to remove pod", "node", klog.KRef("", pod.Spec.NodeName), "pod", klog.KObj(pod)) } else { - if err := n.info.RemovePod(pod); err != nil { + if err := n.info.RemovePod(logger, pod); err != nil { return err } if len(n.info.Pods) == 0 && n.info.Node() == nil { - cache.removeNodeInfoFromList(pod.Spec.NodeName) + cache.removeNodeInfoFromList(logger, pod.Spec.NodeName) } else { - cache.moveNodeInfoToHead(pod.Spec.NodeName) + cache.moveNodeInfoToHead(logger, pod.Spec.NodeName) } } @@ -489,7 +492,7 @@ func (cache *cacheImpl) removePod(pod *v1.Pod) error { return nil } -func (cache *cacheImpl) AddPod(pod *v1.Pod) error { +func (cache *cacheImpl) AddPod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -503,9 +506,9 @@ func (cache *cacheImpl) AddPod(pod *v1.Pod) error { case ok && cache.assumedPods.Has(key): if currState.pod.Spec.NodeName != pod.Spec.NodeName { // The pod was added to a different node than it was assumed to. - klog.InfoS("Pod was added to a different node than it was assumed", "pod", klog.KObj(pod), "assumedNode", klog.KRef("", pod.Spec.NodeName), "currentNode", klog.KRef("", currState.pod.Spec.NodeName)) - if err = cache.updatePod(currState.pod, pod); err != nil { - klog.ErrorS(err, "Error occurred while updating pod") + logger.Info("Pod was added to a different node than it was assumed", "pod", klog.KObj(pod), "assumedNode", klog.KRef("", pod.Spec.NodeName), "currentNode", klog.KRef("", currState.pod.Spec.NodeName)) + if err = cache.updatePod(logger, currState.pod, pod); err != nil { + logger.Error(err, "Error occurred while updating pod") } } else { delete(cache.assumedPods, key) @@ -514,8 +517,8 @@ func (cache *cacheImpl) AddPod(pod *v1.Pod) error { } case !ok: // Pod was expired. We should add it back. - if err = cache.addPod(pod, false); err != nil { - klog.ErrorS(err, "Error occurred while adding pod") + if err = cache.addPod(logger, pod, false); err != nil { + logger.Error(err, "Error occurred while adding pod") } default: return fmt.Errorf("pod %v was already in added state", key) @@ -523,7 +526,7 @@ func (cache *cacheImpl) AddPod(pod *v1.Pod) error { return nil } -func (cache *cacheImpl) UpdatePod(oldPod, newPod *v1.Pod) error { +func (cache *cacheImpl) UpdatePod(logger klog.Logger, oldPod, newPod *v1.Pod) error { key, err := framework.GetPodKey(oldPod) if err != nil { return err @@ -537,16 +540,16 @@ func (cache *cacheImpl) UpdatePod(oldPod, newPod *v1.Pod) error { // before Update event, in which case the state would change from Assumed to Added. if ok && !cache.assumedPods.Has(key) { if currState.pod.Spec.NodeName != newPod.Spec.NodeName { - klog.ErrorS(nil, "Pod updated on a different node than previously added to", "pod", klog.KObj(oldPod)) - klog.ErrorS(nil, "scheduler cache is corrupted and can badly affect scheduling decisions") + logger.Error(nil, "Pod updated on a different node than previously added to", "pod", klog.KObj(oldPod)) + logger.Error(nil, "scheduler cache is corrupted and can badly affect scheduling decisions") os.Exit(1) } - return cache.updatePod(oldPod, newPod) + return cache.updatePod(logger, oldPod, newPod) } return fmt.Errorf("pod %v is not added to scheduler cache, so cannot be updated", key) } -func (cache *cacheImpl) RemovePod(pod *v1.Pod) error { +func (cache *cacheImpl) RemovePod(logger klog.Logger, pod *v1.Pod) error { key, err := framework.GetPodKey(pod) if err != nil { return err @@ -560,15 +563,15 @@ func (cache *cacheImpl) RemovePod(pod *v1.Pod) error { return fmt.Errorf("pod %v is not found in scheduler cache, so cannot be removed from it", key) } if currState.pod.Spec.NodeName != pod.Spec.NodeName { - klog.ErrorS(nil, "Pod was added to a different node than it was assumed", "pod", klog.KObj(pod), "assumedNode", klog.KRef("", pod.Spec.NodeName), "currentNode", klog.KRef("", currState.pod.Spec.NodeName)) + logger.Error(nil, "Pod was added to a different node than it was assumed", "pod", klog.KObj(pod), "assumedNode", klog.KRef("", pod.Spec.NodeName), "currentNode", klog.KRef("", currState.pod.Spec.NodeName)) if pod.Spec.NodeName != "" { // An empty NodeName is possible when the scheduler misses a Delete // event and it gets the last known state from the informer cache. - klog.ErrorS(nil, "scheduler cache is corrupted and can badly affect scheduling decisions") + logger.Error(nil, "scheduler cache is corrupted and can badly affect scheduling decisions") os.Exit(1) } } - return cache.removePod(currState.pod) + return cache.removePod(logger, currState.pod) } func (cache *cacheImpl) IsAssumedPod(pod *v1.Pod) (bool, error) { @@ -602,7 +605,7 @@ func (cache *cacheImpl) GetPod(pod *v1.Pod) (*v1.Pod, error) { return podState.pod, nil } -func (cache *cacheImpl) AddNode(node *v1.Node) *framework.NodeInfo { +func (cache *cacheImpl) AddNode(logger klog.Logger, node *v1.Node) *framework.NodeInfo { cache.mu.Lock() defer cache.mu.Unlock() @@ -613,15 +616,15 @@ func (cache *cacheImpl) AddNode(node *v1.Node) *framework.NodeInfo { } else { cache.removeNodeImageStates(n.info.Node()) } - cache.moveNodeInfoToHead(node.Name) + cache.moveNodeInfoToHead(logger, node.Name) - cache.nodeTree.addNode(node) + cache.nodeTree.addNode(logger, node) cache.addNodeImageStates(node, n.info) n.info.SetNode(node) return n.info.Clone() } -func (cache *cacheImpl) UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInfo { +func (cache *cacheImpl) UpdateNode(logger klog.Logger, oldNode, newNode *v1.Node) *framework.NodeInfo { cache.mu.Lock() defer cache.mu.Unlock() @@ -629,13 +632,13 @@ func (cache *cacheImpl) UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInf if !ok { n = newNodeInfoListItem(framework.NewNodeInfo()) cache.nodes[newNode.Name] = n - cache.nodeTree.addNode(newNode) + cache.nodeTree.addNode(logger, newNode) } else { cache.removeNodeImageStates(n.info.Node()) } - cache.moveNodeInfoToHead(newNode.Name) + cache.moveNodeInfoToHead(logger, newNode.Name) - cache.nodeTree.updateNode(oldNode, newNode) + cache.nodeTree.updateNode(logger, oldNode, newNode) cache.addNodeImageStates(newNode, n.info) n.info.SetNode(newNode) return n.info.Clone() @@ -647,7 +650,7 @@ func (cache *cacheImpl) UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInf // the source of truth. // However, we keep a ghost node with the list of pods until all pod deletion // events have arrived. A ghost node is skipped from snapshots. -func (cache *cacheImpl) RemoveNode(node *v1.Node) error { +func (cache *cacheImpl) RemoveNode(logger klog.Logger, node *v1.Node) error { cache.mu.Lock() defer cache.mu.Unlock() @@ -661,11 +664,11 @@ func (cache *cacheImpl) RemoveNode(node *v1.Node) error { // in a different watch, and thus can potentially be observed later, even though // they happened before node removal. if len(n.info.Pods) == 0 { - cache.removeNodeInfoFromList(node.Name) + cache.removeNodeInfoFromList(logger, node.Name) } else { - cache.moveNodeInfoToHead(node.Name) + cache.moveNodeInfoToHead(logger, node.Name) } - if err := cache.nodeTree.removeNode(node); err != nil { + if err := cache.nodeTree.removeNode(logger, node); err != nil { return err } cache.removeNodeImageStates(node) @@ -723,17 +726,15 @@ func (cache *cacheImpl) removeNodeImageStates(node *v1.Node) { } } -func (cache *cacheImpl) run() { - go wait.Until(cache.cleanupExpiredAssumedPods, cache.period, cache.stop) -} - -func (cache *cacheImpl) cleanupExpiredAssumedPods() { - cache.cleanupAssumedPods(time.Now()) +func (cache *cacheImpl) run(logger klog.Logger) { + go wait.Until(func() { + cache.cleanupAssumedPods(logger, time.Now()) + }, cache.period, cache.stop) } // cleanupAssumedPods exists for making test deterministic by taking time as input argument. // It also reports metrics on the cache size for nodes, pods, and assumed pods. -func (cache *cacheImpl) cleanupAssumedPods(now time.Time) { +func (cache *cacheImpl) cleanupAssumedPods(logger klog.Logger, now time.Time) { cache.mu.Lock() defer cache.mu.Unlock() defer cache.updateMetrics() @@ -742,18 +743,18 @@ func (cache *cacheImpl) cleanupAssumedPods(now time.Time) { for key := range cache.assumedPods { ps, ok := cache.podStates[key] if !ok { - klog.ErrorS(nil, "Key found in assumed set but not in podStates, potentially a logical error") + logger.Error(nil, "Key found in assumed set but not in podStates, potentially a logical error") os.Exit(1) } if !ps.bindingFinished { - klog.V(5).InfoS("Could not expire cache for pod as binding is still in progress", + logger.V(5).Info("Could not expire cache for pod as binding is still in progress", "pod", klog.KObj(ps.pod)) continue } if now.After(*ps.deadline) { - klog.InfoS("Pod expired", "pod", klog.KObj(ps.pod)) - if err := cache.removePod(ps.pod); err != nil { - klog.ErrorS(err, "ExpirePod failed", "pod", klog.KObj(ps.pod)) + logger.Info("Pod expired", "pod", klog.KObj(ps.pod)) + if err := cache.removePod(logger, ps.pod); err != nil { + logger.Error(err, "ExpirePod failed", "pod", klog.KObj(ps.pod)) } } } diff --git a/pkg/scheduler/internal/cache/cache_test.go b/pkg/scheduler/internal/cache/cache_test.go index daaeb1ddeeed..bb31a229b92d 100644 --- a/pkg/scheduler/internal/cache/cache_test.go +++ b/pkg/scheduler/internal/cache/cache_test.go @@ -17,6 +17,7 @@ limitations under the License. package cache import ( + "context" "errors" "fmt" "reflect" @@ -30,6 +31,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/pkg/scheduler/framework" st "k8s.io/kubernetes/pkg/scheduler/testing" schedutil "k8s.io/kubernetes/pkg/scheduler/util" @@ -207,9 +210,12 @@ func TestAssumePodScheduled(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(time.Second, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, time.Second, time.Second) for _, pod := range tt.pods { - if err := cache.AssumePod(pod); err != nil { + if err := cache.AssumePod(logger, pod); err != nil { t.Fatalf("AssumePod failed: %v", err) } } @@ -219,7 +225,7 @@ func TestAssumePodScheduled(t *testing.T) { } for _, pod := range tt.pods { - if err := cache.ForgetPod(pod); err != nil { + if err := cache.ForgetPod(logger, pod); err != nil { t.Fatalf("ForgetPod failed: %v", err) } if err := isForgottenFromCache(pod, cache); err != nil { @@ -236,11 +242,11 @@ type testExpirePodStruct struct { assumedTime time.Time } -func assumeAndFinishBinding(cache *cacheImpl, pod *v1.Pod, assumedTime time.Time) error { - if err := cache.AssumePod(pod); err != nil { +func assumeAndFinishBinding(logger klog.Logger, cache *cacheImpl, pod *v1.Pod, assumedTime time.Time) error { + if err := cache.AssumePod(logger, pod); err != nil { return err } - return cache.finishBinding(pod, assumedTime) + return cache.finishBinding(logger, pod, assumedTime) } // TestExpirePod tests that assumed pods will be removed if expired. @@ -290,22 +296,25 @@ func TestExpirePod(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, pod := range tt.pods { - if err := cache.AssumePod(pod.pod); err != nil { + if err := cache.AssumePod(logger, pod.pod); err != nil { t.Fatal(err) } if !pod.finishBind { continue } - if err := cache.finishBinding(pod.pod, pod.assumedTime); err != nil { + if err := cache.finishBinding(logger, pod.pod, pod.assumedTime); err != nil { t.Fatal(err) } } // pods that got bound and have assumedTime + ttl < cleanupTime will get // expired and removed - cache.cleanupAssumedPods(tt.cleanupTime) + cache.cleanupAssumedPods(logger, tt.cleanupTime) n := cache.nodes[nodeName] if err := deepEqualWithoutGeneration(n, tt.wNodeInfo); err != nil { t.Error(err) @@ -350,18 +359,21 @@ func TestAddPodWillConfirm(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, podToAssume := range tt.podsToAssume { - if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, podToAssume, now); err != nil { t.Fatalf("assumePod failed: %v", err) } } for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Fatalf("AddPod failed: %v", err) } } - cache.cleanupAssumedPods(now.Add(2 * ttl)) + cache.cleanupAssumedPods(logger, now.Add(2*ttl)) // check after expiration. confirmed pods shouldn't be expired. n := cache.nodes[nodeName] if err := deepEqualWithoutGeneration(n, tt.wNodeInfo); err != nil { @@ -390,14 +402,17 @@ func TestDump(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, podToAssume := range tt.podsToAssume { - if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, podToAssume, now); err != nil { t.Errorf("assumePod failed: %v", err) } } for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Errorf("AddPod failed: %v", err) } } @@ -458,19 +473,22 @@ func TestAddPodWillReplaceAssumed(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, podToAssume := range tt.podsToAssume { - if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, podToAssume, now); err != nil { t.Fatalf("assumePod failed: %v", err) } } for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Fatalf("AddPod failed: %v", err) } } for _, podToUpdate := range tt.podsToUpdate { - if err := cache.UpdatePod(podToUpdate[0], podToUpdate[1]); err != nil { + if err := cache.UpdatePod(logger, podToUpdate[0], podToUpdate[1]); err != nil { t.Fatalf("UpdatePod failed: %v", err) } } @@ -512,17 +530,20 @@ func TestAddPodAfterExpiration(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() now := time.Now() - cache := newCache(ttl, time.Second, nil) - if err := assumeAndFinishBinding(cache, tt.pod, now); err != nil { + cache := newCache(ctx, ttl, time.Second) + if err := assumeAndFinishBinding(logger, cache, tt.pod, now); err != nil { t.Fatalf("assumePod failed: %v", err) } - cache.cleanupAssumedPods(now.Add(2 * ttl)) + cache.cleanupAssumedPods(logger, now.Add(2*ttl)) // It should be expired and removed. if err := isForgottenFromCache(tt.pod, cache); err != nil { t.Error(err) } - if err := cache.AddPod(tt.pod); err != nil { + if err := cache.AddPod(logger, tt.pod); err != nil { t.Fatalf("AddPod failed: %v", err) } // check after expiration. confirmed pods shouldn't be expired. @@ -579,9 +600,12 @@ func TestUpdatePod(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Fatalf("AddPod failed: %v", err) } } @@ -590,7 +614,7 @@ func TestUpdatePod(t *testing.T) { if j == 0 { continue } - if err := cache.UpdatePod(tt.podsToUpdate[j-1], tt.podsToUpdate[j]); err != nil { + if err := cache.UpdatePod(logger, tt.podsToUpdate[j-1], tt.podsToUpdate[j]); err != nil { t.Fatalf("UpdatePod failed: %v", err) } // check after expiration. confirmed pods shouldn't be expired. @@ -615,7 +639,7 @@ func TestUpdatePodAndGet(t *testing.T) { pod *v1.Pod podToUpdate *v1.Pod - handler func(cache Cache, pod *v1.Pod) error + handler func(logger klog.Logger, cache Cache, pod *v1.Pod) error assumePod bool }{ @@ -623,8 +647,8 @@ func TestUpdatePodAndGet(t *testing.T) { pod: testPods[0], podToUpdate: testPods[0], - handler: func(cache Cache, pod *v1.Pod) error { - return cache.AssumePod(pod) + handler: func(logger klog.Logger, cache Cache, pod *v1.Pod) error { + return cache.AssumePod(logger, pod) }, assumePod: true, }, @@ -632,8 +656,8 @@ func TestUpdatePodAndGet(t *testing.T) { pod: testPods[0], podToUpdate: testPods[1], - handler: func(cache Cache, pod *v1.Pod) error { - return cache.AddPod(pod) + handler: func(logger klog.Logger, cache Cache, pod *v1.Pod) error { + return cache.AddPod(logger, pod) }, assumePod: false, }, @@ -641,14 +665,17 @@ func TestUpdatePodAndGet(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(ttl, time.Second, nil) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, ttl, time.Second) - if err := tt.handler(cache, tt.pod); err != nil { + if err := tt.handler(logger, cache, tt.pod); err != nil { t.Fatalf("unexpected err: %v", err) } if !tt.assumePod { - if err := cache.UpdatePod(tt.pod, tt.podToUpdate); err != nil { + if err := cache.UpdatePod(logger, tt.pod, tt.podToUpdate); err != nil { t.Fatalf("UpdatePod failed: %v", err) } } @@ -711,17 +738,20 @@ func TestExpireAddUpdatePod(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() now := time.Now() - cache := newCache(ttl, time.Second, nil) + cache := newCache(ctx, ttl, time.Second) for _, podToAssume := range tt.podsToAssume { - if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, podToAssume, now); err != nil { t.Fatalf("assumePod failed: %v", err) } } - cache.cleanupAssumedPods(now.Add(2 * ttl)) + cache.cleanupAssumedPods(logger, now.Add(2*ttl)) for _, podToAdd := range tt.podsToAdd { - if err := cache.AddPod(podToAdd); err != nil { + if err := cache.AddPod(logger, podToAdd); err != nil { t.Fatalf("AddPod failed: %v", err) } } @@ -730,7 +760,7 @@ func TestExpireAddUpdatePod(t *testing.T) { if j == 0 { continue } - if err := cache.UpdatePod(tt.podsToUpdate[j-1], tt.podsToUpdate[j]); err != nil { + if err := cache.UpdatePod(logger, tt.podsToUpdate[j-1], tt.podsToUpdate[j]); err != nil { t.Fatalf("UpdatePod failed: %v", err) } // check after expiration. confirmed pods shouldn't be expired. @@ -776,8 +806,11 @@ func TestEphemeralStorageResource(t *testing.T) { } for i, tt := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - cache := newCache(time.Second, time.Second, nil) - if err := cache.AddPod(tt.pod); err != nil { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, time.Second, time.Second) + if err := cache.AddPod(logger, tt.pod); err != nil { t.Fatalf("AddPod failed: %v", err) } n := cache.nodes[nodeName] @@ -785,7 +818,7 @@ func TestEphemeralStorageResource(t *testing.T) { t.Error(err) } - if err := cache.RemovePod(tt.pod); err != nil { + if err := cache.RemovePod(logger, tt.pod); err != nil { t.Fatalf("RemovePod failed: %v", err) } if _, err := cache.GetPod(tt.pod); err == nil { @@ -828,15 +861,18 @@ func TestRemovePod(t *testing.T) { for name, tt := range tests { t.Run(name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() nodeName := pod.Spec.NodeName - cache := newCache(time.Second, time.Second, nil) + cache := newCache(ctx, time.Second, time.Second) // Add/Assume pod succeeds even before adding the nodes. if tt.assume { - if err := cache.AddPod(pod); err != nil { + if err := cache.AddPod(logger, pod); err != nil { t.Fatalf("AddPod failed: %v", err) } } else { - if err := cache.AssumePod(pod); err != nil { + if err := cache.AssumePod(logger, pod); err != nil { t.Fatalf("AssumePod failed: %v", err) } } @@ -845,10 +881,10 @@ func TestRemovePod(t *testing.T) { t.Error(err) } for _, n := range nodes { - cache.AddNode(n) + cache.AddNode(logger, n) } - if err := cache.RemovePod(pod); err != nil { + if err := cache.RemovePod(logger, pod); err != nil { t.Fatalf("RemovePod failed: %v", err) } @@ -870,10 +906,13 @@ func TestForgetPod(t *testing.T) { pods := []*v1.Pod{basePod} now := time.Now() ttl := 10 * time.Second + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() - cache := newCache(ttl, time.Second, nil) + cache := newCache(ctx, ttl, time.Second) for _, pod := range pods { - if err := assumeAndFinishBinding(cache, pod, now); err != nil { + if err := assumeAndFinishBinding(logger, cache, pod, now); err != nil { t.Fatalf("assumePod failed: %v", err) } isAssumed, err := cache.IsAssumedPod(pod) @@ -895,7 +934,7 @@ func TestForgetPod(t *testing.T) { } } for _, pod := range pods { - if err := cache.ForgetPod(pod); err != nil { + if err := cache.ForgetPod(logger, pod); err != nil { t.Fatalf("ForgetPod failed: %v", err) } if err := isForgottenFromCache(pod, cache); err != nil { @@ -1050,13 +1089,16 @@ func TestNodeOperators(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() expected := buildNodeInfo(test.node, test.pods) node := test.node - cache := newCache(time.Second, time.Second, nil) - cache.AddNode(node) + cache := newCache(ctx, time.Second, time.Second) + cache.AddNode(logger, node) for _, pod := range test.pods { - if err := cache.AddPod(pod); err != nil { + if err := cache.AddPod(logger, pod); err != nil { t.Fatal(err) } } @@ -1082,7 +1124,7 @@ func TestNodeOperators(t *testing.T) { // Step 2: dump cached nodes successfully. cachedNodes := NewEmptySnapshot() - if err := cache.UpdateSnapshot(cachedNodes); err != nil { + if err := cache.UpdateSnapshot(logger, cachedNodes); err != nil { t.Error(err) } newNode, found := cachedNodes.nodeInfoMap[node.Name] @@ -1098,7 +1140,7 @@ func TestNodeOperators(t *testing.T) { node.Status.Allocatable[v1.ResourceMemory] = mem50m expected.Allocatable.Memory = mem50m.Value() - cache.UpdateNode(nil, node) + cache.UpdateNode(logger, nil, node) got, found = cache.nodes[node.Name] if !found { t.Errorf("Failed to find node %v in schedulertypes after UpdateNode.", node.Name) @@ -1121,7 +1163,7 @@ func TestNodeOperators(t *testing.T) { } // Step 4: the node can be removed even if it still has pods. - if err := cache.RemoveNode(node); err != nil { + if err := cache.RemoveNode(logger, node); err != nil { t.Error(err) } if n, err := cache.getNodeInfo(node.Name); err != nil { @@ -1146,7 +1188,7 @@ func TestNodeOperators(t *testing.T) { // Step 5: removing pods for the removed node still succeeds. for _, p := range test.pods { - if err := cache.RemovePod(p); err != nil { + if err := cache.RemovePod(logger, p); err != nil { t.Error(err) } if _, err := cache.GetPod(p); err == nil { @@ -1158,6 +1200,8 @@ func TestNodeOperators(t *testing.T) { } func TestSchedulerCache_UpdateSnapshot(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) + // Create a few nodes to be used in tests. var nodes []*v1.Node for i := 0; i < 10; i++ { @@ -1224,73 +1268,73 @@ func TestSchedulerCache_UpdateSnapshot(t *testing.T) { addNode := func(i int) operation { return func(t *testing.T) { - cache.AddNode(nodes[i]) + cache.AddNode(logger, nodes[i]) } } removeNode := func(i int) operation { return func(t *testing.T) { - if err := cache.RemoveNode(nodes[i]); err != nil { + if err := cache.RemoveNode(logger, nodes[i]); err != nil { t.Error(err) } } } updateNode := func(i int) operation { return func(t *testing.T) { - cache.UpdateNode(nodes[i], updatedNodes[i]) + cache.UpdateNode(logger, nodes[i], updatedNodes[i]) } } addPod := func(i int) operation { return func(t *testing.T) { - if err := cache.AddPod(pods[i]); err != nil { + if err := cache.AddPod(logger, pods[i]); err != nil { t.Error(err) } } } addPodWithAffinity := func(i int) operation { return func(t *testing.T) { - if err := cache.AddPod(podsWithAffinity[i]); err != nil { + if err := cache.AddPod(logger, podsWithAffinity[i]); err != nil { t.Error(err) } } } addPodWithPVC := func(i int) operation { return func(t *testing.T) { - if err := cache.AddPod(podsWithPVC[i]); err != nil { + if err := cache.AddPod(logger, podsWithPVC[i]); err != nil { t.Error(err) } } } removePod := func(i int) operation { return func(t *testing.T) { - if err := cache.RemovePod(pods[i]); err != nil { + if err := cache.RemovePod(logger, pods[i]); err != nil { t.Error(err) } } } removePodWithAffinity := func(i int) operation { return func(t *testing.T) { - if err := cache.RemovePod(podsWithAffinity[i]); err != nil { + if err := cache.RemovePod(logger, podsWithAffinity[i]); err != nil { t.Error(err) } } } removePodWithPVC := func(i int) operation { return func(t *testing.T) { - if err := cache.RemovePod(podsWithPVC[i]); err != nil { + if err := cache.RemovePod(logger, podsWithPVC[i]); err != nil { t.Error(err) } } } updatePod := func(i int) operation { return func(t *testing.T) { - if err := cache.UpdatePod(pods[i], updatedPods[i]); err != nil { + if err := cache.UpdatePod(logger, pods[i], updatedPods[i]); err != nil { t.Error(err) } } } updateSnapshot := func() operation { return func(t *testing.T) { - cache.UpdateSnapshot(snapshot) + cache.UpdateSnapshot(logger, snapshot) if err := compareCacheWithNodeInfoSnapshot(t, cache, snapshot); err != nil { t.Error(err) } @@ -1512,7 +1556,10 @@ func TestSchedulerCache_UpdateSnapshot(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cache = newCache(time.Second, time.Second, nil) + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache = newCache(ctx, time.Second, time.Second) snapshot = NewEmptySnapshot() for _, op := range test.operations { @@ -1546,7 +1593,7 @@ func TestSchedulerCache_UpdateSnapshot(t *testing.T) { } // Always update the snapshot at the end of operations and compare it. - if err := cache.UpdateSnapshot(snapshot); err != nil { + if err := cache.UpdateSnapshot(logger, snapshot); err != nil { t.Error(err) } if err := compareCacheWithNodeInfoSnapshot(t, cache, snapshot); err != nil { @@ -1621,6 +1668,8 @@ func compareCacheWithNodeInfoSnapshot(t *testing.T, cache *cacheImpl, snapshot * } func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) + // Create a few nodes to be used in tests. var nodes []*v1.Node i := 0 @@ -1644,7 +1693,7 @@ func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { var snapshot *Snapshot addNode := func(t *testing.T, i int) { - cache.AddNode(nodes[i]) + cache.AddNode(logger, nodes[i]) _, ok := snapshot.nodeInfoMap[nodes[i].Name] if !ok { snapshot.nodeInfoMap[nodes[i].Name] = cache.nodes[nodes[i].Name].info @@ -1652,7 +1701,7 @@ func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { } updateSnapshot := func(t *testing.T) { - cache.updateNodeInfoSnapshotList(snapshot, true) + cache.updateNodeInfoSnapshotList(logger, snapshot, true) if err := compareCacheWithNodeInfoSnapshot(t, cache, snapshot); err != nil { t.Error(err) } @@ -1742,13 +1791,16 @@ func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cache = newCache(time.Second, time.Second, nil) + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache = newCache(ctx, time.Second, time.Second) snapshot = NewEmptySnapshot() test.operations(t) // Always update the snapshot at the end of operations and compare it. - cache.updateNodeInfoSnapshotList(snapshot, true) + cache.updateNodeInfoSnapshotList(logger, snapshot, true) if err := compareCacheWithNodeInfoSnapshot(t, cache, snapshot); err != nil { t.Error(err) } @@ -1764,11 +1816,12 @@ func TestSchedulerCache_updateNodeInfoSnapshotList(t *testing.T) { } func BenchmarkUpdate1kNodes30kPods(b *testing.B) { + logger, _ := ktesting.NewTestContext(b) cache := setupCacheOf1kNodes30kPods(b) b.ResetTimer() for n := 0; n < b.N; n++ { cachedNodes := NewEmptySnapshot() - cache.UpdateSnapshot(cachedNodes) + cache.UpdateSnapshot(logger, cachedNodes) } } @@ -1787,12 +1840,13 @@ func BenchmarkExpirePods(b *testing.B) { } func benchmarkExpire(b *testing.B, podNum int) { + logger, _ := ktesting.NewTestContext(b) now := time.Now() for n := 0; n < b.N; n++ { b.StopTimer() cache := setupCacheWithAssumedPods(b, podNum, now) b.StartTimer() - cache.cleanupAssumedPods(now.Add(2 * time.Second)) + cache.cleanupAssumedPods(logger, now.Add(2*time.Second)) } } @@ -1821,14 +1875,17 @@ func makeBasePod(t testingMode, nodeName, objName, cpu, mem, extended string, po } func setupCacheOf1kNodes30kPods(b *testing.B) Cache { - cache := newCache(time.Second, time.Second, nil) + logger, ctx := ktesting.NewTestContext(b) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, time.Second, time.Second) for i := 0; i < 1000; i++ { nodeName := fmt.Sprintf("node-%d", i) for j := 0; j < 30; j++ { objName := fmt.Sprintf("%s-pod-%d", nodeName, j) pod := makeBasePod(b, nodeName, objName, "0", "0", "", nil) - if err := cache.AddPod(pod); err != nil { + if err := cache.AddPod(logger, pod); err != nil { b.Fatalf("AddPod failed: %v", err) } } @@ -1837,13 +1894,16 @@ func setupCacheOf1kNodes30kPods(b *testing.B) Cache { } func setupCacheWithAssumedPods(b *testing.B, podNum int, assumedTime time.Time) *cacheImpl { - cache := newCache(time.Second, time.Second, nil) + logger, ctx := ktesting.NewTestContext(b) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cache := newCache(ctx, time.Second, time.Second) for i := 0; i < podNum; i++ { nodeName := fmt.Sprintf("node-%d", i/10) objName := fmt.Sprintf("%s-pod-%d", nodeName, i%10) pod := makeBasePod(b, nodeName, objName, "0", "0", "", nil) - err := assumeAndFinishBinding(cache, pod, assumedTime) + err := assumeAndFinishBinding(logger, cache, pod, assumedTime) if err != nil { b.Fatalf("assumePod failed: %v", err) } diff --git a/pkg/scheduler/internal/cache/debugger/comparer.go b/pkg/scheduler/internal/cache/debugger/comparer.go index f348977b4f90..5d52c9675f4a 100644 --- a/pkg/scheduler/internal/cache/debugger/comparer.go +++ b/pkg/scheduler/internal/cache/debugger/comparer.go @@ -20,7 +20,7 @@ import ( "sort" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/klog/v2" @@ -38,9 +38,9 @@ type CacheComparer struct { } // Compare compares the nodes and pods of NodeLister with Cache.Snapshot. -func (c *CacheComparer) Compare() error { - klog.V(3).InfoS("Cache comparer started") - defer klog.V(3).InfoS("Cache comparer finished") +func (c *CacheComparer) Compare(logger klog.Logger) error { + logger.V(3).Info("Cache comparer started") + defer logger.V(3).Info("Cache comparer finished") nodes, err := c.NodeLister.List(labels.Everything()) if err != nil { @@ -57,11 +57,11 @@ func (c *CacheComparer) Compare() error { pendingPods := c.PodQueue.PendingPods() if missed, redundant := c.CompareNodes(nodes, dump.Nodes); len(missed)+len(redundant) != 0 { - klog.InfoS("Cache mismatch", "missedNodes", missed, "redundantNodes", redundant) + logger.Info("Cache mismatch", "missedNodes", missed, "redundantNodes", redundant) } if missed, redundant := c.ComparePods(pods, pendingPods, dump.Nodes); len(missed)+len(redundant) != 0 { - klog.InfoS("Cache mismatch", "missedPods", missed, "redundantPods", redundant) + logger.Info("Cache mismatch", "missedPods", missed, "redundantPods", redundant) } return nil diff --git a/pkg/scheduler/internal/cache/debugger/debugger.go b/pkg/scheduler/internal/cache/debugger/debugger.go index d8839ec67e83..8a1f80c88b42 100644 --- a/pkg/scheduler/internal/cache/debugger/debugger.go +++ b/pkg/scheduler/internal/cache/debugger/debugger.go @@ -17,10 +17,12 @@ limitations under the License. package debugger import ( + "context" "os" "os/signal" corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/klog/v2" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" ) @@ -54,7 +56,9 @@ func New( // ListenForSignal starts a goroutine that will trigger the CacheDebugger's // behavior when the process receives SIGINT (Windows) or SIGUSER2 (non-Windows). -func (d *CacheDebugger) ListenForSignal(stopCh <-chan struct{}) { +func (d *CacheDebugger) ListenForSignal(ctx context.Context) { + logger := klog.FromContext(ctx) + stopCh := ctx.Done() ch := make(chan os.Signal, 1) signal.Notify(ch, compareSignal) @@ -64,8 +68,8 @@ func (d *CacheDebugger) ListenForSignal(stopCh <-chan struct{}) { case <-stopCh: return case <-ch: - d.Comparer.Compare() - d.Dumper.DumpAll() + d.Comparer.Compare(logger) + d.Dumper.DumpAll(logger) } } }() diff --git a/pkg/scheduler/internal/cache/debugger/dumper.go b/pkg/scheduler/internal/cache/debugger/dumper.go index b2424b67888c..5fb6239249d6 100644 --- a/pkg/scheduler/internal/cache/debugger/dumper.go +++ b/pkg/scheduler/internal/cache/debugger/dumper.go @@ -22,7 +22,7 @@ import ( "k8s.io/klog/v2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/scheduler/framework" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" "k8s.io/kubernetes/pkg/scheduler/internal/queue" @@ -36,34 +36,34 @@ type CacheDumper struct { } // DumpAll writes cached nodes and scheduling queue information to the scheduler logs. -func (d *CacheDumper) DumpAll() { - d.dumpNodes() - d.dumpSchedulingQueue() +func (d *CacheDumper) DumpAll(logger klog.Logger) { + d.dumpNodes(logger) + d.dumpSchedulingQueue(logger) } // dumpNodes writes NodeInfo to the scheduler logs. -func (d *CacheDumper) dumpNodes() { +func (d *CacheDumper) dumpNodes(logger klog.Logger) { dump := d.cache.Dump() nodeInfos := make([]string, 0, len(dump.Nodes)) for name, nodeInfo := range dump.Nodes { - nodeInfos = append(nodeInfos, d.printNodeInfo(name, nodeInfo)) + nodeInfos = append(nodeInfos, d.printNodeInfo(logger, name, nodeInfo)) } // Extra blank line added between node entries for readability. - klog.InfoS("Dump of cached NodeInfo", "nodes", strings.Join(nodeInfos, "\n\n")) + logger.Info("Dump of cached NodeInfo", "nodes", strings.Join(nodeInfos, "\n\n")) } // dumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs. -func (d *CacheDumper) dumpSchedulingQueue() { +func (d *CacheDumper) dumpSchedulingQueue(logger klog.Logger) { pendingPods := d.podQueue.PendingPods() var podData strings.Builder for _, p := range pendingPods { podData.WriteString(printPod(p)) } - klog.InfoS("Dump of scheduling queue", "pods", podData.String()) + logger.Info("Dump of scheduling queue", "pods", podData.String()) } // printNodeInfo writes parts of NodeInfo to a string. -func (d *CacheDumper) printNodeInfo(name string, n *framework.NodeInfo) string { +func (d *CacheDumper) printNodeInfo(logger klog.Logger, name string, n *framework.NodeInfo) string { var nodeData strings.Builder nodeData.WriteString(fmt.Sprintf("Node name: %s\nDeleted: %t\nRequested Resources: %+v\nAllocatable Resources:%+v\nScheduled Pods(number: %v):\n", name, n.Node() == nil, n.Requested, n.Allocatable, len(n.Pods))) @@ -72,7 +72,7 @@ func (d *CacheDumper) printNodeInfo(name string, n *framework.NodeInfo) string { nodeData.WriteString(printPod(p.Pod)) } // Dumping nominated pods info on the node - nominatedPodInfos := d.podQueue.NominatedPodsForNode(name) + nominatedPodInfos := d.podQueue.NominatedPodsForNode(logger, name) if len(nominatedPodInfos) != 0 { nodeData.WriteString(fmt.Sprintf("Nominated Pods(number: %v):\n", len(nominatedPodInfos))) for _, pi := range nominatedPodInfos { diff --git a/pkg/scheduler/internal/cache/fake/fake_cache.go b/pkg/scheduler/internal/cache/fake/fake_cache.go index 0bcf7e13ae69..1a39f9cef51d 100644 --- a/pkg/scheduler/internal/cache/fake/fake_cache.go +++ b/pkg/scheduler/internal/cache/fake/fake_cache.go @@ -18,6 +18,7 @@ package fake import ( v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/framework" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" ) @@ -31,28 +32,28 @@ type Cache struct { } // AssumePod is a fake method for testing. -func (c *Cache) AssumePod(pod *v1.Pod) error { +func (c *Cache) AssumePod(logger klog.Logger, pod *v1.Pod) error { c.AssumeFunc(pod) return nil } // FinishBinding is a fake method for testing. -func (c *Cache) FinishBinding(pod *v1.Pod) error { return nil } +func (c *Cache) FinishBinding(logger klog.Logger, pod *v1.Pod) error { return nil } // ForgetPod is a fake method for testing. -func (c *Cache) ForgetPod(pod *v1.Pod) error { +func (c *Cache) ForgetPod(logger klog.Logger, pod *v1.Pod) error { c.ForgetFunc(pod) return nil } // AddPod is a fake method for testing. -func (c *Cache) AddPod(pod *v1.Pod) error { return nil } +func (c *Cache) AddPod(logger klog.Logger, pod *v1.Pod) error { return nil } // UpdatePod is a fake method for testing. -func (c *Cache) UpdatePod(oldPod, newPod *v1.Pod) error { return nil } +func (c *Cache) UpdatePod(logger klog.Logger, oldPod, newPod *v1.Pod) error { return nil } // RemovePod is a fake method for testing. -func (c *Cache) RemovePod(pod *v1.Pod) error { return nil } +func (c *Cache) RemovePod(logger klog.Logger, pod *v1.Pod) error { return nil } // IsAssumedPod is a fake method for testing. func (c *Cache) IsAssumedPod(pod *v1.Pod) (bool, error) { @@ -65,16 +66,18 @@ func (c *Cache) GetPod(pod *v1.Pod) (*v1.Pod, error) { } // AddNode is a fake method for testing. -func (c *Cache) AddNode(node *v1.Node) *framework.NodeInfo { return nil } +func (c *Cache) AddNode(logger klog.Logger, node *v1.Node) *framework.NodeInfo { return nil } // UpdateNode is a fake method for testing. -func (c *Cache) UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInfo { return nil } +func (c *Cache) UpdateNode(logger klog.Logger, oldNode, newNode *v1.Node) *framework.NodeInfo { + return nil +} // RemoveNode is a fake method for testing. -func (c *Cache) RemoveNode(node *v1.Node) error { return nil } +func (c *Cache) RemoveNode(logger klog.Logger, node *v1.Node) error { return nil } // UpdateSnapshot is a fake method for testing. -func (c *Cache) UpdateSnapshot(snapshot *internalcache.Snapshot) error { +func (c *Cache) UpdateSnapshot(logger klog.Logger, snapshot *internalcache.Snapshot) error { return nil } diff --git a/pkg/scheduler/internal/cache/interface.go b/pkg/scheduler/internal/cache/interface.go index 4cd6d8f88692..2d09922bcd6a 100644 --- a/pkg/scheduler/internal/cache/interface.go +++ b/pkg/scheduler/internal/cache/interface.go @@ -19,6 +19,7 @@ package cache import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -68,23 +69,23 @@ type Cache interface { // AssumePod assumes a pod scheduled and aggregates the pod's information into its node. // The implementation also decides the policy to expire pod before being confirmed (receiving Add event). // After expiration, its information would be subtracted. - AssumePod(pod *v1.Pod) error + AssumePod(logger klog.Logger, pod *v1.Pod) error // FinishBinding signals that cache for assumed pod can be expired - FinishBinding(pod *v1.Pod) error + FinishBinding(logger klog.Logger, pod *v1.Pod) error // ForgetPod removes an assumed pod from cache. - ForgetPod(pod *v1.Pod) error + ForgetPod(logger klog.Logger, pod *v1.Pod) error // AddPod either confirms a pod if it's assumed, or adds it back if it's expired. // If added back, the pod's information would be added again. - AddPod(pod *v1.Pod) error + AddPod(logger klog.Logger, pod *v1.Pod) error // UpdatePod removes oldPod's information and adds newPod's information. - UpdatePod(oldPod, newPod *v1.Pod) error + UpdatePod(logger klog.Logger, oldPod, newPod *v1.Pod) error // RemovePod removes a pod. The pod's information would be subtracted from assigned node. - RemovePod(pod *v1.Pod) error + RemovePod(logger klog.Logger, pod *v1.Pod) error // GetPod returns the pod from the cache with the same namespace and the // same name of the specified pod. @@ -95,21 +96,21 @@ type Cache interface { // AddNode adds overall information about node. // It returns a clone of added NodeInfo object. - AddNode(node *v1.Node) *framework.NodeInfo + AddNode(logger klog.Logger, node *v1.Node) *framework.NodeInfo // UpdateNode updates overall information about node. // It returns a clone of updated NodeInfo object. - UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInfo + UpdateNode(logger klog.Logger, oldNode, newNode *v1.Node) *framework.NodeInfo // RemoveNode removes overall information about node. - RemoveNode(node *v1.Node) error + RemoveNode(logger klog.Logger, node *v1.Node) error // UpdateSnapshot updates the passed infoSnapshot to the current contents of Cache. // The node info contains aggregated information of pods scheduled (including assumed to be) // on this node. // The snapshot only includes Nodes that are not deleted at the time this function is called. // nodeinfo.Node() is guaranteed to be not nil for all the nodes in the snapshot. - UpdateSnapshot(nodeSnapshot *Snapshot) error + UpdateSnapshot(logger klog.Logger, nodeSnapshot *Snapshot) error // Dump produces a dump of the current cache. Dump() *Dump diff --git a/pkg/scheduler/internal/cache/node_tree.go b/pkg/scheduler/internal/cache/node_tree.go index a0005ddeed50..b56f4d69a649 100644 --- a/pkg/scheduler/internal/cache/node_tree.go +++ b/pkg/scheduler/internal/cache/node_tree.go @@ -36,24 +36,24 @@ type nodeTree struct { } // newNodeTree creates a NodeTree from nodes. -func newNodeTree(nodes []*v1.Node) *nodeTree { +func newNodeTree(logger klog.Logger, nodes []*v1.Node) *nodeTree { nt := &nodeTree{ tree: make(map[string][]string), } for _, n := range nodes { - nt.addNode(n) + nt.addNode(logger, n) } return nt } // addNode adds a node and its corresponding zone to the tree. If the zone already exists, the node // is added to the array of nodes in that zone. -func (nt *nodeTree) addNode(n *v1.Node) { +func (nt *nodeTree) addNode(logger klog.Logger, n *v1.Node) { zone := utilnode.GetZoneKey(n) if na, ok := nt.tree[zone]; ok { for _, nodeName := range na { if nodeName == n.Name { - klog.InfoS("Node already exists in the NodeTree", "node", klog.KObj(n)) + logger.Info("Node already exists in the NodeTree", "node", klog.KObj(n)) return } } @@ -62,12 +62,12 @@ func (nt *nodeTree) addNode(n *v1.Node) { nt.zones = append(nt.zones, zone) nt.tree[zone] = []string{n.Name} } - klog.V(2).InfoS("Added node in listed group to NodeTree", "node", klog.KObj(n), "zone", zone) + logger.V(2).Info("Added node in listed group to NodeTree", "node", klog.KObj(n), "zone", zone) nt.numNodes++ } // removeNode removes a node from the NodeTree. -func (nt *nodeTree) removeNode(n *v1.Node) error { +func (nt *nodeTree) removeNode(logger klog.Logger, n *v1.Node) error { zone := utilnode.GetZoneKey(n) if na, ok := nt.tree[zone]; ok { for i, nodeName := range na { @@ -76,13 +76,13 @@ func (nt *nodeTree) removeNode(n *v1.Node) error { if len(nt.tree[zone]) == 0 { nt.removeZone(zone) } - klog.V(2).InfoS("Removed node in listed group from NodeTree", "node", klog.KObj(n), "zone", zone) + logger.V(2).Info("Removed node in listed group from NodeTree", "node", klog.KObj(n), "zone", zone) nt.numNodes-- return nil } } } - klog.ErrorS(nil, "Node in listed group was not found", "node", klog.KObj(n), "zone", zone) + logger.Error(nil, "Node in listed group was not found", "node", klog.KObj(n), "zone", zone) return fmt.Errorf("node %q in group %q was not found", n.Name, zone) } @@ -99,7 +99,7 @@ func (nt *nodeTree) removeZone(zone string) { } // updateNode updates a node in the NodeTree. -func (nt *nodeTree) updateNode(old, new *v1.Node) { +func (nt *nodeTree) updateNode(logger klog.Logger, old, new *v1.Node) { var oldZone string if old != nil { oldZone = utilnode.GetZoneKey(old) @@ -110,8 +110,8 @@ func (nt *nodeTree) updateNode(old, new *v1.Node) { if oldZone == newZone { return } - nt.removeNode(old) // No error checking. We ignore whether the old node exists or not. - nt.addNode(new) + nt.removeNode(logger, old) // No error checking. We ignore whether the old node exists or not. + nt.addNode(logger, new) } // list returns the list of names of the node. NodeTree iterates over zones and in each zone iterates diff --git a/pkg/scheduler/internal/cache/node_tree_test.go b/pkg/scheduler/internal/cache/node_tree_test.go index a316a0ef4cf7..64749cc70464 100644 --- a/pkg/scheduler/internal/cache/node_tree_test.go +++ b/pkg/scheduler/internal/cache/node_tree_test.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2/ktesting" ) var allNodes = []*v1.Node{ @@ -199,9 +200,10 @@ func TestNodeTree_AddNode(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(nil) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, nil) for _, n := range test.nodesToAdd { - nt.addNode(n) + nt.addNode(logger, n) } verifyNodeTree(t, nt, test.expectedTree) }) @@ -256,9 +258,10 @@ func TestNodeTree_RemoveNode(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(test.existingNodes) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, test.existingNodes) for _, n := range test.nodesToRemove { - err := nt.removeNode(n) + err := nt.removeNode(logger, n) if test.expectError == (err == nil) { t.Errorf("unexpected returned error value: %v", err) } @@ -332,7 +335,8 @@ func TestNodeTree_UpdateNode(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(test.existingNodes) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, test.existingNodes) var oldNode *v1.Node for _, n := range allNodes { if n.Name == test.nodeToUpdate.Name { @@ -343,7 +347,7 @@ func TestNodeTree_UpdateNode(t *testing.T) { if oldNode == nil { oldNode = &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "nonexisting-node"}} } - nt.updateNode(oldNode, test.nodeToUpdate) + nt.updateNode(logger, oldNode, test.nodeToUpdate) verifyNodeTree(t, nt, test.expectedTree) }) } @@ -379,7 +383,8 @@ func TestNodeTree_List(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(test.nodesToAdd) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, test.nodesToAdd) output, err := nt.list() if err != nil { @@ -393,7 +398,8 @@ func TestNodeTree_List(t *testing.T) { } func TestNodeTree_List_Exhausted(t *testing.T) { - nt := newNodeTree(allNodes[:9]) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, allNodes[:9]) nt.numNodes++ _, err := nt.list() if err == nil { @@ -448,7 +454,8 @@ func TestNodeTreeMultiOperations(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - nt := newNodeTree(nil) + logger, _ := ktesting.NewTestContext(t) + nt := newNodeTree(logger, nil) addIndex := 0 removeIndex := 0 for _, op := range test.operations { @@ -457,14 +464,14 @@ func TestNodeTreeMultiOperations(t *testing.T) { if addIndex >= len(test.nodesToAdd) { t.Error("more add operations than nodesToAdd") } else { - nt.addNode(test.nodesToAdd[addIndex]) + nt.addNode(logger, test.nodesToAdd[addIndex]) addIndex++ } case "remove": if removeIndex >= len(test.nodesToRemove) { t.Error("more remove operations than nodesToRemove") } else { - nt.removeNode(test.nodesToRemove[removeIndex]) + nt.removeNode(logger, test.nodesToRemove[removeIndex]) removeIndex++ } default: diff --git a/pkg/scheduler/internal/queue/scheduling_queue.go b/pkg/scheduler/internal/queue/scheduling_queue.go index a5a4b30ecadd..edbbb5e420c1 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue.go +++ b/pkg/scheduler/internal/queue/scheduling_queue.go @@ -81,15 +81,15 @@ type PreEnqueueCheck func(pod *v1.Pod) bool // makes it easy to use those data structures as a SchedulingQueue. type SchedulingQueue interface { framework.PodNominator - Add(pod *v1.Pod) error + Add(logger klog.Logger, pod *v1.Pod) error // Activate moves the given pods to activeQ iff they're in unschedulablePods or backoffQ. // The passed-in pods are originally compiled from plugins that want to activate Pods, // by injecting the pods through a reserved CycleState struct (PodsToActivate). - Activate(pods map[string]*v1.Pod) + Activate(logger klog.Logger, pods map[string]*v1.Pod) // AddUnschedulableIfNotPresent adds an unschedulable pod back to scheduling queue. // The podSchedulingCycle represents the current scheduling cycle number which can be // returned by calling SchedulingCycle(). - AddUnschedulableIfNotPresent(pod *framework.QueuedPodInfo, podSchedulingCycle int64) error + AddUnschedulableIfNotPresent(logger klog.Logger, pod *framework.QueuedPodInfo, podSchedulingCycle int64) error // SchedulingCycle returns the current number of scheduling cycle which is // cached by scheduling queue. Normally, incrementing this number whenever // a pod is popped (e.g. called Pop()) is enough. @@ -97,17 +97,17 @@ type SchedulingQueue interface { // Pop removes the head of the queue and returns it. It blocks if the // queue is empty and waits until a new item is added to the queue. Pop() (*framework.QueuedPodInfo, error) - Update(oldPod, newPod *v1.Pod) error - Delete(pod *v1.Pod) error - MoveAllToActiveOrBackoffQueue(event framework.ClusterEvent, preCheck PreEnqueueCheck) - AssignedPodAdded(pod *v1.Pod) - AssignedPodUpdated(pod *v1.Pod) + Update(logger klog.Logger, oldPod, newPod *v1.Pod) error + Delete(logger klog.Logger, pod *v1.Pod) error + MoveAllToActiveOrBackoffQueue(logger klog.Logger, event framework.ClusterEvent, preCheck PreEnqueueCheck) + AssignedPodAdded(logger klog.Logger, pod *v1.Pod) + AssignedPodUpdated(logger klog.Logger, pod *v1.Pod) PendingPods() []*v1.Pod // Close closes the SchedulingQueue so that the goroutine which is // waiting to pop items can exit gracefully. Close() // Run starts the goroutines managing the queue. - Run() + Run(logger klog.Logger) } // NewSchedulingQueue initializes a priority queue as a new scheduling queue. @@ -290,44 +290,48 @@ func NewPriorityQueue( } // Run starts the goroutine to pump from podBackoffQ to activeQ -func (p *PriorityQueue) Run() { - go wait.Until(p.flushBackoffQCompleted, 1.0*time.Second, p.stop) - go wait.Until(p.flushUnschedulablePodsLeftover, 30*time.Second, p.stop) +func (p *PriorityQueue) Run(logger klog.Logger) { + go wait.Until(func() { + p.flushBackoffQCompleted(logger) + }, 1.0*time.Second, p.stop) + go wait.Until(func() { + p.flushUnschedulablePodsLeftover(logger) + }, 30*time.Second, p.stop) } // Add adds a pod to the active queue. It should be called only when a new pod // is added so there is no chance the pod is already in active/unschedulable/backoff queues -func (p *PriorityQueue) Add(pod *v1.Pod) error { +func (p *PriorityQueue) Add(logger klog.Logger, pod *v1.Pod) error { p.lock.Lock() defer p.lock.Unlock() pInfo := p.newQueuedPodInfo(pod) if err := p.activeQ.Add(pInfo); err != nil { - klog.ErrorS(err, "Error adding pod to the active queue", "pod", klog.KObj(pod)) + logger.Error(err, "Error adding pod to the active queue", "pod", klog.KObj(pod)) return err } if p.unschedulablePods.get(pod) != nil { - klog.ErrorS(nil, "Error: pod is already in the unschedulable queue", "pod", klog.KObj(pod)) + logger.Error(nil, "Error: pod is already in the unschedulable queue", "pod", klog.KObj(pod)) p.unschedulablePods.delete(pod) } // Delete pod from backoffQ if it is backing off if err := p.podBackoffQ.Delete(pInfo); err == nil { - klog.ErrorS(nil, "Error: pod is already in the podBackoff queue", "pod", klog.KObj(pod)) + logger.Error(nil, "Error: pod is already in the podBackoff queue", "pod", klog.KObj(pod)) } metrics.SchedulerQueueIncomingPods.WithLabelValues("active", PodAdd).Inc() - p.PodNominator.AddNominatedPod(pInfo.PodInfo, nil) + p.PodNominator.AddNominatedPod(logger, pInfo.PodInfo, nil) p.cond.Broadcast() return nil } // Activate moves the given pods to activeQ iff they're in unschedulablePods or backoffQ. -func (p *PriorityQueue) Activate(pods map[string]*v1.Pod) { +func (p *PriorityQueue) Activate(logger klog.Logger, pods map[string]*v1.Pod) { p.lock.Lock() defer p.lock.Unlock() activated := false for _, pod := range pods { - if p.activate(pod) { + if p.activate(logger, pod) { activated = true } } @@ -337,7 +341,7 @@ func (p *PriorityQueue) Activate(pods map[string]*v1.Pod) { } } -func (p *PriorityQueue) activate(pod *v1.Pod) bool { +func (p *PriorityQueue) activate(logger klog.Logger, pod *v1.Pod) bool { // Verify if the pod is present in activeQ. if _, exists, _ := p.activeQ.Get(newQueuedPodInfoForLookup(pod)); exists { // No need to activate if it's already present in activeQ. @@ -348,7 +352,7 @@ func (p *PriorityQueue) activate(pod *v1.Pod) bool { if pInfo = p.unschedulablePods.get(pod); pInfo == nil { // If the pod doesn't belong to unschedulablePods or backoffQ, don't activate it. if obj, exists, _ := p.podBackoffQ.Get(newQueuedPodInfoForLookup(pod)); !exists { - klog.ErrorS(nil, "To-activate pod does not exist in unschedulablePods or backoffQ", "pod", klog.KObj(pod)) + logger.Error(nil, "To-activate pod does not exist in unschedulablePods or backoffQ", "pod", klog.KObj(pod)) return false } else { pInfo = obj.(*framework.QueuedPodInfo) @@ -357,18 +361,18 @@ func (p *PriorityQueue) activate(pod *v1.Pod) bool { if pInfo == nil { // Redundant safe check. We shouldn't reach here. - klog.ErrorS(nil, "Internal error: cannot obtain pInfo") + logger.Error(nil, "Internal error: cannot obtain pInfo") return false } if err := p.activeQ.Add(pInfo); err != nil { - klog.ErrorS(err, "Error adding pod to the scheduling queue", "pod", klog.KObj(pod)) + logger.Error(err, "Error adding pod to the scheduling queue", "pod", klog.KObj(pod)) return false } p.unschedulablePods.delete(pod) p.podBackoffQ.Delete(pInfo) metrics.SchedulerQueueIncomingPods.WithLabelValues("active", ForceActivate).Inc() - p.PodNominator.AddNominatedPod(pInfo.PodInfo, nil) + p.PodNominator.AddNominatedPod(logger, pInfo.PodInfo, nil) return true } @@ -390,7 +394,7 @@ func (p *PriorityQueue) SchedulingCycle() int64 { // the queue, unless it is already in the queue. Normally, PriorityQueue puts // unschedulable pods in `unschedulablePods`. But if there has been a recent move // request, then the pod is put in `podBackoffQ`. -func (p *PriorityQueue) AddUnschedulableIfNotPresent(pInfo *framework.QueuedPodInfo, podSchedulingCycle int64) error { +func (p *PriorityQueue) AddUnschedulableIfNotPresent(logger klog.Logger, pInfo *framework.QueuedPodInfo, podSchedulingCycle int64) error { p.lock.Lock() defer p.lock.Unlock() pod := pInfo.Pod @@ -424,12 +428,12 @@ func (p *PriorityQueue) AddUnschedulableIfNotPresent(pInfo *framework.QueuedPodI } - p.PodNominator.AddNominatedPod(pInfo.PodInfo, nil) + p.PodNominator.AddNominatedPod(logger, pInfo.PodInfo, nil) return nil } // flushBackoffQCompleted Moves all pods from backoffQ which have completed backoff in to activeQ -func (p *PriorityQueue) flushBackoffQCompleted() { +func (p *PriorityQueue) flushBackoffQCompleted(logger klog.Logger) { p.lock.Lock() defer p.lock.Unlock() activated := false @@ -445,7 +449,7 @@ func (p *PriorityQueue) flushBackoffQCompleted() { } _, err := p.podBackoffQ.Pop() if err != nil { - klog.ErrorS(err, "Unable to pop pod from backoff queue despite backoff completion", "pod", klog.KObj(pod)) + logger.Error(err, "Unable to pop pod from backoff queue despite backoff completion", "pod", klog.KObj(pod)) break } p.activeQ.Add(rawPodInfo) @@ -460,7 +464,7 @@ func (p *PriorityQueue) flushBackoffQCompleted() { // flushUnschedulablePodsLeftover moves pods which stay in unschedulablePods // longer than podMaxInUnschedulablePodsDuration to backoffQ or activeQ. -func (p *PriorityQueue) flushUnschedulablePodsLeftover() { +func (p *PriorityQueue) flushUnschedulablePodsLeftover(logger klog.Logger) { p.lock.Lock() defer p.lock.Unlock() @@ -474,7 +478,7 @@ func (p *PriorityQueue) flushUnschedulablePodsLeftover() { } if len(podsToMove) > 0 { - p.movePodsToActiveOrBackoffQueue(podsToMove, UnschedulableTimeout) + p.movePodsToActiveOrBackoffQueue(logger, podsToMove, UnschedulableTimeout) } } @@ -522,7 +526,7 @@ func isPodUpdated(oldPod, newPod *v1.Pod) bool { // the item from the unschedulable queue if pod is updated in a way that it may // become schedulable and adds the updated one to the active queue. // If pod is not present in any of the queues, it is added to the active queue. -func (p *PriorityQueue) Update(oldPod, newPod *v1.Pod) error { +func (p *PriorityQueue) Update(logger klog.Logger, oldPod, newPod *v1.Pod) error { p.lock.Lock() defer p.lock.Unlock() @@ -531,14 +535,14 @@ func (p *PriorityQueue) Update(oldPod, newPod *v1.Pod) error { // If the pod is already in the active queue, just update it there. if oldPodInfo, exists, _ := p.activeQ.Get(oldPodInfo); exists { pInfo := updatePod(oldPodInfo, newPod) - p.PodNominator.UpdateNominatedPod(oldPod, pInfo.PodInfo) + p.PodNominator.UpdateNominatedPod(logger, oldPod, pInfo.PodInfo) return p.activeQ.Update(pInfo) } // If the pod is in the backoff queue, update it there. if oldPodInfo, exists, _ := p.podBackoffQ.Get(oldPodInfo); exists { pInfo := updatePod(oldPodInfo, newPod) - p.PodNominator.UpdateNominatedPod(oldPod, pInfo.PodInfo) + p.PodNominator.UpdateNominatedPod(logger, oldPod, pInfo.PodInfo) return p.podBackoffQ.Update(pInfo) } } @@ -546,7 +550,7 @@ func (p *PriorityQueue) Update(oldPod, newPod *v1.Pod) error { // If the pod is in the unschedulable queue, updating it may make it schedulable. if usPodInfo := p.unschedulablePods.get(newPod); usPodInfo != nil { pInfo := updatePod(usPodInfo, newPod) - p.PodNominator.UpdateNominatedPod(oldPod, pInfo.PodInfo) + p.PodNominator.UpdateNominatedPod(logger, oldPod, pInfo.PodInfo) if isPodUpdated(oldPod, newPod) { if p.isPodBackingoff(usPodInfo) { if err := p.podBackoffQ.Add(pInfo); err != nil { @@ -572,17 +576,17 @@ func (p *PriorityQueue) Update(oldPod, newPod *v1.Pod) error { if err := p.activeQ.Add(pInfo); err != nil { return err } - p.PodNominator.AddNominatedPod(pInfo.PodInfo, nil) + p.PodNominator.AddNominatedPod(logger, pInfo.PodInfo, nil) p.cond.Broadcast() return nil } // Delete deletes the item from either of the two queues. It assumes the pod is // only in one queue. -func (p *PriorityQueue) Delete(pod *v1.Pod) error { +func (p *PriorityQueue) Delete(logger klog.Logger, pod *v1.Pod) error { p.lock.Lock() defer p.lock.Unlock() - p.PodNominator.DeleteNominatedPodIfExists(pod) + p.PodNominator.DeleteNominatedPodIfExists(logger, pod) if err := p.activeQ.Delete(newQueuedPodInfoForLookup(pod)); err != nil { // The item was probably not found in the activeQ. p.podBackoffQ.Delete(newQueuedPodInfoForLookup(pod)) @@ -593,17 +597,17 @@ func (p *PriorityQueue) Delete(pod *v1.Pod) error { // AssignedPodAdded is called when a bound pod is added. Creation of this pod // may make pending pods with matching affinity terms schedulable. -func (p *PriorityQueue) AssignedPodAdded(pod *v1.Pod) { +func (p *PriorityQueue) AssignedPodAdded(logger klog.Logger, pod *v1.Pod) { p.lock.Lock() - p.movePodsToActiveOrBackoffQueue(p.getUnschedulablePodsWithMatchingAffinityTerm(pod), AssignedPodAdd) + p.movePodsToActiveOrBackoffQueue(logger, p.getUnschedulablePodsWithMatchingAffinityTerm(logger, pod), AssignedPodAdd) p.lock.Unlock() } // AssignedPodUpdated is called when a bound pod is updated. Change of labels // may make pending pods with matching affinity terms schedulable. -func (p *PriorityQueue) AssignedPodUpdated(pod *v1.Pod) { +func (p *PriorityQueue) AssignedPodUpdated(logger klog.Logger, pod *v1.Pod) { p.lock.Lock() - p.movePodsToActiveOrBackoffQueue(p.getUnschedulablePodsWithMatchingAffinityTerm(pod), AssignedPodUpdate) + p.movePodsToActiveOrBackoffQueue(logger, p.getUnschedulablePodsWithMatchingAffinityTerm(logger, pod), AssignedPodUpdate) p.lock.Unlock() } @@ -611,7 +615,7 @@ func (p *PriorityQueue) AssignedPodUpdated(pod *v1.Pod) { // This function adds all pods and then signals the condition variable to ensure that // if Pop() is waiting for an item, it receives the signal after all the pods are in the // queue and the head is the highest priority pod. -func (p *PriorityQueue) MoveAllToActiveOrBackoffQueue(event framework.ClusterEvent, preCheck PreEnqueueCheck) { +func (p *PriorityQueue) MoveAllToActiveOrBackoffQueue(logger klog.Logger, event framework.ClusterEvent, preCheck PreEnqueueCheck) { p.lock.Lock() defer p.lock.Unlock() unschedulablePods := make([]*framework.QueuedPodInfo, 0, len(p.unschedulablePods.podInfoMap)) @@ -620,11 +624,11 @@ func (p *PriorityQueue) MoveAllToActiveOrBackoffQueue(event framework.ClusterEve unschedulablePods = append(unschedulablePods, pInfo) } } - p.movePodsToActiveOrBackoffQueue(unschedulablePods, event) + p.movePodsToActiveOrBackoffQueue(logger, unschedulablePods, event) } // NOTE: this function assumes lock has been acquired in caller -func (p *PriorityQueue) movePodsToActiveOrBackoffQueue(podInfoList []*framework.QueuedPodInfo, event framework.ClusterEvent) { +func (p *PriorityQueue) movePodsToActiveOrBackoffQueue(logger klog.Logger, podInfoList []*framework.QueuedPodInfo, event framework.ClusterEvent) { activated := false for _, pInfo := range podInfoList { // If the event doesn't help making the Pod schedulable, continue. @@ -637,14 +641,14 @@ func (p *PriorityQueue) movePodsToActiveOrBackoffQueue(podInfoList []*framework. pod := pInfo.Pod if p.isPodBackingoff(pInfo) { if err := p.podBackoffQ.Add(pInfo); err != nil { - klog.ErrorS(err, "Error adding pod to the backoff queue", "pod", klog.KObj(pod)) + logger.Error(err, "Error adding pod to the backoff queue", "pod", klog.KObj(pod)) } else { metrics.SchedulerQueueIncomingPods.WithLabelValues("backoff", event.Label).Inc() p.unschedulablePods.delete(pod) } } else { if err := p.activeQ.Add(pInfo); err != nil { - klog.ErrorS(err, "Error adding pod to the scheduling queue", "pod", klog.KObj(pod)) + logger.Error(err, "Error adding pod to the scheduling queue", "pod", klog.KObj(pod)) } else { activated = true metrics.SchedulerQueueIncomingPods.WithLabelValues("active", event.Label).Inc() @@ -661,9 +665,9 @@ func (p *PriorityQueue) movePodsToActiveOrBackoffQueue(podInfoList []*framework. // getUnschedulablePodsWithMatchingAffinityTerm returns unschedulable pods which have // any affinity term that matches "pod". // NOTE: this function assumes lock has been acquired in caller. -func (p *PriorityQueue) getUnschedulablePodsWithMatchingAffinityTerm(pod *v1.Pod) []*framework.QueuedPodInfo { +func (p *PriorityQueue) getUnschedulablePodsWithMatchingAffinityTerm(logger klog.Logger, pod *v1.Pod) []*framework.QueuedPodInfo { var nsLabels labels.Set - nsLabels = interpodaffinity.GetNamespaceLabelsSnapshot(pod.Namespace, p.nsLister) + nsLabels = interpodaffinity.GetNamespaceLabelsSnapshot(logger, pod.Namespace, p.nsLister) var podsToMove []*framework.QueuedPodInfo for _, pInfo := range p.unschedulablePods.podInfoMap { @@ -706,7 +710,7 @@ func (p *PriorityQueue) Close() { } // DeleteNominatedPodIfExists deletes from nominatedPods. -func (npm *nominator) DeleteNominatedPodIfExists(pod *v1.Pod) { +func (npm *nominator) DeleteNominatedPodIfExists(logger klog.Logger, pod *v1.Pod) { npm.Lock() npm.delete(pod) npm.Unlock() @@ -716,15 +720,15 @@ func (npm *nominator) DeleteNominatedPodIfExists(pod *v1.Pod) { // This is called during the preemption process after a node is nominated to run // the pod. We update the structure before sending a request to update the pod // object to avoid races with the following scheduling cycles. -func (npm *nominator) AddNominatedPod(pi *framework.PodInfo, nominatingInfo *framework.NominatingInfo) { +func (npm *nominator) AddNominatedPod(logger klog.Logger, pi *framework.PodInfo, nominatingInfo *framework.NominatingInfo) { npm.Lock() - npm.add(pi, nominatingInfo) + npm.add(logger, pi, nominatingInfo) npm.Unlock() } // NominatedPodsForNode returns a copy of pods that are nominated to run on the given node, // but they are waiting for other pods to be removed from the node. -func (npm *nominator) NominatedPodsForNode(nodeName string) []*framework.PodInfo { +func (npm *nominator) NominatedPodsForNode(logger klog.Logger, nodeName string) []*framework.PodInfo { npm.RLock() defer npm.RUnlock() // Make a copy of the nominated Pods so the caller can mutate safely. @@ -855,7 +859,7 @@ type nominator struct { sync.RWMutex } -func (npm *nominator) add(pi *framework.PodInfo, nominatingInfo *framework.NominatingInfo) { +func (npm *nominator) add(logger klog.Logger, pi *framework.PodInfo, nominatingInfo *framework.NominatingInfo) { // Always delete the pod if it already exists, to ensure we never store more than // one instance of the pod. npm.delete(pi.Pod) @@ -874,11 +878,11 @@ func (npm *nominator) add(pi *framework.PodInfo, nominatingInfo *framework.Nomin // If the pod was removed or if it was already scheduled, don't nominate it. updatedPod, err := npm.podLister.Pods(pi.Pod.Namespace).Get(pi.Pod.Name) if err != nil { - klog.V(4).InfoS("Pod doesn't exist in podLister, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod)) + logger.V(4).Info("Pod doesn't exist in podLister, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod)) return } if updatedPod.Spec.NodeName != "" { - klog.V(4).InfoS("Pod is already scheduled to a node, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod), "node", updatedPod.Spec.NodeName) + logger.V(4).Info("Pod is already scheduled to a node, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod), "node", updatedPod.Spec.NodeName) return } } @@ -886,7 +890,7 @@ func (npm *nominator) add(pi *framework.PodInfo, nominatingInfo *framework.Nomin npm.nominatedPodToNode[pi.Pod.UID] = nodeName for _, npi := range npm.nominatedPods[nodeName] { if npi.Pod.UID == pi.Pod.UID { - klog.V(4).InfoS("Pod already exists in the nominator", "pod", klog.KObj(npi.Pod)) + logger.V(4).Info("Pod already exists in the nominator", "pod", klog.KObj(npi.Pod)) return } } @@ -911,7 +915,7 @@ func (npm *nominator) delete(p *v1.Pod) { } // UpdateNominatedPod updates the with . -func (npm *nominator) UpdateNominatedPod(oldPod *v1.Pod, newPodInfo *framework.PodInfo) { +func (npm *nominator) UpdateNominatedPod(logger klog.Logger, oldPod *v1.Pod, newPodInfo *framework.PodInfo) { npm.Lock() defer npm.Unlock() // In some cases, an Update event with no "NominatedNode" present is received right @@ -934,7 +938,7 @@ func (npm *nominator) UpdateNominatedPod(oldPod *v1.Pod, newPodInfo *framework.P // We update irrespective of the nominatedNodeName changed or not, to ensure // that pod pointer is updated. npm.delete(oldPod) - npm.add(newPodInfo, nominatingInfo) + npm.add(logger, newPodInfo, nominatingInfo) } // NewPodNominator creates a nominator as a backing of framework.PodNominator. @@ -950,17 +954,17 @@ func NewPodNominator(podLister listersv1.PodLister) framework.PodNominator { // MakeNextPodFunc returns a function to retrieve the next pod from a given // scheduling queue -func MakeNextPodFunc(queue SchedulingQueue) func() *framework.QueuedPodInfo { +func MakeNextPodFunc(logger klog.Logger, queue SchedulingQueue) func() *framework.QueuedPodInfo { return func() *framework.QueuedPodInfo { podInfo, err := queue.Pop() if err == nil { - klog.V(4).InfoS("About to try and schedule pod", "pod", klog.KObj(podInfo.Pod)) + logger.V(4).Info("About to try and schedule pod", "pod", klog.KObj(podInfo.Pod)) for plugin := range podInfo.UnschedulablePlugins { metrics.UnschedulableReason(plugin, podInfo.Pod.Spec.SchedulerName).Dec() } return podInfo } - klog.ErrorS(err, "Error while retrieving next pod from scheduling queue") + logger.Error(err, "Error while retrieving next pod from scheduling queue") return nil } } diff --git a/pkg/scheduler/internal/queue/scheduling_queue_test.go b/pkg/scheduler/internal/queue/scheduling_queue_test.go index d1d56980a8e6..8435976c64fb 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue_test.go +++ b/pkg/scheduler/internal/queue/scheduling_queue_test.go @@ -36,6 +36,8 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "k8s.io/component-base/metrics/testutil" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" @@ -88,16 +90,17 @@ func getUnschedulablePod(p *PriorityQueue, pod *v1.Pod) *v1.Pod { func TestPriorityQueue_Add(t *testing.T) { objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - if err := q.Add(medPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, medPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } - if err := q.Add(unschedulablePodInfo.Pod); err != nil { + if err := q.Add(logger, unschedulablePodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } - if err := q.Add(highPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, highPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } expectedNominatedPods := &nominator{ @@ -133,13 +136,14 @@ func newDefaultQueueSort() framework.LessFunc { func TestPriorityQueue_AddWithReversePriorityLessFunc(t *testing.T) { objs := []runtime.Object{medPriorityPodInfo.Pod, highPriorityPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - if err := q.Add(medPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, medPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } - if err := q.Add(highPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, highPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } if p, err := q.Pop(); err != nil || p.Pod != highPriorityPodInfo.Pod { @@ -152,12 +156,13 @@ func TestPriorityQueue_AddWithReversePriorityLessFunc(t *testing.T) { func TestPriorityQueue_AddUnschedulableIfNotPresent(t *testing.T) { objs := []runtime.Object{highPriNominatedPodInfo.Pod, unschedulablePodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - q.Add(highPriNominatedPodInfo.Pod) - q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(highPriNominatedPodInfo.Pod), q.SchedulingCycle()) // Must not add anything. - q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(unschedulablePodInfo.Pod), q.SchedulingCycle()) + q.Add(logger, highPriNominatedPodInfo.Pod) + q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(highPriNominatedPodInfo.Pod), q.SchedulingCycle()) // Must not add anything. + q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(unschedulablePodInfo.Pod), q.SchedulingCycle()) expectedNominatedPods := &nominator{ nominatedPodToNode: map[types.UID]string{ unschedulablePodInfo.Pod.UID: "node1", @@ -186,7 +191,8 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent(t *testing.T) { // Pods in and before current scheduling cycle will be put back to activeQueue // if we were trying to schedule them when we received move request. func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(time.Now()))) totalNum := 10 @@ -196,7 +202,7 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { p := st.MakePod().Name(fmt.Sprintf("pod%d", i)).Namespace(fmt.Sprintf("ns%d", i)).UID(fmt.Sprintf("upns%d", i)).Priority(priority).Obj() expectedPods = append(expectedPods, *p) // priority is to make pods ordered in the PriorityQueue - q.Add(p) + q.Add(logger, p) } // Pop all pods except for the first one @@ -208,7 +214,7 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { } // move all pods to active queue when we were trying to schedule them - q.MoveAllToActiveOrBackoffQueue(TestEvent, nil) + q.MoveAllToActiveOrBackoffQueue(logger, TestEvent, nil) oldCycle := q.SchedulingCycle() firstPod, _ := q.Pop() @@ -229,7 +235,7 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { }, } - if err := q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(unschedulablePod), oldCycle); err != nil { + if err := q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(unschedulablePod), oldCycle); err != nil { t.Errorf("Failed to call AddUnschedulableIfNotPresent(%v): %v", unschedulablePod.Name, err) } } @@ -245,7 +251,8 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) { func TestPriorityQueue_Pop(t *testing.T) { objs := []runtime.Object{medPriorityPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) wg := sync.WaitGroup{} @@ -259,17 +266,18 @@ func TestPriorityQueue_Pop(t *testing.T) { t.Errorf("Expected medPriorityPodInfo to be present in nomindatePods: %v", q.PodNominator.(*nominator).nominatedPods["node1"]) } }() - q.Add(medPriorityPodInfo.Pod) + q.Add(logger, medPriorityPodInfo.Pod) wg.Wait() } func TestPriorityQueue_Update(t *testing.T) { objs := []runtime.Object{highPriorityPodInfo.Pod, unschedulablePodInfo.Pod, medPriorityPodInfo.Pod} c := testingclock.NewFakeClock(time.Now()) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs, WithClock(c)) - q.Update(nil, highPriorityPodInfo.Pod) + q.Update(logger, nil, highPriorityPodInfo.Pod) if _, exists, _ := q.activeQ.Get(newQueuedPodInfoForLookup(highPriorityPodInfo.Pod)); !exists { t.Errorf("Expected %v to be added to activeQ.", highPriorityPodInfo.Pod.Name) } @@ -277,7 +285,7 @@ func TestPriorityQueue_Update(t *testing.T) { t.Errorf("Expected nomindatePods to be empty: %v", q.PodNominator) } // Update highPriorityPodInfo and add a nominatedNodeName to it. - q.Update(highPriorityPodInfo.Pod, highPriNominatedPodInfo.Pod) + q.Update(logger, highPriorityPodInfo.Pod, highPriNominatedPodInfo.Pod) if q.activeQ.Len() != 1 { t.Error("Expected only one item in activeQ.") } @@ -286,12 +294,12 @@ func TestPriorityQueue_Update(t *testing.T) { } // Updating an unschedulable pod which is not in any of the two queues, should // add the pod to activeQ. - q.Update(unschedulablePodInfo.Pod, unschedulablePodInfo.Pod) + q.Update(logger, unschedulablePodInfo.Pod, unschedulablePodInfo.Pod) if _, exists, _ := q.activeQ.Get(newQueuedPodInfoForLookup(unschedulablePodInfo.Pod)); !exists { t.Errorf("Expected %v to be added to activeQ.", unschedulablePodInfo.Pod.Name) } // Updating a pod that is already in activeQ, should not change it. - q.Update(unschedulablePodInfo.Pod, unschedulablePodInfo.Pod) + q.Update(logger, unschedulablePodInfo.Pod, unschedulablePodInfo.Pod) if len(q.unschedulablePods.podInfoMap) != 0 { t.Error("Expected unschedulablePods to be empty.") } @@ -308,7 +316,7 @@ func TestPriorityQueue_Update(t *testing.T) { if err := q.podBackoffQ.Add(podInfo); err != nil { t.Errorf("adding pod to backoff queue error: %v", err) } - q.Update(podInfo.Pod, podInfo.Pod) + q.Update(logger, podInfo.Pod, podInfo.Pod) rawPodInfo, err := q.podBackoffQ.Pop() podGotFromBackoffQ := rawPodInfo.(*framework.QueuedPodInfo).Pod if err != nil || podGotFromBackoffQ != medPriorityPodInfo.Pod { @@ -317,13 +325,13 @@ func TestPriorityQueue_Update(t *testing.T) { // updating a pod which is in unschedulable queue, and it is still backing off, // we will move it to backoff queue - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(medPriorityPodInfo.Pod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(medPriorityPodInfo.Pod), q.SchedulingCycle()) if len(q.unschedulablePods.podInfoMap) != 1 { t.Error("Expected unschedulablePods to be 1.") } updatedPod := medPriorityPodInfo.Pod.DeepCopy() updatedPod.Annotations["foo"] = "test" - q.Update(medPriorityPodInfo.Pod, updatedPod) + q.Update(logger, medPriorityPodInfo.Pod, updatedPod) rawPodInfo, err = q.podBackoffQ.Pop() podGotFromBackoffQ = rawPodInfo.(*framework.QueuedPodInfo).Pod if err != nil || podGotFromBackoffQ != updatedPod { @@ -332,7 +340,7 @@ func TestPriorityQueue_Update(t *testing.T) { // updating a pod which is in unschedulable queue, and it is not backing off, // we will move it to active queue - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(medPriorityPodInfo.Pod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(medPriorityPodInfo.Pod), q.SchedulingCycle()) if len(q.unschedulablePods.podInfoMap) != 1 { t.Error("Expected unschedulablePods to be 1.") } @@ -341,7 +349,7 @@ func TestPriorityQueue_Update(t *testing.T) { // Move clock by podInitialBackoffDuration, so that pods in the unschedulablePods would pass the backing off, // and the pods will be moved into activeQ. c.Step(q.podInitialBackoffDuration) - q.Update(medPriorityPodInfo.Pod, updatedPod) + q.Update(logger, medPriorityPodInfo.Pod, updatedPod) if p, err := q.Pop(); err != nil || p.Pod != updatedPod { t.Errorf("Expected: %v after Pop, but got: %v", updatedPod.Name, p.Pod.Name) } @@ -349,12 +357,13 @@ func TestPriorityQueue_Update(t *testing.T) { func TestPriorityQueue_Delete(t *testing.T) { objs := []runtime.Object{highPriorityPodInfo.Pod, unschedulablePodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - q.Update(highPriorityPodInfo.Pod, highPriNominatedPodInfo.Pod) - q.Add(unschedulablePodInfo.Pod) - if err := q.Delete(highPriNominatedPodInfo.Pod); err != nil { + q.Update(logger, highPriorityPodInfo.Pod, highPriNominatedPodInfo.Pod) + q.Add(logger, unschedulablePodInfo.Pod) + if err := q.Delete(logger, highPriNominatedPodInfo.Pod); err != nil { t.Errorf("delete failed: %v", err) } if _, exists, _ := q.activeQ.Get(newQueuedPodInfoForLookup(unschedulablePodInfo.Pod)); !exists { @@ -366,7 +375,7 @@ func TestPriorityQueue_Delete(t *testing.T) { if len(q.PodNominator.(*nominator).nominatedPods) != 1 { t.Errorf("Expected nomindatePods to have only 'unschedulablePodInfo': %v", q.PodNominator.(*nominator).nominatedPods) } - if err := q.Delete(unschedulablePodInfo.Pod); err != nil { + if err := q.Delete(logger, unschedulablePodInfo.Pod); err != nil { t.Errorf("delete failed: %v", err) } if len(q.PodNominator.(*nominator).nominatedPods) != 0 { @@ -411,7 +420,8 @@ func TestPriorityQueue_Activate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var objs []runtime.Object - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) @@ -429,7 +439,7 @@ func TestPriorityQueue_Activate(t *testing.T) { } // Activate specific pod according to the table - q.Activate(map[string]*v1.Pod{"test_pod": tt.qPodInfoToActivate.PodInfo.Pod}) + q.Activate(logger, map[string]*v1.Pod{"test_pod": tt.qPodInfoToActivate.PodInfo.Pod}) // Check the result after activation by the length of activeQ if wantLen := len(tt.want); q.activeQ.Len() != wantLen { @@ -500,6 +510,7 @@ func BenchmarkMoveAllToActiveOrBackoffQueue(b *testing.B) { for _, tt := range tests { for _, podsInUnschedulablePods := range []int{1000, 5000} { b.Run(fmt.Sprintf("%v-%v", tt.name, podsInUnschedulablePods), func(b *testing.B) { + logger, _ := ktesting.NewTestContext(b) for i := 0; i < b.N; i++ { b.StopTimer() c := testingclock.NewFakeClock(time.Now()) @@ -545,15 +556,15 @@ func BenchmarkMoveAllToActiveOrBackoffQueue(b *testing.B) { // Random case. podInfo = q.newQueuedPodInfo(p, plugins[j%len(plugins)]) } - q.AddUnschedulableIfNotPresent(podInfo, q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, podInfo, q.SchedulingCycle()) } b.StartTimer() if tt.moveEvent.Resource != "" { - q.MoveAllToActiveOrBackoffQueue(tt.moveEvent, nil) + q.MoveAllToActiveOrBackoffQueue(logger, tt.moveEvent, nil) } else { // Random case. - q.MoveAllToActiveOrBackoffQueue(events[i%len(events)], nil) + q.MoveAllToActiveOrBackoffQueue(logger, events[i%len(events)], nil) } } }) @@ -566,22 +577,23 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueue(t *testing.T) { m := map[framework.ClusterEvent]sets.String{ {Resource: framework.Node, ActionType: framework.Add}: sets.NewString("fooPlugin"), } - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c), WithClusterEventMap(m)) - q.Add(medPriorityPodInfo.Pod) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) + q.Add(logger, medPriorityPodInfo.Pod) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) // Construct a Pod, but don't associate its scheduler failure to any plugin hpp1 := highPriorityPodInfo.Pod.DeepCopy() hpp1.Name = "hpp1" - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(hpp1), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(hpp1), q.SchedulingCycle()) // Construct another Pod, and associate its scheduler failure to plugin "barPlugin". hpp2 := highPriorityPodInfo.Pod.DeepCopy() hpp2.Name = "hpp2" - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(hpp2, "barPlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(hpp2, "barPlugin"), q.SchedulingCycle()) // Pods is still backing off, move the pod into backoffQ. - q.MoveAllToActiveOrBackoffQueue(NodeAdd, nil) + q.MoveAllToActiveOrBackoffQueue(logger, NodeAdd, nil) if q.activeQ.Len() != 1 { t.Errorf("Expected 1 item to be in activeQ, but got: %v", q.activeQ.Len()) } @@ -596,9 +608,9 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueue(t *testing.T) { } q.schedulingCycle++ - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(hpp1), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(hpp1), q.SchedulingCycle()) for _, pod := range []*v1.Pod{unschedulablePodInfo.Pod, highPriorityPodInfo.Pod, hpp1, hpp2} { if q.unschedulablePods.get(pod) == nil { t.Errorf("Expected %v in the unschedulablePods", pod.Name) @@ -607,7 +619,7 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueue(t *testing.T) { // Move clock by podInitialBackoffDuration, so that pods in the unschedulablePods would pass the backing off, // and the pods will be moved into activeQ. c.Step(q.podInitialBackoffDuration) - q.MoveAllToActiveOrBackoffQueue(NodeAdd, nil) + q.MoveAllToActiveOrBackoffQueue(logger, NodeAdd, nil) // hpp2 won't be moved regardless of its backoff timer. if q.activeQ.Len() != 4 { t.Errorf("Expected 4 items to be in activeQ, but got: %v", q.activeQ.Len()) @@ -621,24 +633,26 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueue(t *testing.T) { // when a pod with pod affinity is in unschedulablePods and another pod with a // matching label is added, the unschedulable pod is moved to activeQ. func TestPriorityQueue_AssignedPodAdded(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + affinityPod := st.MakePod().Name("afp").Namespace("ns1").UID("upns1").Annotation("annot2", "val2").Priority(mediumPriority).NominatedNodeName("node1").PodAffinityExists("service", "region", st.PodAffinityWithRequiredReq).Obj() labelPod := st.MakePod().Name("lbp").Namespace(affinityPod.Namespace).Label("service", "securityscan").Node("node1").Obj() c := testingclock.NewFakeClock(time.Now()) m := map[framework.ClusterEvent]sets.String{AssignedPodAdd: sets.NewString("fakePlugin")} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c), WithClusterEventMap(m)) - q.Add(medPriorityPodInfo.Pod) + q.Add(logger, medPriorityPodInfo.Pod) // Add a couple of pods to the unschedulablePods. - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fakePlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(affinityPod, "fakePlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fakePlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(affinityPod, "fakePlugin"), q.SchedulingCycle()) // Move clock to make the unschedulable pods complete backoff. c.Step(DefaultPodInitialBackoffDuration + time.Second) // Simulate addition of an assigned pod. The pod has matching labels for // affinityPod. So, affinityPod should go to activeQ. - q.AssignedPodAdded(labelPod) + q.AssignedPodAdded(logger, labelPod) if getUnschedulablePod(q, affinityPod) != nil { t.Error("affinityPod is still in the unschedulablePods.") } @@ -653,25 +667,26 @@ func TestPriorityQueue_AssignedPodAdded(t *testing.T) { func TestPriorityQueue_NominatedPodsForNode(t *testing.T) { objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - q.Add(medPriorityPodInfo.Pod) - q.Add(unschedulablePodInfo.Pod) - q.Add(highPriorityPodInfo.Pod) + q.Add(logger, medPriorityPodInfo.Pod) + q.Add(logger, unschedulablePodInfo.Pod) + q.Add(logger, highPriorityPodInfo.Pod) if p, err := q.Pop(); err != nil || p.Pod != highPriorityPodInfo.Pod { t.Errorf("Expected: %v after Pop, but got: %v", highPriorityPodInfo.Pod.Name, p.Pod.Name) } expectedList := []*framework.PodInfo{medPriorityPodInfo, unschedulablePodInfo} - podInfos := q.NominatedPodsForNode("node1") + podInfos := q.NominatedPodsForNode(logger, "node1") if diff := cmp.Diff(expectedList, podInfos); diff != "" { t.Errorf("Unexpected list of nominated Pods for node: (-want, +got):\n%s", diff) } podInfos[0].Pod.Name = "not mpp" - if diff := cmp.Diff(podInfos, q.NominatedPodsForNode("node1")); diff == "" { + if diff := cmp.Diff(podInfos, q.NominatedPodsForNode(logger, "node1")); diff == "" { t.Error("Expected list of nominated Pods for node2 is different from podInfos") } - if len(q.NominatedPodsForNode("node2")) != 0 { + if len(q.NominatedPodsForNode(logger, "node2")) != 0 { t.Error("Expected list of nominated Pods for node2 to be empty.") } } @@ -703,6 +718,7 @@ func TestPriorityQueue_NominatedPodDeleted(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) cs := fake.NewSimpleClientset(tt.podInfo.Pod) informerFactory := informers.NewSharedInformerFactory(cs, 0) podLister := informerFactory.Core().V1().Pods().Lister() @@ -719,9 +735,9 @@ func TestPriorityQueue_NominatedPodDeleted(t *testing.T) { informerFactory.Core().V1().Pods().Informer().GetStore().Delete(tt.podInfo.Pod) } - q.AddNominatedPod(tt.podInfo, nil) + q.AddNominatedPod(logger, tt.podInfo, nil) - if got := len(q.NominatedPodsForNode(tt.podInfo.Pod.Status.NominatedNodeName)) == 1; got != tt.want { + if got := len(q.NominatedPodsForNode(logger, tt.podInfo.Pod.Status.NominatedNodeName)) == 1; got != tt.want { t.Errorf("Want %v, but got %v", tt.want, got) } }) @@ -737,38 +753,40 @@ func TestPriorityQueue_PendingPods(t *testing.T) { return pendingSet } - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort()) - q.Add(medPriorityPodInfo.Pod) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(unschedulablePodInfo.Pod), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPriorityPodInfo.Pod), q.SchedulingCycle()) + q.Add(logger, medPriorityPodInfo.Pod) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(unschedulablePodInfo.Pod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(highPriorityPodInfo.Pod), q.SchedulingCycle()) expectedSet := makeSet([]*v1.Pod{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod}) if !reflect.DeepEqual(expectedSet, makeSet(q.PendingPods())) { t.Error("Unexpected list of pending Pods.") } // Move all to active queue. We should still see the same set of pods. - q.MoveAllToActiveOrBackoffQueue(TestEvent, nil) + q.MoveAllToActiveOrBackoffQueue(logger, TestEvent, nil) if !reflect.DeepEqual(expectedSet, makeSet(q.PendingPods())) { t.Error("Unexpected list of pending Pods...") } } func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { - objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod, scheduledPodInfo.Pod} - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() + objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod, scheduledPodInfo.Pod} q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs) - if err := q.Add(medPriorityPodInfo.Pod); err != nil { + if err := q.Add(logger, medPriorityPodInfo.Pod); err != nil { t.Errorf("add failed: %v", err) } // Update unschedulablePodInfo on a different node than specified in the pod. - q.AddNominatedPod(framework.NewPodInfo(unschedulablePodInfo.Pod), + q.AddNominatedPod(logger, framework.NewPodInfo(unschedulablePodInfo.Pod), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node5"}) // Update nominated node name of a pod on a node that is not specified in the pod object. - q.AddNominatedPod(framework.NewPodInfo(highPriorityPodInfo.Pod), + q.AddNominatedPod(logger, framework.NewPodInfo(highPriorityPodInfo.Pod), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node2"}) expectedNominatedPods := &nominator{ nominatedPodToNode: map[types.UID]string{ @@ -794,7 +812,7 @@ func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { } // Update one of the nominated pods that doesn't have nominatedNodeName in the // pod object. It should be updated correctly. - q.AddNominatedPod(highPriorityPodInfo, &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node4"}) + q.AddNominatedPod(logger, highPriorityPodInfo, &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node4"}) expectedNominatedPods = &nominator{ nominatedPodToNode: map[types.UID]string{ medPriorityPodInfo.Pod.UID: "node1", @@ -813,7 +831,7 @@ func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { // Attempt to nominate a pod that was deleted from the informer cache. // Nothing should change. - q.AddNominatedPod(nonExistentPodInfo, &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) + q.AddNominatedPod(logger, nonExistentPodInfo, &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominator{}), cmpopts.IgnoreFields(nominator{}, "podLister", "RWMutex")); diff != "" { t.Errorf("Unexpected diff after nominating a deleted pod (-want, +got):\n%s", diff) } @@ -821,14 +839,14 @@ func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { // Nothing should change. scheduledPodCopy := scheduledPodInfo.Pod.DeepCopy() scheduledPodInfo.Pod.Spec.NodeName = "" - q.AddNominatedPod(framework.NewPodInfo(scheduledPodCopy), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) + q.AddNominatedPod(logger, framework.NewPodInfo(scheduledPodCopy), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominator{}), cmpopts.IgnoreFields(nominator{}, "podLister", "RWMutex")); diff != "" { t.Errorf("Unexpected diff after nominating a scheduled pod (-want, +got):\n%s", diff) } // Delete a nominated pod that doesn't have nominatedNodeName in the pod // object. It should be deleted. - q.DeleteNominatedPodIfExists(highPriorityPodInfo.Pod) + q.DeleteNominatedPodIfExists(logger, highPriorityPodInfo.Pod) expectedNominatedPods = &nominator{ nominatedPodToNode: map[types.UID]string{ medPriorityPodInfo.Pod.UID: "node1", @@ -1001,13 +1019,14 @@ func TestSchedulingQueue_Close(t *testing.T) { // are frequent events that move pods to the active queue. func TestRecentlyTriedPodsGoBack(t *testing.T) { c := testingclock.NewFakeClock(time.Now()) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) // Add a few pods to priority queue. for i := 0; i < 5; i++ { p := st.MakePod().Name(fmt.Sprintf("test-pod-%v", i)).Namespace("ns1").UID(fmt.Sprintf("tp00%v", i)).Priority(highPriority).Node("node1").NominatedNodeName("node1").Obj() - q.Add(p) + q.Add(logger, p) } c.Step(time.Microsecond) // Simulate a pod being popped by the scheduler, determined unschedulable, and @@ -1025,10 +1044,10 @@ func TestRecentlyTriedPodsGoBack(t *testing.T) { LastProbeTime: metav1.Now(), }) // Put in the unschedulable queue. - q.AddUnschedulableIfNotPresent(p1, q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, p1, q.SchedulingCycle()) c.Step(DefaultPodInitialBackoffDuration) // Move all unschedulable pods to the active queue. - q.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + q.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) // Simulation is over. Now let's pop all pods. The pod popped first should be // the last one we pop here. for i := 0; i < 5; i++ { @@ -1048,7 +1067,8 @@ func TestRecentlyTriedPodsGoBack(t *testing.T) { // are frequent events that move pods to the active queue. func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { c := testingclock.NewFakeClock(time.Now()) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) @@ -1066,11 +1086,11 @@ func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { }) // Put in the unschedulable queue - q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(unschedulablePod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(unschedulablePod), q.SchedulingCycle()) // Move clock to make the unschedulable pods complete backoff. c.Step(DefaultPodInitialBackoffDuration + time.Second) // Move all unschedulable pods to the active queue. - q.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + q.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) // Simulate a pod being popped by the scheduler, // At this time, unschedulable pod should be popped. @@ -1085,7 +1105,7 @@ func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { // Assume newer pod was added just after unschedulable pod // being popped and before being pushed back to the queue. newerPod := st.MakePod().Name("test-newer-pod").Namespace("ns1").UID("tp002").CreationTimestamp(metav1.Now()).Priority(highPriority).NominatedNodeName("node1").Obj() - q.Add(newerPod) + q.Add(logger, newerPod) // And then unschedulablePodInfo was determined as unschedulable AGAIN. podutil.UpdatePodCondition(&unschedulablePod.Status, &v1.PodCondition{ @@ -1096,11 +1116,11 @@ func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { }) // And then, put unschedulable pod to the unschedulable queue - q.AddUnschedulableIfNotPresent(newQueuedPodInfoForLookup(unschedulablePod), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, newQueuedPodInfoForLookup(unschedulablePod), q.SchedulingCycle()) // Move clock to make the unschedulable pods complete backoff. c.Step(DefaultPodInitialBackoffDuration + time.Second) // Move all unschedulable pods to the active queue. - q.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + q.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) // At this time, newerPod should be popped // because it is the oldest tried pod. @@ -1116,14 +1136,15 @@ func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) { // TestHighPriorityBackoff tests that a high priority pod does not block // other pods if it is unschedulable func TestHighPriorityBackoff(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort()) midPod := st.MakePod().Name("test-midpod").Namespace("ns1").UID("tp-mid").Priority(midPriority).NominatedNodeName("node1").Obj() highPod := st.MakePod().Name("test-highpod").Namespace("ns1").UID("tp-high").Priority(highPriority).NominatedNodeName("node1").Obj() - q.Add(midPod) - q.Add(highPod) + q.Add(logger, midPod) + q.Add(logger, highPod) // Simulate a pod being popped by the scheduler, determined unschedulable, and // then moved back to the active queue. p, err := q.Pop() @@ -1141,9 +1162,9 @@ func TestHighPriorityBackoff(t *testing.T) { Message: "fake scheduling failure", }) // Put in the unschedulable queue. - q.AddUnschedulableIfNotPresent(p, q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, p, q.SchedulingCycle()) // Move all unschedulable pods to the active queue. - q.MoveAllToActiveOrBackoffQueue(TestEvent, nil) + q.MoveAllToActiveOrBackoffQueue(logger, TestEvent, nil) p, err = q.Pop() if err != nil { @@ -1161,7 +1182,8 @@ func TestHighPriorityFlushUnschedulablePodsLeftover(t *testing.T) { m := map[framework.ClusterEvent]sets.String{ NodeAdd: sets.NewString("fakePlugin"), } - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c), WithClusterEventMap(m)) midPod := st.MakePod().Name("test-midpod").Namespace("ns1").UID("tp-mid").Priority(midPriority).NominatedNodeName("node1").Obj() @@ -1183,10 +1205,10 @@ func TestHighPriorityFlushUnschedulablePodsLeftover(t *testing.T) { Message: "fake scheduling failure", }) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPod, "fakePlugin"), q.SchedulingCycle()) - q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(midPod, "fakePlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(highPod, "fakePlugin"), q.SchedulingCycle()) + q.AddUnschedulableIfNotPresent(logger, q.newQueuedPodInfo(midPod, "fakePlugin"), q.SchedulingCycle()) c.Step(DefaultPodMaxInUnschedulablePodsDuration + time.Second) - q.flushUnschedulablePodsLeftover() + q.flushUnschedulablePodsLeftover(logger) if p, err := q.Pop(); err != nil || p.Pod != highPod { t.Errorf("Expected: %v after Pop, but got: %v", highPriorityPodInfo.Pod.Name, p.Pod.Name) @@ -1242,7 +1264,8 @@ func TestPriorityQueue_initPodMaxInUnschedulablePodsDuration(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() var queue *PriorityQueue if test.podMaxInUnschedulablePodsDuration > 0 { @@ -1257,7 +1280,7 @@ func TestPriorityQueue_initPodMaxInUnschedulablePodsDuration(t *testing.T) { var podInfoList []*framework.QueuedPodInfo for i, op := range test.operations { - op(queue, test.operands[i]) + op(logger, queue, test.operands[i]) } expectedLen := len(test.expected) @@ -1280,25 +1303,25 @@ func TestPriorityQueue_initPodMaxInUnschedulablePodsDuration(t *testing.T) { } } -type operation func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) +type operation func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) var ( - add = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { - queue.Add(pInfo.Pod) + add = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + queue.Add(logger, pInfo.Pod) } - addUnschedulablePodBackToUnschedulablePods = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { - queue.AddUnschedulableIfNotPresent(pInfo, 0) + addUnschedulablePodBackToUnschedulablePods = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + queue.AddUnschedulableIfNotPresent(logger, pInfo, 0) } - addUnschedulablePodBackToBackoffQ = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { - queue.AddUnschedulableIfNotPresent(pInfo, -1) + addUnschedulablePodBackToBackoffQ = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + queue.AddUnschedulableIfNotPresent(logger, pInfo, -1) } - addPodActiveQ = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + addPodActiveQ = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { queue.activeQ.Add(pInfo) } - updatePodActiveQ = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + updatePodActiveQ = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { queue.activeQ.Update(pInfo) } - addPodUnschedulablePods = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + addPodUnschedulablePods = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { // Update pod condition to unschedulable. podutil.UpdatePodCondition(&pInfo.Pod.Status, &v1.PodCondition{ Type: v1.PodScheduled, @@ -1308,22 +1331,22 @@ var ( }) queue.unschedulablePods.addOrUpdate(pInfo) } - addPodBackoffQ = func(queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { + addPodBackoffQ = func(logger klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) { queue.podBackoffQ.Add(pInfo) } - moveAllToActiveOrBackoffQ = func(queue *PriorityQueue, _ *framework.QueuedPodInfo) { - queue.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + moveAllToActiveOrBackoffQ = func(logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) { + queue.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) } - flushBackoffQ = func(queue *PriorityQueue, _ *framework.QueuedPodInfo) { + flushBackoffQ = func(logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) { queue.clock.(*testingclock.FakeClock).Step(2 * time.Second) - queue.flushBackoffQCompleted() + queue.flushBackoffQCompleted(logger) } - moveClockForward = func(queue *PriorityQueue, _ *framework.QueuedPodInfo) { + moveClockForward = func(logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) { queue.clock.(*testingclock.FakeClock).Step(2 * time.Second) } - flushUnschedulerQ = func(queue *PriorityQueue, _ *framework.QueuedPodInfo) { + flushUnschedulerQ = func(logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) { queue.clock.(*testingclock.FakeClock).Step(queue.podMaxInUnschedulablePodsDuration) - queue.flushUnschedulablePodsLeftover() + queue.flushUnschedulablePodsLeftover(logger) } ) @@ -1392,13 +1415,14 @@ func TestPodTimestamp(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(timestamp))) var podInfoList []*framework.QueuedPodInfo for i, op := range test.operations { - op(queue, test.operands[i]) + op(logger, queue, test.operands[i]) } expectedLen := len(test.expected) @@ -1556,12 +1580,13 @@ scheduler_pending_pods{queue="unschedulable"} 0 for _, test := range tests { t.Run(test.name, func(t *testing.T) { resetMetrics() - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(timestamp))) for i, op := range test.operations { for _, pInfo := range test.operands[i] { - op(queue, pInfo) + op(logger, queue, pInfo) } } @@ -1581,10 +1606,11 @@ func TestPerPodSchedulingMetrics(t *testing.T) { // Case 1: A pod is created and scheduled after 1 attempt. The queue operations are // Add -> Pop. c := testingclock.NewFakeClock(timestamp) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) - queue.Add(pod) + queue.Add(logger, pod) pInfo, err := queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) @@ -1595,16 +1621,16 @@ func TestPerPodSchedulingMetrics(t *testing.T) { // Add -> Pop -> AddUnschedulableIfNotPresent -> flushUnschedulablePodsLeftover -> Pop. c = testingclock.NewFakeClock(timestamp) queue = NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) - queue.Add(pod) + queue.Add(logger, pod) pInfo, err = queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) } - queue.AddUnschedulableIfNotPresent(pInfo, 1) + queue.AddUnschedulableIfNotPresent(logger, pInfo, 1) // Override clock to exceed the DefaultPodMaxInUnschedulablePodsDuration so that unschedulable pods // will be moved to activeQ c.SetTime(timestamp.Add(DefaultPodMaxInUnschedulablePodsDuration + 1)) - queue.flushUnschedulablePodsLeftover() + queue.flushUnschedulablePodsLeftover(logger) pInfo, err = queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) @@ -1615,19 +1641,19 @@ func TestPerPodSchedulingMetrics(t *testing.T) { // Add -> Pop -> AddUnschedulableIfNotPresent -> flushUnschedulablePodsLeftover -> Update -> Pop. c = testingclock.NewFakeClock(timestamp) queue = NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c)) - queue.Add(pod) + queue.Add(logger, pod) pInfo, err = queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) } - queue.AddUnschedulableIfNotPresent(pInfo, 1) + queue.AddUnschedulableIfNotPresent(logger, pInfo, 1) // Override clock to exceed the DefaultPodMaxInUnschedulablePodsDuration so that unschedulable pods // will be moved to activeQ c.SetTime(timestamp.Add(DefaultPodMaxInUnschedulablePodsDuration + 1)) - queue.flushUnschedulablePodsLeftover() + queue.flushUnschedulablePodsLeftover(logger) newPod := pod.DeepCopy() newPod.Generation = 1 - queue.Update(pod, newPod) + queue.Update(logger, pod, newPod) pInfo, err = queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) @@ -1707,12 +1733,13 @@ func TestIncomingPodsMetrics(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { metrics.SchedulerQueueIncomingPods.Reset() - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(timestamp))) for _, op := range test.operations { for _, pInfo := range pInfos { - op(queue, pInfo) + op(logger, queue, pInfo) } } metricName := metrics.SchedulerSubsystem + "_" + metrics.SchedulerQueueIncomingPods.Name @@ -1735,7 +1762,8 @@ func checkPerPodSchedulingMetrics(name string, t *testing.T, pInfo *framework.Qu func TestBackOffFlow(t *testing.T) { cl := testingclock.NewFakeClock(time.Now()) - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(cl)) steps := []struct { @@ -1755,7 +1783,7 @@ func TestBackOffFlow(t *testing.T) { Namespace: pod.Namespace, Name: pod.Name, } - if err := q.Add(pod); err != nil { + if err := q.Add(logger, pod); err != nil { t.Fatal(err) } @@ -1770,12 +1798,12 @@ func TestBackOffFlow(t *testing.T) { if podInfo.Attempts != i+1 { t.Errorf("got attempts %d, want %d", podInfo.Attempts, i+1) } - if err := q.AddUnschedulableIfNotPresent(podInfo, int64(i)); err != nil { + if err := q.AddUnschedulableIfNotPresent(logger, podInfo, int64(i)); err != nil { t.Fatal(err) } // An event happens. - q.MoveAllToActiveOrBackoffQueue(UnschedulableTimeout, nil) + q.MoveAllToActiveOrBackoffQueue(logger, UnschedulableTimeout, nil) if _, ok, _ := q.podBackoffQ.Get(podInfo); !ok { t.Errorf("pod %v is not in the backoff queue", podID) @@ -1790,14 +1818,14 @@ func TestBackOffFlow(t *testing.T) { // Simulate routine that continuously flushes the backoff queue. cl.Step(time.Millisecond) - q.flushBackoffQCompleted() + q.flushBackoffQCompleted(logger) // Still in backoff queue after an early flush. if _, ok, _ := q.podBackoffQ.Get(podInfo); !ok { t.Errorf("pod %v is not in the backoff queue", podID) } // Moved out of the backoff queue after timeout. cl.Step(backoff) - q.flushBackoffQCompleted() + q.flushBackoffQCompleted(logger) if _, ok, _ := q.podBackoffQ.Get(podInfo); ok { t.Errorf("pod %v is still in the backoff queue", podID) } @@ -1946,13 +1974,14 @@ func TestMoveAllToActiveOrBackoffQueue_PreEnqueueChecks(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() q := NewTestQueue(ctx, newDefaultQueueSort()) for _, podInfo := range tt.podInfos { - q.AddUnschedulableIfNotPresent(podInfo, q.schedulingCycle) + q.AddUnschedulableIfNotPresent(logger, podInfo, q.schedulingCycle) } - q.MoveAllToActiveOrBackoffQueue(TestEvent, tt.preEnqueueCheck) + q.MoveAllToActiveOrBackoffQueue(logger, TestEvent, tt.preEnqueueCheck) var got []string for q.podBackoffQ.Len() != 0 { obj, err := q.podBackoffQ.Pop() diff --git a/pkg/scheduler/schedule_one.go b/pkg/scheduler/schedule_one.go index dbb739778f6c..cdbf9c703afa 100644 --- a/pkg/scheduler/schedule_one.go +++ b/pkg/scheduler/schedule_one.go @@ -64,6 +64,7 @@ var clearNominatedNode = &framework.NominatingInfo{NominatingMode: framework.Mod // scheduleOne does the entire scheduling workflow for a single pod. It is serialized on the scheduling algorithm's host fitting. func (sched *Scheduler) scheduleOne(ctx context.Context) { + logger := klog.FromContext(ctx) podInfo := sched.NextPod() // pod could be nil when schedulerQueue is closed if podInfo == nil || podInfo.Pod == nil { @@ -74,14 +75,14 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { if err != nil { // This shouldn't happen, because we only accept for scheduling the pods // which specify a scheduler name that matches one of the profiles. - klog.ErrorS(err, "Error occurred") + logger.Error(err, "Error occurred") return } - if sched.skipPodSchedule(fwk, pod) { + if sched.skipPodSchedule(ctx, fwk, pod) { return } - klog.V(3).InfoS("Attempting to schedule pod", "pod", klog.KObj(pod)) + logger.V(3).Info("Attempting to schedule pod", "pod", klog.KObj(pod)) // Synchronously attempt to find a fit for the pod. start := time.Now() @@ -102,15 +103,15 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { var nominatingInfo *framework.NominatingInfo if fitError, ok := err.(*framework.FitError); ok { if !fwk.HasPostFilterPlugins() { - klog.V(3).InfoS("No PostFilter plugins are registered, so no preemption will be performed") + logger.V(3).Info("No PostFilter plugins are registered, so no preemption will be performed") } else { // Run PostFilter plugins to try to make the pod schedulable in a future scheduling cycle. result, status := fwk.RunPostFilterPlugins(ctx, state, pod, fitError.Diagnosis.NodeToStatusMap) if status.Code() == framework.Error { - klog.ErrorS(nil, "Status after running PostFilter plugins for pod", "pod", klog.KObj(pod), "status", status) + logger.Error(nil, "Status after running PostFilter plugins for pod", "pod", klog.KObj(pod), "status", status) } else { fitError.Diagnosis.PostFilterMsg = status.Message() - klog.V(5).InfoS("Status after running PostFilter plugins for pod", "pod", klog.KObj(pod), "status", status) + logger.V(5).Info("Status after running PostFilter plugins for pod", "pod", klog.KObj(pod), "status", status) } if result != nil { nominatingInfo = result.NominatingInfo @@ -126,7 +127,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { metrics.PodUnschedulable(fwk.ProfileName(), metrics.SinceInSeconds(start)) } else { nominatingInfo = clearNominatedNode - klog.ErrorS(err, "Error selecting node for pod", "pod", klog.KObj(pod)) + logger.Error(err, "Error selecting node for pod", "pod", klog.KObj(pod)) metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) } sched.handleSchedulingFailure(ctx, fwk, podInfo, err, v1.PodReasonUnschedulable, nominatingInfo) @@ -138,7 +139,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { assumedPodInfo := podInfo.DeepCopy() assumedPod := assumedPodInfo.Pod // assume modifies `assumedPod` by setting NodeName=scheduleResult.SuggestedHost - err = sched.assume(assumedPod, scheduleResult.SuggestedHost) + err = sched.assume(logger, assumedPod, scheduleResult.SuggestedHost) if err != nil { metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) // This is most probably result of a BUG in retrying logic. @@ -155,8 +156,8 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) // trigger un-reserve to clean up state associated with the reserved Pod fwk.RunReservePluginsUnreserve(schedulingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if forgetErr := sched.Cache.ForgetPod(assumedPod); forgetErr != nil { - klog.ErrorS(forgetErr, "Scheduler cache ForgetPod failed") + if forgetErr := sched.Cache.ForgetPod(logger, assumedPod); forgetErr != nil { + logger.Error(forgetErr, "Scheduler cache ForgetPod failed") } sched.handleSchedulingFailure(ctx, fwk, assumedPodInfo, sts.AsError(), SchedulerError, clearNominatedNode) return @@ -175,8 +176,8 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { } // One of the plugins returned status different than success or wait. fwk.RunReservePluginsUnreserve(schedulingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if forgetErr := sched.Cache.ForgetPod(assumedPod); forgetErr != nil { - klog.ErrorS(forgetErr, "Scheduler cache ForgetPod failed") + if forgetErr := sched.Cache.ForgetPod(logger, assumedPod); forgetErr != nil { + logger.Error(forgetErr, "Scheduler cache ForgetPod failed") } sched.handleSchedulingFailure(ctx, fwk, assumedPodInfo, runPermitStatus.AsError(), reason, clearNominatedNode) return @@ -184,7 +185,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { // At the end of a successful scheduling cycle, pop and move up Pods if needed. if len(podsToActivate.Map) != 0 { - sched.SchedulingQueue.Activate(podsToActivate.Map) + sched.SchedulingQueue.Activate(logger, podsToActivate.Map) // Clear the entries after activation. podsToActivate.Map = make(map[string]*v1.Pod) } @@ -208,8 +209,8 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { } // trigger un-reserve plugins to clean up state associated with the reserved Pod fwk.RunReservePluginsUnreserve(bindingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if forgetErr := sched.Cache.ForgetPod(assumedPod); forgetErr != nil { - klog.ErrorS(forgetErr, "scheduler cache ForgetPod failed") + if forgetErr := sched.Cache.ForgetPod(logger, assumedPod); forgetErr != nil { + logger.Error(forgetErr, "scheduler cache ForgetPod failed") } else { // "Forget"ing an assumed Pod in binding cycle should be treated as a PodDelete event, // as the assumed Pod had occupied a certain amount of resources in scheduler cache. @@ -217,7 +218,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { // Avoid moving the assumed Pod itself as it's always Unschedulable. // It's intentional to "defer" this operation; otherwise MoveAllToActiveOrBackoffQueue() would // update `q.moveRequest` and thus move the assumed pod to backoffQ anyways. - defer sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(internalqueue.AssignedPodDelete, func(pod *v1.Pod) bool { + defer sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, internalqueue.AssignedPodDelete, func(pod *v1.Pod) bool { return assumedPod.UID != pod.UID }) } @@ -231,13 +232,13 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) // trigger un-reserve plugins to clean up state associated with the reserved Pod fwk.RunReservePluginsUnreserve(bindingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if forgetErr := sched.Cache.ForgetPod(assumedPod); forgetErr != nil { - klog.ErrorS(forgetErr, "scheduler cache ForgetPod failed") + if forgetErr := sched.Cache.ForgetPod(logger, assumedPod); forgetErr != nil { + logger.Error(forgetErr, "scheduler cache ForgetPod failed") } else { // "Forget"ing an assumed Pod in binding cycle should be treated as a PodDelete event, // as the assumed Pod had occupied a certain amount of resources in scheduler cache. // TODO(#103853): de-duplicate the logic. - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(internalqueue.AssignedPodDelete, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, internalqueue.AssignedPodDelete, nil) } sched.handleSchedulingFailure(ctx, fwk, assumedPodInfo, preBindStatus.AsError(), SchedulerError, clearNominatedNode) return @@ -248,19 +249,19 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { metrics.PodScheduleError(fwk.ProfileName(), metrics.SinceInSeconds(start)) // trigger un-reserve plugins to clean up state associated with the reserved Pod fwk.RunReservePluginsUnreserve(bindingCycleCtx, state, assumedPod, scheduleResult.SuggestedHost) - if err := sched.Cache.ForgetPod(assumedPod); err != nil { - klog.ErrorS(err, "scheduler cache ForgetPod failed") + if err := sched.Cache.ForgetPod(logger, assumedPod); err != nil { + logger.Error(err, "scheduler cache ForgetPod failed") } else { // "Forget"ing an assumed Pod in binding cycle should be treated as a PodDelete event, // as the assumed Pod had occupied a certain amount of resources in scheduler cache. // TODO(#103853): de-duplicate the logic. - sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(internalqueue.AssignedPodDelete, nil) + sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(logger, internalqueue.AssignedPodDelete, nil) } sched.handleSchedulingFailure(ctx, fwk, assumedPodInfo, fmt.Errorf("binding rejected: %w", err), SchedulerError, clearNominatedNode) return } // Calculating nodeResourceString can be heavy. Avoid it if klog verbosity is below 2. - klog.V(2).InfoS("Successfully bound pod to node", "pod", klog.KObj(pod), "node", scheduleResult.SuggestedHost, "evaluatedNodes", scheduleResult.EvaluatedNodes, "feasibleNodes", scheduleResult.FeasibleNodes) + logger.V(2).Info("Successfully bound pod to node", "pod", klog.KObj(pod), "node", scheduleResult.SuggestedHost, "evaluatedNodes", scheduleResult.EvaluatedNodes, "feasibleNodes", scheduleResult.FeasibleNodes) metrics.PodScheduled(fwk.ProfileName(), metrics.SinceInSeconds(start)) metrics.PodSchedulingAttempts.Observe(float64(podInfo.Attempts)) metrics.PodSchedulingDuration.WithLabelValues(getAttemptsLabel(podInfo)).Observe(metrics.SinceInSeconds(podInfo.InitialAttemptTimestamp)) @@ -270,7 +271,7 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { // At the end of a successful binding cycle, move up Pods if needed. if len(podsToActivate.Map) != 0 { - sched.SchedulingQueue.Activate(podsToActivate.Map) + sched.SchedulingQueue.Activate(logger, podsToActivate.Map) // Unlike the logic in scheduling cycle, we don't bother deleting the entries // as `podsToActivate.Map` is no longer consumed. } @@ -286,11 +287,11 @@ func (sched *Scheduler) frameworkForPod(pod *v1.Pod) (framework.Framework, error } // skipPodSchedule returns true if we could skip scheduling the pod for specified cases. -func (sched *Scheduler) skipPodSchedule(fwk framework.Framework, pod *v1.Pod) bool { +func (sched *Scheduler) skipPodSchedule(ctx context.Context, fwk framework.Framework, pod *v1.Pod) bool { // Case 1: pod is being deleted. if pod.DeletionTimestamp != nil { fwk.EventRecorder().Eventf(pod, nil, v1.EventTypeWarning, "FailedScheduling", "Scheduling", "skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name) - klog.V(3).InfoS("Skip schedule deleting pod", "pod", klog.KObj(pod)) + klog.FromContext(ctx).V(3).Info("Skip schedule deleting pod", "pod", klog.KObj(pod)) return true } @@ -299,6 +300,7 @@ func (sched *Scheduler) skipPodSchedule(fwk framework.Framework, pod *v1.Pod) bo // during its previous scheduling cycle but before getting assumed. isAssumed, err := sched.Cache.IsAssumedPod(pod) if err != nil { + // TODO: pass ctx into a revised HandleError utilruntime.HandleError(fmt.Errorf("failed to check whether pod %s/%s is assumed: %v", pod.Namespace, pod.Name, err)) return false } @@ -311,8 +313,9 @@ func (sched *Scheduler) skipPodSchedule(fwk framework.Framework, pod *v1.Pod) bo func (sched *Scheduler) schedulePod(ctx context.Context, fwk framework.Framework, state *framework.CycleState, pod *v1.Pod) (result ScheduleResult, err error) { trace := utiltrace.New("Scheduling", utiltrace.Field{Key: "namespace", Value: pod.Namespace}, utiltrace.Field{Key: "name", Value: pod.Name}) defer trace.LogIfLong(100 * time.Millisecond) + logger := klog.FromContext(ctx) - if err := sched.Cache.UpdateSnapshot(sched.nodeInfoSnapshot); err != nil { + if err := sched.Cache.UpdateSnapshot(logger, sched.nodeInfoSnapshot); err != nil { return result, err } trace.Step("Snapshotting scheduler cache and node infos done") @@ -362,6 +365,7 @@ func (sched *Scheduler) schedulePod(ctx context.Context, fwk framework.Framework // Filters the nodes to find the ones that fit the pod based on the framework // filter plugins and filter extenders. func (sched *Scheduler) findNodesThatFitPod(ctx context.Context, fwk framework.Framework, state *framework.CycleState, pod *v1.Pod) ([]*v1.Node, framework.Diagnosis, error) { + logger := klog.FromContext(ctx) diagnosis := framework.Diagnosis{ NodeToStatusMap: make(framework.NodeToStatusMap), UnschedulablePlugins: sets.NewString(), @@ -394,7 +398,7 @@ func (sched *Scheduler) findNodesThatFitPod(ctx context.Context, fwk framework.F if len(pod.Status.NominatedNodeName) > 0 { feasibleNodes, err := sched.evaluateNominatedNode(ctx, pod, fwk, state, diagnosis) if err != nil { - klog.ErrorS(err, "Evaluation failed on nominated node", "pod", klog.KObj(pod), "node", pod.Status.NominatedNodeName) + logger.Error(err, "Evaluation failed on nominated node", "pod", klog.KObj(pod), "node", pod.Status.NominatedNodeName) } // Nominated node passes all the filters, scheduler is good to assign this node to the pod. if len(feasibleNodes) != 0 { @@ -418,7 +422,7 @@ func (sched *Scheduler) findNodesThatFitPod(ctx context.Context, fwk framework.F return nil, diagnosis, err } - feasibleNodes, err = findNodesThatPassExtenders(sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) + feasibleNodes, err = findNodesThatPassExtenders(ctx, sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) if err != nil { return nil, diagnosis, err } @@ -437,7 +441,7 @@ func (sched *Scheduler) evaluateNominatedNode(ctx context.Context, pod *v1.Pod, return nil, err } - feasibleNodes, err = findNodesThatPassExtenders(sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) + feasibleNodes, err = findNodesThatPassExtenders(ctx, sched.Extenders, pod, feasibleNodes, diagnosis.NodeToStatusMap) if err != nil { return nil, err } @@ -544,7 +548,8 @@ func (sched *Scheduler) numFeasibleNodesToFind(numAllNodes int32) (numNodes int3 return numNodes } -func findNodesThatPassExtenders(extenders []framework.Extender, pod *v1.Pod, feasibleNodes []*v1.Node, statuses framework.NodeToStatusMap) ([]*v1.Node, error) { +func findNodesThatPassExtenders(ctx context.Context, extenders []framework.Extender, pod *v1.Pod, feasibleNodes []*v1.Node, statuses framework.NodeToStatusMap) ([]*v1.Node, error) { + logger := klog.FromContext(ctx) // Extenders are called sequentially. // Nodes in original feasibleNodes can be excluded in one extender, and pass on to the next // extender in a decreasing manner. @@ -564,7 +569,7 @@ func findNodesThatPassExtenders(extenders []framework.Extender, pod *v1.Pod, fea feasibleList, failedMap, failedAndUnresolvableMap, err := extender.Filter(pod, feasibleNodes) if err != nil { if extender.IsIgnorable() { - klog.InfoS("Skipping extender as it returned error and has ignorable flag set", "extender", extender, "err", err) + logger.Info("Skipping extender as it returned error and has ignorable flag set", "extender", extender, "err", err) continue } return nil, err @@ -636,11 +641,12 @@ func prioritizeNodes( } // Additional details logged at level 10 if enabled. - klogV := klog.V(10) - if klogV.Enabled() { + logger := klog.FromContext(ctx) + loggerV := logger.V(10) + if loggerV.Enabled() { for plugin, nodeScoreList := range scoresMap { for _, nodeScore := range nodeScoreList { - klogV.InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", plugin, "node", nodeScore.Name, "score", nodeScore.Score) + loggerV.Info("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", plugin, "node", nodeScore.Name, "score", nodeScore.Score) } } } @@ -673,14 +679,14 @@ func prioritizeNodes( prioritizedList, weight, err := extenders[extIndex].Prioritize(pod, nodes) if err != nil { // Prioritization errors from extender can be ignored, let k8s/other extenders determine the priorities - klog.V(5).InfoS("Failed to run extender's priority function. No score given by this extender.", "error", err, "pod", klog.KObj(pod), "extender", extenders[extIndex].Name()) + logger.V(5).Info("Failed to run extender's priority function. No score given by this extender.", "error", err, "pod", klog.KObj(pod), "extender", extenders[extIndex].Name()) return } mu.Lock() for i := range *prioritizedList { host, score := (*prioritizedList)[i].Host, (*prioritizedList)[i].Score - if klogV.Enabled() { - klogV.InfoS("Extender scored node for pod", "pod", klog.KObj(pod), "extender", extenders[extIndex].Name(), "node", host, "score", score) + if loggerV.Enabled() { + loggerV.Info("Extender scored node for pod", "pod", klog.KObj(pod), "extender", extenders[extIndex].Name(), "node", host, "score", score) } combinedScores[host] += score * weight } @@ -696,9 +702,9 @@ func prioritizeNodes( } } - if klogV.Enabled() { + if loggerV.Enabled() { for i := range result { - klogV.InfoS("Calculated node's final score for pod", "pod", klog.KObj(pod), "node", result[i].Name, "score", result[i].Score) + loggerV.Info("Calculated node's final score for pod", "pod", klog.KObj(pod), "node", result[i].Name, "score", result[i].Score) } } return result, nil @@ -731,20 +737,20 @@ func selectHost(nodeScoreList framework.NodeScoreList) (string, error) { // assume signals to the cache that a pod is already in the cache, so that binding can be asynchronous. // assume modifies `assumed`. -func (sched *Scheduler) assume(assumed *v1.Pod, host string) error { +func (sched *Scheduler) assume(logger klog.Logger, assumed *v1.Pod, host string) error { // Optimistically assume that the binding will succeed and send it to apiserver // in the background. // If the binding fails, scheduler will release resources allocated to assumed pod // immediately. assumed.Spec.NodeName = host - if err := sched.Cache.AssumePod(assumed); err != nil { - klog.ErrorS(err, "Scheduler cache AssumePod failed") + if err := sched.Cache.AssumePod(logger, assumed); err != nil { + logger.Error(err, "Scheduler cache AssumePod failed") return err } // if "assumed" is a nominated pod, we should remove it from internal cache if sched.SchedulingQueue != nil { - sched.SchedulingQueue.DeleteNominatedPodIfExists(assumed) + sched.SchedulingQueue.DeleteNominatedPodIfExists(logger, assumed) } return nil @@ -754,8 +760,9 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error { // The precedence for binding is: (1) extenders and (2) framework plugins. // We expect this to run asynchronously, so we handle binding metrics internally. func (sched *Scheduler) bind(ctx context.Context, fwk framework.Framework, assumed *v1.Pod, targetNode string, state *framework.CycleState) (err error) { + logger := klog.FromContext(ctx) defer func() { - sched.finishBinding(fwk, assumed, targetNode, err) + sched.finishBinding(logger, fwk, assumed, targetNode, err) }() bound, err := sched.extendersBinding(assumed, targetNode) @@ -786,12 +793,12 @@ func (sched *Scheduler) extendersBinding(pod *v1.Pod, node string) (bool, error) return false, nil } -func (sched *Scheduler) finishBinding(fwk framework.Framework, assumed *v1.Pod, targetNode string, err error) { - if finErr := sched.Cache.FinishBinding(assumed); finErr != nil { - klog.ErrorS(finErr, "Scheduler cache FinishBinding failed") +func (sched *Scheduler) finishBinding(logger klog.Logger, fwk framework.Framework, assumed *v1.Pod, targetNode string, err error) { + if finErr := sched.Cache.FinishBinding(logger, assumed); finErr != nil { + logger.Error(finErr, "Scheduler cache FinishBinding failed") } if err != nil { - klog.V(1).InfoS("Failed to bind pod", "pod", klog.KObj(assumed)) + logger.V(1).Info("Failed to bind pod", "pod", klog.KObj(assumed)) return } @@ -810,14 +817,15 @@ func getAttemptsLabel(p *framework.QueuedPodInfo) string { // handleSchedulingFailure records an event for the pod that indicates the // pod has failed to schedule. Also, update the pod condition and nominated node name if set. func (sched *Scheduler) handleSchedulingFailure(ctx context.Context, fwk framework.Framework, podInfo *framework.QueuedPodInfo, err error, reason string, nominatingInfo *framework.NominatingInfo) { - sched.Error(podInfo, err) + sched.Error(ctx, podInfo, err) // Update the scheduling queue with the nominated pod information. Without // this, there would be a race condition between the next scheduling cycle // and the time the scheduler receives a Pod Update for the nominated pod. // Here we check for nil only for tests. if sched.SchedulingQueue != nil { - sched.SchedulingQueue.AddNominatedPod(podInfo.PodInfo, nominatingInfo) + logger := klog.FromContext(ctx) + sched.SchedulingQueue.AddNominatedPod(logger, podInfo.PodInfo, nominatingInfo) } pod := podInfo.Pod @@ -829,7 +837,7 @@ func (sched *Scheduler) handleSchedulingFailure(ctx context.Context, fwk framewo Reason: reason, Message: err.Error(), }, nominatingInfo); err != nil { - klog.ErrorS(err, "Error updating pod", "pod", klog.KObj(pod)) + klog.FromContext(ctx).Error(err, "Error updating pod", "pod", klog.KObj(pod)) } } @@ -844,7 +852,8 @@ func truncateMessage(message string) string { } func updatePod(ctx context.Context, client clientset.Interface, pod *v1.Pod, condition *v1.PodCondition, nominatingInfo *framework.NominatingInfo) error { - klog.V(3).InfoS("Updating pod condition", "pod", klog.KObj(pod), "conditionType", condition.Type, "conditionStatus", condition.Status, "conditionReason", condition.Reason) + logger := klog.FromContext(ctx) + logger.V(3).Info("Updating pod condition", "pod", klog.KObj(pod), "conditionType", condition.Type, "conditionStatus", condition.Status, "conditionReason", condition.Reason) podStatusCopy := pod.Status.DeepCopy() // NominatedNodeName is updated only if we are trying to set it, and the value is // different from the existing one. diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go index c563e6c89524..48f0e69f26b5 100644 --- a/pkg/scheduler/schedule_one_test.go +++ b/pkg/scheduler/schedule_one_test.go @@ -44,6 +44,8 @@ import ( clientcache "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" "k8s.io/component-helpers/storage/volume" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" extenderv1 "k8s.io/kube-scheduler/extender/v1" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" @@ -96,6 +98,7 @@ func (f *fakeExtender) IsIgnorable() bool { } func (f *fakeExtender) ProcessPreemption( + _ klog.Logger, _ *v1.Pod, _ map[string]*extenderv1.Victims, _ framework.NodeInfoLister, @@ -355,11 +358,11 @@ func TestSchedulerMultipleProfilesScheduling(t *testing.T) { informerFactory := informers.NewSharedInformerFactory(client, 0) sched, err := New( + ctx, client, informerFactory, nil, profile.NewRecorderFactory(broadcaster), - ctx.Done(), WithProfiles( schedulerapi.KubeSchedulerProfile{SchedulerName: "match-node2", Plugins: &schedulerapi.Plugins{ @@ -542,6 +545,10 @@ func TestSchedulerScheduleOne(t *testing.T) { for _, item := range table { t.Run(item.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var gotError error var gotPod *v1.Pod var gotForgetPod *v1.Pod @@ -581,16 +588,14 @@ func TestSchedulerScheduleOne(t *testing.T) { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s := newScheduler( + logger, cache, nil, func() *framework.QueuedPodInfo { return &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(item.sendPod)} }, - func(p *framework.QueuedPodInfo, err error) { + func(_ context.Context, p *framework.QueuedPodInfo, err error) { gotPod = p.Pod gotError = err }, @@ -636,13 +641,14 @@ func TestSchedulerScheduleOne(t *testing.T) { } func TestSchedulerNoPhantomPodAfterExpire(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queuedPodStore := clientcache.NewFIFO(clientcache.MetaNamespaceKeyFunc) - scache := internalcache.New(100*time.Millisecond, ctx.Done()) + scache := internalcache.New(ctx, 100*time.Millisecond) pod := podWithPort("pod.Name", "", 8080) node := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1", UID: types.UID("node1")}} - scache.AddNode(&node) + scache.AddNode(logger, &node) fns := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), @@ -701,13 +707,14 @@ func TestSchedulerNoPhantomPodAfterExpire(t *testing.T) { } func TestSchedulerNoPhantomPodAfterDelete(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queuedPodStore := clientcache.NewFIFO(clientcache.MetaNamespaceKeyFunc) - scache := internalcache.New(10*time.Minute, ctx.Done()) + scache := internalcache.New(ctx, 10*time.Minute) firstPod := podWithPort("pod.Name", "", 8080) node := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1", UID: types.UID("node1")}} - scache.AddNode(&node) + scache.AddNode(logger, &node) fns := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), @@ -746,10 +753,10 @@ func TestSchedulerNoPhantomPodAfterDelete(t *testing.T) { // and would be removed itself (without any explicit actions on schedulernodeinfo). Even in that case, // explicitly AddPod will as well correct the behavior. firstPod.Spec.NodeName = node.Name - if err := scache.AddPod(firstPod); err != nil { + if err := scache.AddPod(logger, firstPod); err != nil { t.Fatalf("err: %v", err) } - if err := scache.RemovePod(firstPod); err != nil { + if err := scache.RemovePod(logger, firstPod); err != nil { t.Fatalf("err: %v", err) } @@ -770,10 +777,11 @@ func TestSchedulerNoPhantomPodAfterDelete(t *testing.T) { } func TestSchedulerFailedSchedulingReasons(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() queuedPodStore := clientcache.NewFIFO(clientcache.MetaNamespaceKeyFunc) - scache := internalcache.New(10*time.Minute, ctx.Done()) + scache := internalcache.New(ctx, 10*time.Minute) // Design the baseline for the pods, and we will make nodes that don't fit it later. var cpu = int64(4) @@ -805,7 +813,7 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) { v1.ResourcePods: *(resource.NewQuantity(10, resource.DecimalSI)), }}, } - scache.AddNode(&node) + scache.AddNode(logger, &node) nodes = append(nodes, &node) objects = append(objects, &node) } @@ -1034,6 +1042,10 @@ func TestSchedulerBinding(t *testing.T) { for _, test := range table { t.Run(test.name, func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + pod := st.MakePod().Name(test.podName).Obj() defaultBound := false client := clientsetfake.NewSimpleClientset(pod) @@ -1050,15 +1062,13 @@ func TestSchedulerBinding(t *testing.T) { if err != nil { t.Fatal(err) } - stop := make(chan struct{}) - defer close(stop) sched := &Scheduler{ Extenders: test.extenders, - Cache: internalcache.New(100*time.Millisecond, stop), + Cache: internalcache.New(ctx, 100*time.Millisecond), nodeInfoSnapshot: nil, percentageOfNodesToScore: 0, } - err = sched.bind(context.Background(), fwk, pod, "node", nil) + err = sched.bind(ctx, fwk, pod, "node", nil) if err != nil { t.Error(err) } @@ -1481,13 +1491,14 @@ func TestFindNodesThatPassExtenders(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) var extenders []framework.Extender for ii := range tt.extenders { extenders = append(extenders, &tt.extenders[ii]) } pod := st.MakePod().Name("1").UID("1").Obj() - got, err := findNodesThatPassExtenders(extenders, pod, tt.nodes, tt.filteredNodesStatuses) + got, err := findNodesThatPassExtenders(ctx, extenders, pod, tt.nodes, tt.filteredNodesStatuses) if tt.expectsErr { if err == nil { t.Error("Unexpected non-error") @@ -1984,19 +1995,21 @@ func TestSchedulerSchedulePod(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cache := internalcache.New(time.Duration(0), wait.NeverStop) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + cache := internalcache.New(ctx, time.Duration(0)) for _, pod := range test.pods { - cache.AddPod(pod) + cache.AddPod(logger, pod) } var nodes []*v1.Node for _, name := range test.nodes { node := &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"hostname": name}}} nodes = append(nodes, node) - cache.AddNode(node) + cache.AddNode(logger, node) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() cs := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(cs, 0) for _, pvc := range test.pvcs { @@ -2019,6 +2032,7 @@ func TestSchedulerSchedulePod(t *testing.T) { } scheduler := newScheduler( + logger, cache, nil, nil, @@ -2059,8 +2073,9 @@ func TestSchedulerSchedulePod(t *testing.T) { } func TestFindFitAllError(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) nodes := makeNodeList([]string{"3", "2", "1"}) - scheduler := makeScheduler(nodes) + scheduler := makeScheduler(ctx, nodes) fwk, err := st.NewFramework( []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), @@ -2094,8 +2109,9 @@ func TestFindFitAllError(t *testing.T) { } func TestFindFitSomeError(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) nodes := makeNodeList([]string{"3", "2", "1"}) - scheduler := makeScheduler(nodes) + scheduler := makeScheduler(ctx, nodes) fwk, err := st.NewFramework( []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), @@ -2161,6 +2177,10 @@ func TestFindFitPredicateCallCounts(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + nodes := makeNodeList([]string{"1"}) plugin := st.FakeFilterPlugin{} @@ -2183,11 +2203,11 @@ func TestFindFitPredicateCallCounts(t *testing.T) { t.Fatal(err) } - scheduler := makeScheduler(nodes) - if err := scheduler.Cache.UpdateSnapshot(scheduler.nodeInfoSnapshot); err != nil { + scheduler := makeScheduler(ctx, nodes) + if err := scheduler.Cache.UpdateSnapshot(logger, scheduler.nodeInfoSnapshot); err != nil { t.Fatal(err) } - fwk.AddNominatedPod(framework.NewPodInfo(st.MakePod().UID("nominated").Priority(midPriority).Obj()), + fwk.AddNominatedPod(logger, framework.NewPodInfo(st.MakePod().UID("nominated").Priority(midPriority).Obj()), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "1"}) _, _, err = scheduler.findNodesThatFitPod(context.Background(), fwk, framework.NewCycleState(), test.pod) @@ -2296,6 +2316,10 @@ func TestZeroRequest(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) @@ -2321,6 +2345,7 @@ func TestZeroRequest(t *testing.T) { } scheduler := newScheduler( + logger, nil, nil, nil, @@ -2332,7 +2357,6 @@ func TestZeroRequest(t *testing.T) { snapshot, schedulerapi.DefaultPercentageOfNodesToScore) - ctx := context.Background() state := framework.NewCycleState() _, _, err = scheduler.findNodesThatFitPod(ctx, fwk, state, test.pod) if err != nil { @@ -2408,13 +2432,17 @@ func TestNumFeasibleNodesToFind(t *testing.T) { } func TestFairEvaluationForNodes(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + numAllNodes := 500 nodeNames := make([]string, 0, numAllNodes) for i := 0; i < numAllNodes; i++ { nodeNames = append(nodeNames, strconv.Itoa(i)) } nodes := makeNodeList(nodeNames) - sched := makeScheduler(nodes) + sched := makeScheduler(ctx, nodes) fwk, err := st.NewFramework( []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), @@ -2475,13 +2503,17 @@ func TestPreferNominatedNodeFilterCallCounts(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + // create three nodes in the cluster. nodes := makeNodeList([]string{"node1", "node2", "node3"}) client := clientsetfake.NewSimpleClientset(test.pod) informerFactory := informers.NewSharedInformerFactory(client, 0) - cache := internalcache.New(time.Duration(0), wait.NeverStop) + cache := internalcache.New(ctx, time.Duration(0)) for _, n := range nodes { - cache.AddNode(n) + cache.AddNode(logger, n) } plugin := st.FakeFilterPlugin{FailedNodeReturnCodeMap: test.nodeReturnCodeMap} registerFakeFilterFunc := st.RegisterFilterPlugin( @@ -2505,6 +2537,7 @@ func TestPreferNominatedNodeFilterCallCounts(t *testing.T) { } snapshot := internalcache.NewSnapshot(nil, nodes) scheduler := newScheduler( + logger, cache, nil, nil, @@ -2560,13 +2593,15 @@ func makeNodeList(nodeNames []string) []*v1.Node { } // makeScheduler makes a simple Scheduler for testing. -func makeScheduler(nodes []*v1.Node) *Scheduler { - cache := internalcache.New(time.Duration(0), wait.NeverStop) +func makeScheduler(ctx context.Context, nodes []*v1.Node) *Scheduler { + logger := klog.FromContext(ctx) + cache := internalcache.New(ctx, time.Duration(0)) for _, n := range nodes { - cache.AddNode(n) + cache.AddNode(logger, n) } s := newScheduler( + logger, cache, nil, nil, @@ -2577,7 +2612,7 @@ func makeScheduler(nodes []*v1.Node) *Scheduler { nil, emptySnapshot, schedulerapi.DefaultPercentageOfNodesToScore) - cache.UpdateSnapshot(s.nodeInfoSnapshot) + cache.UpdateSnapshot(logger, s.nodeInfoSnapshot) return s } @@ -2632,6 +2667,7 @@ func setupTestSchedulerWithOnePodOnNode(ctx context.Context, t *testing.T, queue // queuedPodStore: pods queued before processing. // scache: scheduler cache that might contain assumed pods. func setupTestScheduler(ctx context.Context, queuedPodStore *clientcache.FIFO, cache internalcache.Cache, informerFactory informers.SharedInformerFactory, broadcaster events.EventBroadcaster, fns ...st.RegisterPluginFunc) (*Scheduler, chan *v1.Binding, chan error) { + logger := klog.FromContext(ctx) bindingChan := make(chan *v1.Binding, 1) client := clientsetfake.NewSimpleClientset() client.PrependReactor("create", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { @@ -2666,12 +2702,13 @@ func setupTestScheduler(ctx context.Context, queuedPodStore *clientcache.FIFO, c errChan := make(chan error, 1) sched := newScheduler( + logger, cache, nil, func() *framework.QueuedPodInfo { return &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(clientcache.Pop(queuedPodStore).(*v1.Pod))} }, - func(p *framework.QueuedPodInfo, err error) { + func(_ context.Context, p *framework.QueuedPodInfo, err error) { errChan <- err }, nil, @@ -2686,6 +2723,7 @@ func setupTestScheduler(ctx context.Context, queuedPodStore *clientcache.FIFO, c } func setupTestSchedulerWithVolumeBinding(ctx context.Context, volumeBinder volumebinding.SchedulerVolumeBinder, broadcaster events.EventBroadcaster) (*Scheduler, chan *v1.Binding, chan error) { + logger := klog.FromContext(ctx) testNode := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1", UID: types.UID("node1")}} queuedPodStore := clientcache.NewFIFO(clientcache.MetaNamespaceKeyFunc) pod := podWithID("foo", "") @@ -2693,8 +2731,8 @@ func setupTestSchedulerWithVolumeBinding(ctx context.Context, volumeBinder volum pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{Name: "testVol", VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: "testPVC"}}}) queuedPodStore.Add(pod) - scache := internalcache.New(10*time.Minute, ctx.Done()) - scache.AddNode(&testNode) + scache := internalcache.New(ctx, 10*time.Minute) + scache.AddNode(logger, &testNode) testPVC := v1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "testPVC", Namespace: pod.Namespace, UID: types.UID("testPVC")}} client := clientsetfake.NewSimpleClientset(&testNode, &testPVC) informerFactory := informers.NewSharedInformerFactory(client, 0) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index c165b34ffc29..9e9315cab080 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -76,7 +76,7 @@ type Scheduler struct { // Error is called if there is an error. It is passed the pod in // question, and the error - Error func(*framework.QueuedPodInfo, error) + Error func(context.Context, *framework.QueuedPodInfo, error) // SchedulePod tries to schedule the given pod to one of the nodes in the node list. // Return a struct of ScheduleResult with the name of suggested host on success, @@ -99,6 +99,11 @@ type Scheduler struct { percentageOfNodesToScore int32 nextStartNodeIndex int + + // logger *must* be initialized when creating a Scheduler, + // otherwise logging functions will access a nil sink and + // panic. + logger klog.Logger } type schedulerOptions struct { @@ -231,17 +236,15 @@ var defaultSchedulerOptions = schedulerOptions{ } // New returns a Scheduler -func New(client clientset.Interface, +func New(ctx context.Context, + client clientset.Interface, informerFactory informers.SharedInformerFactory, dynInformerFactory dynamicinformer.DynamicSharedInformerFactory, recorderFactory profile.RecorderFactory, - stopCh <-chan struct{}, opts ...Option) (*Scheduler, error) { - stopEverything := stopCh - if stopEverything == nil { - stopEverything = wait.NeverStop - } + logger := klog.FromContext(ctx) + stopEverything := ctx.Done() options := defaultSchedulerOptions for _, opt := range opts { @@ -265,7 +268,7 @@ func New(client clientset.Interface, metrics.Register() - extenders, err := buildExtenders(options.extenders, options.profiles) + extenders, err := buildExtenders(logger, options.extenders, options.profiles) if err != nil { return nil, fmt.Errorf("couldn't build extenders: %w", err) } @@ -278,7 +281,7 @@ func New(client clientset.Interface, snapshot := internalcache.NewEmptySnapshot() clusterEventMap := make(map[framework.ClusterEvent]sets.String) - profiles, err := profile.NewMap(options.profiles, registry, recorderFactory, stopCh, + profiles, err := profile.NewMap(options.profiles, registry, recorderFactory, stopEverything, frameworkruntime.WithComponentConfigVersion(options.componentConfigVersion), frameworkruntime.WithClientSet(client), frameworkruntime.WithKubeConfig(options.kubeConfig), @@ -308,16 +311,17 @@ func New(client clientset.Interface, internalqueue.WithPodMaxInUnschedulablePodsDuration(options.podMaxInUnschedulablePodsDuration), ) - schedulerCache := internalcache.New(durationToExpireAssumedPod, stopEverything) + schedulerCache := internalcache.New(ctx, durationToExpireAssumedPod) // Setup cache debugger. debugger := cachedebugger.New(nodeLister, podLister, schedulerCache, podQueue) - debugger.ListenForSignal(stopEverything) + debugger.ListenForSignal(ctx) sched := newScheduler( + logger, schedulerCache, extenders, - internalqueue.MakeNextPodFunc(podQueue), + internalqueue.MakeNextPodFunc(logger, podQueue), MakeDefaultErrorFunc(client, podLister, podQueue, schedulerCache), stopEverything, podQueue, @@ -334,7 +338,8 @@ func New(client clientset.Interface, // Run begins watching and scheduling. It starts scheduling and blocked until the context is done. func (sched *Scheduler) Run(ctx context.Context) { - sched.SchedulingQueue.Run() + logger := klog.FromContext(ctx) + sched.SchedulingQueue.Run(logger) // We need to start scheduleOne loop in a dedicated goroutine, // because scheduleOne function hangs on getting the next item @@ -349,17 +354,18 @@ func (sched *Scheduler) Run(ctx context.Context) { } // MakeDefaultErrorFunc construct a function to handle pod scheduler error -func MakeDefaultErrorFunc(client clientset.Interface, podLister corelisters.PodLister, podQueue internalqueue.SchedulingQueue, schedulerCache internalcache.Cache) func(*framework.QueuedPodInfo, error) { - return func(podInfo *framework.QueuedPodInfo, err error) { +func MakeDefaultErrorFunc(client clientset.Interface, podLister corelisters.PodLister, podQueue internalqueue.SchedulingQueue, schedulerCache internalcache.Cache) func(context.Context, *framework.QueuedPodInfo, error) { + return func(ctx context.Context, podInfo *framework.QueuedPodInfo, err error) { + logger := klog.FromContext(ctx) pod := podInfo.Pod if err == ErrNoNodesAvailable { - klog.V(2).InfoS("Unable to schedule pod; no nodes are registered to the cluster; waiting", "pod", klog.KObj(pod)) + logger.V(2).Info("Unable to schedule pod; no nodes are registered to the cluster; waiting", "pod", klog.KObj(pod)) } else if fitError, ok := err.(*framework.FitError); ok { // Inject UnschedulablePlugins to PodInfo, which will be used later for moving Pods between queues efficiently. podInfo.UnschedulablePlugins = fitError.Diagnosis.UnschedulablePlugins - klog.V(2).InfoS("Unable to schedule pod; no fit; waiting", "pod", klog.KObj(pod), "err", err) + logger.V(2).Info("Unable to schedule pod; no fit; waiting", "pod", klog.KObj(pod), "err", err) } else if apierrors.IsNotFound(err) { - klog.V(2).InfoS("Unable to schedule pod, possibly due to node not found; waiting", "pod", klog.KObj(pod), "err", err) + logger.V(2).Info("Unable to schedule pod, possibly due to node not found; waiting", "pod", klog.KObj(pod), "err", err) if errStatus, ok := err.(apierrors.APIStatus); ok && errStatus.Status().Details.Kind == "node" { nodeName := errStatus.Status().Details.Name // when node is not found, We do not remove the node right away. Trying again to get @@ -367,33 +373,33 @@ func MakeDefaultErrorFunc(client clientset.Interface, podLister corelisters.PodL _, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil && apierrors.IsNotFound(err) { node := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}} - if err := schedulerCache.RemoveNode(&node); err != nil { - klog.V(4).InfoS("Node is not found; failed to remove it from the cache", "node", node.Name) + if err := schedulerCache.RemoveNode(logger, &node); err != nil { + logger.V(4).Info("Node is not found; failed to remove it from the cache", "node", node.Name) } } } } else { - klog.ErrorS(err, "Error scheduling pod; retrying", "pod", klog.KObj(pod)) + logger.Error(err, "Error scheduling pod; retrying", "pod", klog.KObj(pod)) } // Check if the Pod exists in informer cache. cachedPod, err := podLister.Pods(pod.Namespace).Get(pod.Name) if err != nil { - klog.InfoS("Pod doesn't exist in informer cache", "pod", klog.KObj(pod), "err", err) + logger.Info("Pod doesn't exist in informer cache", "pod", klog.KObj(pod), "err", err) return } // In the case of extender, the pod may have been bound successfully, but timed out returning its response to the scheduler. // It could result in the live version to carry .spec.nodeName, and that's inconsistent with the internal-queued version. if len(cachedPod.Spec.NodeName) != 0 { - klog.InfoS("Pod has been assigned to node. Abort adding it back to queue.", "pod", klog.KObj(pod), "node", cachedPod.Spec.NodeName) + logger.Info("Pod has been assigned to node. Abort adding it back to queue.", "pod", klog.KObj(pod), "node", cachedPod.Spec.NodeName) return } // As is from SharedInformer, we need to do a DeepCopy() here. podInfo.PodInfo = framework.NewPodInfo(cachedPod.DeepCopy()) - if err := podQueue.AddUnschedulableIfNotPresent(podInfo, podQueue.SchedulingCycle()); err != nil { - klog.ErrorS(err, "Error occurred") + if err := podQueue.AddUnschedulableIfNotPresent(logger, podInfo, podQueue.SchedulingCycle()); err != nil { + logger.Error(err, "Error occurred") } } } @@ -406,7 +412,7 @@ func NewInformerFactory(cs clientset.Interface, resyncPeriod time.Duration) info return informerFactory } -func buildExtenders(extenders []schedulerapi.Extender, profiles []schedulerapi.KubeSchedulerProfile) ([]framework.Extender, error) { +func buildExtenders(logger klog.Logger, extenders []schedulerapi.Extender, profiles []schedulerapi.KubeSchedulerProfile) ([]framework.Extender, error) { var fExtenders []framework.Extender if len(extenders) == 0 { return nil, nil @@ -415,7 +421,7 @@ func buildExtenders(extenders []schedulerapi.Extender, profiles []schedulerapi.K var ignoredExtendedResources []string var ignorableExtenders []framework.Extender for i := range extenders { - klog.V(2).InfoS("Creating extender", "extender", extenders[i]) + logger.V(2).Info("Creating extender", "extender", extenders[i]) extender, err := NewHTTPExtender(&extenders[i]) if err != nil { return nil, err @@ -466,10 +472,11 @@ func buildExtenders(extenders []schedulerapi.Extender, profiles []schedulerapi.K // newScheduler creates a Scheduler object. func newScheduler( + logger klog.Logger, cache internalcache.Cache, extenders []framework.Extender, nextPod func() *framework.QueuedPodInfo, - Error func(*framework.QueuedPodInfo, error), + Error func(context.Context, *framework.QueuedPodInfo, error), stopEverything <-chan struct{}, schedulingQueue internalqueue.SchedulingQueue, profiles profile.Map, @@ -487,6 +494,7 @@ func newScheduler( client: client, nodeInfoSnapshot: nodeInfoSnapshot, percentageOfNodesToScore: percentageOfNodesToScore, + logger: logger, } sched.SchedulePod = sched.schedulePod return &sched diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 47298d5869a4..7c8c1f9aaed6 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/events" + "k8s.io/klog/v2/ktesting" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" @@ -163,19 +164,20 @@ func TestSchedulerCreation(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() client := fake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()}) - stopCh := make(chan struct{}) - defer close(stopCh) s, err := New( + ctx, client, informerFactory, nil, profile.NewRecorderFactory(eventBroadcaster), - stopCh, tc.opts..., ) @@ -259,8 +261,9 @@ func TestDefaultErrorFunc(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - stopCh := make(chan struct{}) - defer close(stopCh) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() client := fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testPod}}) informerFactory := informers.NewSharedInformerFactory(client, 0) @@ -269,23 +272,23 @@ func TestDefaultErrorFunc(t *testing.T) { podInformer.Informer().GetStore().Add(testPod) queue := internalqueue.NewPriorityQueue(nil, informerFactory, internalqueue.WithClock(testingclock.NewFakeClock(time.Now()))) - schedulerCache := internalcache.New(30*time.Second, stopCh) + schedulerCache := internalcache.New(ctx, 30*time.Second) - queue.Add(testPod) + queue.Add(logger, testPod) queue.Pop() if tt.podUpdatedDuringScheduling { podInformer.Informer().GetStore().Update(testPodUpdated) - queue.Update(testPod, testPodUpdated) + queue.Update(logger, testPod, testPodUpdated) } if tt.podDeletedDuringScheduling { podInformer.Informer().GetStore().Delete(testPod) - queue.Delete(testPod) + queue.Delete(logger, testPod) } testPodInfo := &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(testPod)} errFunc := MakeDefaultErrorFunc(client, podInformer.Lister(), queue, schedulerCache) - errFunc(testPodInfo, tt.injectErr) + errFunc(ctx, testPodInfo, tt.injectErr) var got *v1.Pod if tt.podUpdatedDuringScheduling { @@ -333,7 +336,8 @@ func TestDefaultErrorFunc_NodeNotFound(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) defer cancel() client := fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testPod}}, &v1.NodeList{Items: tt.nodes}) @@ -343,12 +347,12 @@ func TestDefaultErrorFunc_NodeNotFound(t *testing.T) { podInformer.Informer().GetStore().Add(testPod) queue := internalqueue.NewPriorityQueue(nil, informerFactory, internalqueue.WithClock(testingclock.NewFakeClock(time.Now()))) - schedulerCache := internalcache.New(30*time.Second, ctx.Done()) + schedulerCache := internalcache.New(ctx, 30*time.Second) for i := range tt.nodes { node := tt.nodes[i] // Add node to schedulerCache no matter it's deleted in API server or not. - schedulerCache.AddNode(&node) + schedulerCache.AddNode(logger, &node) if node.Name == tt.nodeNameToDelete { client.CoreV1().Nodes().Delete(ctx, node.Name, metav1.DeleteOptions{}) } @@ -356,7 +360,7 @@ func TestDefaultErrorFunc_NodeNotFound(t *testing.T) { testPodInfo := &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(testPod)} errFunc := MakeDefaultErrorFunc(client, podInformer.Lister(), queue, schedulerCache) - errFunc(testPodInfo, tt.injectErr) + errFunc(ctx, testPodInfo, tt.injectErr) gotNodes := schedulerCache.Dump().Nodes gotNodeNames := sets.NewString() @@ -371,8 +375,9 @@ func TestDefaultErrorFunc_NodeNotFound(t *testing.T) { } func TestDefaultErrorFunc_PodAlreadyBound(t *testing.T) { - stopCh := make(chan struct{}) - defer close(stopCh) + logger, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() nodeFoo := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} testPod := st.MakePod().Name("test-pod").Namespace(v1.NamespaceDefault).Node("foo").Obj() @@ -384,14 +389,14 @@ func TestDefaultErrorFunc_PodAlreadyBound(t *testing.T) { podInformer.Informer().GetStore().Add(testPod) queue := internalqueue.NewPriorityQueue(nil, informerFactory, internalqueue.WithClock(testingclock.NewFakeClock(time.Now()))) - schedulerCache := internalcache.New(30*time.Second, stopCh) + schedulerCache := internalcache.New(ctx, 30*time.Second) // Add node to schedulerCache no matter it's deleted in API server or not. - schedulerCache.AddNode(&nodeFoo) + schedulerCache.AddNode(logger, &nodeFoo) testPodInfo := &framework.QueuedPodInfo{PodInfo: framework.NewPodInfo(testPod)} errFunc := MakeDefaultErrorFunc(client, podInformer.Lister(), queue, schedulerCache) - errFunc(testPodInfo, fmt.Errorf("binding rejected: timeout")) + errFunc(ctx, testPodInfo, fmt.Errorf("binding rejected: timeout")) pod := getPodFromPriorityQueue(queue, testPod) if pod != nil { diff --git a/pkg/scheduler/testing/fake_extender.go b/pkg/scheduler/testing/fake_extender.go index ab66d6c8ca80..e9d933ccea13 100644 --- a/pkg/scheduler/testing/fake_extender.go +++ b/pkg/scheduler/testing/fake_extender.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" corev1helpers "k8s.io/component-helpers/scheduling/corev1" + "k8s.io/klog/v2" extenderv1 "k8s.io/kube-scheduler/extender/v1" "k8s.io/kubernetes/pkg/scheduler/framework" frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" @@ -182,6 +183,7 @@ func (f *FakeExtender) SupportsPreemption() bool { // ProcessPreemption implements the extender preempt function. func (f *FakeExtender) ProcessPreemption( + logger klog.Logger, pod *v1.Pod, nodeNameToVictims map[string]*extenderv1.Victims, nodeInfos framework.NodeInfoLister, @@ -199,7 +201,7 @@ func (f *FakeExtender) ProcessPreemption( for nodeName, victims := range nodeNameToVictimsCopy { // Try to do preemption on extender side. nodeInfo, _ := nodeInfos.Get(nodeName) - extenderVictimPods, extenderPDBViolations, fits, err := f.selectVictimsOnNodeByExtender(pod, nodeInfo.Node()) + extenderVictimPods, extenderPDBViolations, fits, err := f.selectVictimsOnNodeByExtender(logger, pod, nodeInfo.Node()) if err != nil { return nil, err } @@ -221,7 +223,7 @@ func (f *FakeExtender) ProcessPreemption( // 1. More victim pods (if any) amended by preemption phase of extender. // 2. Number of violating victim (used to calculate PDB). // 3. Fits or not after preemption phase on extender's side. -func (f *FakeExtender) selectVictimsOnNodeByExtender(pod *v1.Pod, node *v1.Node) ([]*v1.Pod, int, bool, error) { +func (f *FakeExtender) selectVictimsOnNodeByExtender(logger klog.Logger, pod *v1.Pod, node *v1.Node) ([]*v1.Pod, int, bool, error) { // If a extender support preemption but have no cached node info, let's run filter to make sure // default scheduler's decision still stand with given pod and node. if !f.NodeCacheCapable { @@ -242,7 +244,7 @@ func (f *FakeExtender) selectVictimsOnNodeByExtender(pod *v1.Pod, node *v1.Node) var potentialVictims []*v1.Pod removePod := func(rp *v1.Pod) { - nodeInfoCopy.RemovePod(rp) + nodeInfoCopy.RemovePod(logger, rp) } addPod := func(ap *v1.Pod) { nodeInfoCopy.AddPod(ap) diff --git a/pkg/scheduler/util/utils.go b/pkg/scheduler/util/utils.go index 929dfb8fa443..d431a953ac33 100644 --- a/pkg/scheduler/util/utils.go +++ b/pkg/scheduler/util/utils.go @@ -56,7 +56,7 @@ func GetPodStartTime(pod *v1.Pod) *metav1.Time { func GetEarliestPodStartTime(victims *extenderv1.Victims) *metav1.Time { if len(victims.Pods) == 0 { // should not reach here. - klog.ErrorS(fmt.Errorf("victims.Pods is empty. Should not reach here"), "") + klog.Background().Error(nil, "victims.Pods is empty. Should not reach here") return nil } diff --git a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go index 35d5f0a866be..8a9a03f3a24d 100644 --- a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go +++ b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go @@ -33,6 +33,7 @@ import ( "k8s.io/cloud-provider/app/config" "k8s.io/cloud-provider/options" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog/v2" ) // TearDownFunc is to be called to tear down a test server. @@ -47,20 +48,14 @@ type TestServer struct { TmpDir string // Temp Dir used, by the apiserver } -// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie -type Logger interface { - Errorf(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) - Logf(format string, args ...interface{}) -} - // StartTestServer starts a cloud-controller-manager. A rest client config and a tear-down func, // and location of the tmpdir are returned. // // Note: we return a tear-down func instead of a stop channel because the later will leak temporary // files that because Golang testing's call to os.Exit will not give a stop channel go routine // enough time to remove temporary files. -func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { +func StartTestServer(ctx context.Context, customFlags []string) (result TestServer, err error) { + logger := klog.FromContext(ctx) stopCh := make(chan struct{}) configDoneCh := make(chan struct{}) var capturedConfig config.CompletedConfig @@ -93,11 +88,11 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) if err != nil { - t.Fatalf("Cloud provider could not be initialized: %v", err) + panic(fmt.Errorf("Cloud provider could not be initialized: %v", err)) } s.SecureServing.ServerCert.CertDirectory = result.TmpDir if cloud == nil { - t.Fatalf("Cloud provider is nil") + panic("Cloud provider is nil") } return cloud } @@ -129,7 +124,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err commandArgs = append(commandArgs, fmt.Sprintf("--secure-port=%d", bindPort)) commandArgs = append(commandArgs, fmt.Sprintf("--cert-dir=%s", result.TmpDir)) - t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort) + logger.Info("cloud-controller-manager will listen securely", "port", bindPort) } for _, listener := range listeners { listener.Close() @@ -151,7 +146,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err return result, err } - t.Logf("Waiting for /healthz to be ok...") + logger.Info("Waiting for /healthz to be ok...") client, err := kubernetes.NewForConfig(capturedConfig.LoopbackClientConfig) if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) @@ -185,14 +180,13 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t Logger, flags []string) *TestServer { - result, err := StartTestServer(t, flags) +func StartTestServerOrDie(ctx context.Context, flags []string) *TestServer { + result, err := StartTestServer(ctx, flags) if err == nil { return &result } - t.Fatalf("failed to launch server: %v", err) - return nil + panic(fmt.Errorf("failed to launch server: %v", err)) } func createListenerOnFreePort() (net.Listener, int, error) { diff --git a/test/integration/daemonset/daemonset_test.go b/test/integration/daemonset/daemonset_test.go index fe1cb4e0cb06..72aa494447b4 100644 --- a/test/integration/daemonset/daemonset_test.go +++ b/test/integration/daemonset/daemonset_test.go @@ -81,11 +81,11 @@ func setup(t *testing.T) (context.Context, kubeapiservertesting.TearDownFunc, *d }) sched, err := scheduler.New( + ctx, clientSet, informers, nil, profile.NewRecorderFactory(eventBroadcaster), - ctx.Done(), ) if err != nil { t.Fatalf("Couldn't create scheduler: %v", err) diff --git a/test/integration/scheduler/queue_test.go b/test/integration/scheduler/queue_test.go index 8c9364deb4fe..a0b37d408fa6 100644 --- a/test/integration/scheduler/queue_test.go +++ b/test/integration/scheduler/queue_test.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" "k8s.io/kube-scheduler/config/v1beta3" apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/scheduler" @@ -105,7 +106,7 @@ func TestCoreResourceEnqueue(t *testing.T) { if fitError == nil { t.Fatalf("Expect Pod %v to fail at scheduling.", podInfo.Pod.Name) } - testCtx.Scheduler.Error(podInfo, fitError) + testCtx.Scheduler.Error(ctx, podInfo, fitError) } // Trigger a NodeTaintChange event. @@ -248,6 +249,7 @@ func TestCustomResourceEnqueue(t *testing.T) { defer testutils.CleanupTest(t, testCtx) cs, ns, ctx := testCtx.ClientSet, testCtx.NS.Name, testCtx.Ctx + logger := klog.FromContext(ctx) // Create one Node. node := st.MakeNode().Name("fake-node").Obj() if _, err := cs.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}); err != nil { @@ -280,11 +282,11 @@ func TestCustomResourceEnqueue(t *testing.T) { if fitError == nil { t.Fatalf("Expect Pod %v to fail at scheduling.", podInfo.Pod.Name) } - testCtx.Scheduler.Error(podInfo, fitError) + testCtx.Scheduler.Error(ctx, podInfo, fitError) // Scheduling cycle is incremented from 0 to 1 after NextPod() is called, so // pass a number larger than 1 to move Pod to unschedulablePods. - testCtx.Scheduler.SchedulingQueue.AddUnschedulableIfNotPresent(podInfo, 10) + testCtx.Scheduler.SchedulingQueue.AddUnschedulableIfNotPresent(logger, podInfo, 10) // Trigger a Custom Resource event. // We expect this event to trigger moving the test Pod from unschedulablePods to activeQ. diff --git a/test/integration/scheduler_perf/main_test.go b/test/integration/scheduler_perf/main_test.go index 16275396eba4..d32ce1287807 100644 --- a/test/integration/scheduler_perf/main_test.go +++ b/test/integration/scheduler_perf/main_test.go @@ -17,11 +17,39 @@ limitations under the License. package benchmark import ( + "flag" + "fmt" + "os" "testing" + "github.com/spf13/pflag" + + logsapi "k8s.io/component-base/logs/api/v1" + _ "k8s.io/component-base/logs/json/register" + "k8s.io/klog/v2/ktesting" "k8s.io/kubernetes/test/integration/framework" ) func TestMain(m *testing.M) { + ktesting.DefaultConfig = ktesting.NewConfig(ktesting.Verbosity(0)) // Run with -v=0 by default. + ktesting.DefaultConfig.AddFlags(flag.CommandLine) + + c := logsapi.NewLoggingConfiguration() + + // component-base only supports pflag at the moment. + var fs pflag.FlagSet + logsapi.AddFlags(c, &fs) + // Not ideal. https://github.com/spf13/pflag/pull/330 would be better. + fs.VisitAll(func(f *pflag.Flag) { + if flag.CommandLine.Lookup(f.Name) == nil { + flag.CommandLine.Var(f.Value, f.Name, f.Usage) + } + }) + flag.Parse() + if err := logsapi.ValidateAndApply(c, nil /* no feature gates */); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + framework.EtcdMain(m.Run) } diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go index e4f2e067d8ad..b96acb7135d0 100644 --- a/test/integration/scheduler_perf/scheduler_perf_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_test.go @@ -634,7 +634,7 @@ func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem { b.Fatalf("validate scheduler config file failed: %v", err) } } - finalFunc, podInformer, client, dynClient := mustSetupScheduler(cfg) + finalFunc, podInformer, client, dynClient := mustSetupScheduler(b, cfg) b.Cleanup(finalFunc) var mu sync.Mutex diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index 114c119a660d..82dd9587012b 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -40,6 +40,7 @@ import ( "k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/testutil" "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta2" "k8s.io/kubernetes/pkg/scheduler/apis/config" kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" @@ -74,7 +75,7 @@ func newDefaultComponentConfig() (*config.KubeSchedulerConfiguration, error) { // remove resources after finished. // Notes on rate limiter: // - client rate limit is set to 5000. -func mustSetupScheduler(config *config.KubeSchedulerConfiguration) (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface, dynamic.Interface) { +func mustSetupScheduler(tl ktesting.TL, config *config.KubeSchedulerConfiguration) (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface, dynamic.Interface) { // Run API server with minimimal logging by default. Can be raised with -v. framework.MinVerbosity = 0 apiURL, apiShutdown := util.StartApiserver() @@ -102,7 +103,7 @@ func mustSetupScheduler(config *config.KubeSchedulerConfiguration) (util.Shutdow // Not all config options will be effective but only those mostly related with scheduler performance will // be applied to start a scheduler, most of them are defined in `scheduler.schedulerOptions`. - _, podInformer, schedulerShutdown := util.StartScheduler(client, cfg, config) + _, podInformer, schedulerShutdown := util.StartScheduler(tl, client, cfg, config) fakePVControllerShutdown := util.StartFakePVController(client) shutdownFunc := func() { diff --git a/test/integration/serving/serving_test.go b/test/integration/serving/serving_test.go index 0e2bd16e07b9..4640c518540e 100644 --- a/test/integration/serving/serving_test.go +++ b/test/integration/serving/serving_test.go @@ -17,6 +17,7 @@ limitations under the License. package serving import ( + "context" "crypto/tls" "crypto/x509" "fmt" @@ -32,6 +33,7 @@ import ( cloudprovider "k8s.io/cloud-provider" cloudctrlmgrtesting "k8s.io/cloud-provider/app/testing" "k8s.io/cloud-provider/fake" + "k8s.io/klog/v2/ktesting" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing" @@ -39,15 +41,15 @@ import ( ) type componentTester interface { - StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) + StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) } type kubeControllerManagerTester struct{} -func (kubeControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { +func (kubeControllerManagerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { // avoid starting any controller loops, we're just testing serving customFlags = append([]string{"--controllers="}, customFlags...) - gotResult, err := kubectrlmgrtesting.StartTestServer(t, customFlags) + gotResult, err := kubectrlmgrtesting.StartTestServer(ctx, customFlags) if err != nil { return nil, nil, nil, nil, err } @@ -56,8 +58,8 @@ func (kubeControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, type cloudControllerManagerTester struct{} -func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { - gotResult, err := cloudctrlmgrtesting.StartTestServer(t, customFlags) +func (cloudControllerManagerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := cloudctrlmgrtesting.StartTestServer(ctx, customFlags) if err != nil { return nil, nil, nil, nil, err } @@ -66,8 +68,8 @@ func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, type kubeSchedulerTester struct{} -func (kubeSchedulerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { - gotResult, err := kubeschedulertesting.StartTestServer(t, customFlags) +func (kubeSchedulerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := kubeschedulertesting.StartTestServer(ctx, customFlags) if err != nil { return nil, nil, nil, nil, err } @@ -252,7 +254,8 @@ func testComponent(t *testing.T, tester componentTester, kubeconfig, brokenKubec } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - secureOptions, secureInfo, insecureInfo, tearDownFn, err := tester.StartTestServer(t, append(append([]string{}, tt.flags...), extraFlags...)) + _, ctx := ktesting.NewTestContext(t) + secureOptions, secureInfo, insecureInfo, tearDownFn, err := tester.StartTestServer(ctx, append(append([]string{}, tt.flags...), extraFlags...)) if tearDownFn != nil { defer tearDownFn() } @@ -375,7 +378,8 @@ func testComponentWithSecureServing(t *testing.T, tester componentTester, kubeco } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - secureOptions, secureInfo, _, tearDownFn, err := tester.StartTestServer(t, append(append([]string{}, tt.flags...), extraFlags...)) + _, ctx := ktesting.NewTestContext(t) + secureOptions, secureInfo, _, tearDownFn, err := tester.StartTestServer(ctx, append(append([]string{}, tt.flags...), extraFlags...)) if tearDownFn != nil { defer tearDownFn() } diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 27dfbd37a160..d486df53eb6b 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -44,6 +44,7 @@ import ( "k8s.io/client-go/tools/events" pvutil "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" podutil "k8s.io/kubernetes/pkg/api/v1/pod" @@ -79,8 +80,9 @@ func StartApiserver() (string, ShutdownFunc) { // StartScheduler configures and starts a scheduler given a handle to the clientSet interface // and event broadcaster. It returns the running scheduler, podInformer and the shutdown function to stop it. -func StartScheduler(clientSet clientset.Interface, kubeConfig *restclient.Config, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) { - ctx, cancel := context.WithCancel(context.Background()) +func StartScheduler(tl ktesting.TL, clientSet clientset.Interface, kubeConfig *restclient.Config, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) { + _, ctx := ktesting.NewTestContext(tl) + ctx, cancel := context.WithCancel(ctx) informerFactory := scheduler.NewInformerFactory(clientSet, 0) evtBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{ @@ -89,11 +91,11 @@ func StartScheduler(clientSet clientset.Interface, kubeConfig *restclient.Config evtBroadcaster.StartRecordingToSink(ctx.Done()) sched, err := scheduler.New( + ctx, clientSet, informerFactory, nil, profile.NewRecorderFactory(evtBroadcaster), - ctx.Done(), scheduler.WithKubeConfig(kubeConfig), scheduler.WithProfiles(cfg.Profiles...), scheduler.WithPercentageOfNodesToScore(cfg.PercentageOfNodesToScore), @@ -405,11 +407,11 @@ func InitTestSchedulerWithOptions( opts = append(opts, scheduler.WithKubeConfig(testCtx.KubeConfig)) testCtx.Scheduler, err = scheduler.New( + testCtx.Ctx, testCtx.ClientSet, testCtx.InformerFactory, testCtx.DynInformerFactory, profile.NewRecorderFactory(eventBroadcaster), - testCtx.Ctx.Done(), opts..., )