From 420c5308bbf13eeb93cfab3319d37e977e970c12 Mon Sep 17 00:00:00 2001 From: Mike Dame Date: Mon, 11 Oct 2021 13:47:23 -0400 Subject: [PATCH] Add MultiPoint scheduler plugin config field --- .../app/options/options_test.go | 40 +-- cmd/kube-scheduler/app/server_test.go | 107 +++++- .../apis/config/testing/defaults/defaults.go | 63 +++- pkg/scheduler/apis/config/types.go | 3 + .../config/v1beta2/zz_generated.conversion.go | 6 + .../apis/config/v1beta3/default_plugins.go | 103 ++---- .../config/v1beta3/default_plugins_test.go | 333 +++++++++++------- pkg/scheduler/apis/config/v1beta3/defaults.go | 2 +- .../apis/config/v1beta3/defaults_test.go | 66 +--- .../config/v1beta3/zz_generated.conversion.go | 6 + .../apis/config/zz_generated.deepcopy.go | 1 + pkg/scheduler/framework/runtime/framework.go | 135 ++++++- .../framework/runtime/framework_test.go | 279 +++++++++++++++ pkg/scheduler/profile/profile.go | 23 +- pkg/scheduler/profile/profile_test.go | 20 +- .../kube-scheduler/config/v1beta2/types.go | 3 + .../config/v1beta2/zz_generated.deepcopy.go | 1 + .../kube-scheduler/config/v1beta3/types.go | 18 + .../config/v1beta3/zz_generated.deepcopy.go | 1 + test/integration/scheduler/framework_test.go | 5 + 20 files changed, 872 insertions(+), 343 deletions(-) diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index b545a4006b9c..c5d674322b50 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -690,24 +690,24 @@ profiles: { SchedulerName: "default-scheduler", Plugins: &kubeschedulerconfig.Plugins{ - QueueSort: defaults.PluginsV1beta3.QueueSort, - PreFilter: defaults.PluginsV1beta3.PreFilter, - Filter: defaults.PluginsV1beta3.Filter, - PostFilter: defaults.PluginsV1beta3.PostFilter, - PreScore: defaults.PluginsV1beta3.PreScore, - Score: defaults.PluginsV1beta3.Score, Reserve: kubeschedulerconfig.PluginSet{ Enabled: []kubeschedulerconfig.Plugin{ {Name: "foo"}, {Name: "bar"}, }, + Disabled: []kubeschedulerconfig.Plugin{ + {Name: names.VolumeBinding}, + }, }, PreBind: kubeschedulerconfig.PluginSet{ Enabled: []kubeschedulerconfig.Plugin{ {Name: "foo"}, }, + Disabled: []kubeschedulerconfig.Plugin{ + {Name: names.VolumeBinding}, + }, }, - Bind: defaults.PluginsV1beta3.Bind, + MultiPoint: defaults.PluginsV1beta3.MultiPoint, }, PluginConfig: []kubeschedulerconfig.PluginConfig{ { @@ -918,19 +918,15 @@ profiles: { SchedulerName: "foo-profile", Plugins: &kubeschedulerconfig.Plugins{ - QueueSort: defaults.PluginsV1beta3.QueueSort, - PreFilter: defaults.PluginsV1beta3.PreFilter, - Filter: defaults.PluginsV1beta3.Filter, - PostFilter: defaults.PluginsV1beta3.PostFilter, - PreScore: defaults.PluginsV1beta3.PreScore, - Score: defaults.PluginsV1beta3.Score, - Bind: defaults.PluginsV1beta3.Bind, - PreBind: defaults.PluginsV1beta3.PreBind, + MultiPoint: defaults.PluginsV1beta3.MultiPoint, Reserve: kubeschedulerconfig.PluginSet{ Enabled: []kubeschedulerconfig.Plugin{ {Name: "foo"}, {Name: names.VolumeBinding}, }, + Disabled: []kubeschedulerconfig.Plugin{ + {Name: names.VolumeBinding}, + }, }, }, PluginConfig: defaults.PluginConfigsV1beta3, @@ -938,14 +934,12 @@ profiles: { SchedulerName: "bar-profile", Plugins: &kubeschedulerconfig.Plugins{ - QueueSort: defaults.PluginsV1beta3.QueueSort, - PreFilter: defaults.PluginsV1beta3.PreFilter, - Filter: defaults.PluginsV1beta3.Filter, - PostFilter: defaults.PluginsV1beta3.PostFilter, - PreScore: defaults.PluginsV1beta3.PreScore, - Score: defaults.PluginsV1beta3.Score, - Bind: defaults.PluginsV1beta3.Bind, - Reserve: defaults.PluginsV1beta3.Reserve, + MultiPoint: defaults.PluginsV1beta3.MultiPoint, + PreBind: kubeschedulerconfig.PluginSet{ + Disabled: []kubeschedulerconfig.Plugin{ + {Name: names.VolumeBinding}, + }, + }, }, PluginConfig: []kubeschedulerconfig.PluginConfig{ { diff --git a/cmd/kube-scheduler/app/server_test.go b/cmd/kube-scheduler/app/server_test.go index c6b52185a231..3a8ef8e941fa 100644 --- a/cmd/kube-scheduler/app/server_test.go +++ b/cmd/kube-scheduler/app/server_test.go @@ -82,8 +82,46 @@ users: } // plugin config - pluginConfigFile := filepath.Join(tmpDir, "plugin.yaml") - if err := ioutil.WriteFile(pluginConfigFile, []byte(fmt.Sprintf(` + pluginConfigFilev1beta3 := filepath.Join(tmpDir, "pluginv1beta3.yaml") + if err := ioutil.WriteFile(pluginConfigFilev1beta3, []byte(fmt.Sprintf(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "%s" +profiles: +- plugins: + multiPoint: + enabled: + - name: DefaultBinder + - name: PrioritySort + - name: DefaultPreemption + - name: VolumeBinding + - name: NodeResourcesFit + - name: NodePorts + - name: InterPodAffinity + - name: TaintToleration + disabled: + - name: "*" + preFilter: + disabled: + - name: VolumeBinding + - name: InterPodAffinity + filter: + disabled: + - name: VolumeBinding + - name: InterPodAffinity + - name: TaintToleration + score: + disabled: + - name: VolumeBinding + - name: NodeResourcesFit +`, configKubeconfig)), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + // plugin config + pluginConfigFilev1beta2 := filepath.Join(tmpDir, "pluginv1beta2.yaml") + if err := ioutil.WriteFile(pluginConfigFilev1beta2, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1beta2 kind: KubeSchedulerConfiguration clientConnection: @@ -188,20 +226,19 @@ leaderElection: wantPlugins: map[string]*config.Plugins{ "default-scheduler": func() *config.Plugins { plugins := &config.Plugins{ - QueueSort: defaults.PluginsV1beta3.QueueSort, - PreFilter: defaults.PluginsV1beta3.PreFilter, - Filter: defaults.PluginsV1beta3.Filter, - PostFilter: defaults.PluginsV1beta3.PostFilter, - PreScore: defaults.PluginsV1beta3.PreScore, - Score: defaults.PluginsV1beta3.Score, - Bind: defaults.PluginsV1beta3.Bind, - PreBind: defaults.PluginsV1beta3.PreBind, - Reserve: defaults.PluginsV1beta3.Reserve, + QueueSort: defaults.ExpandedPluginsV1beta3.QueueSort, + PreFilter: defaults.ExpandedPluginsV1beta3.PreFilter, + Filter: defaults.ExpandedPluginsV1beta3.Filter, + PostFilter: defaults.ExpandedPluginsV1beta3.PostFilter, + PreScore: defaults.ExpandedPluginsV1beta3.PreScore, + Score: defaults.ExpandedPluginsV1beta3.Score, + Bind: defaults.ExpandedPluginsV1beta3.Bind, + PreBind: defaults.ExpandedPluginsV1beta3.PreBind, + Reserve: defaults.ExpandedPluginsV1beta3.Reserve, } plugins.PreScore.Enabled = append(plugins.PreScore.Enabled, config.Plugin{Name: names.SelectorSpread, Weight: 0}) plugins.Score.Enabled = append( plugins.Score.Enabled, - config.Plugin{Name: names.VolumeBinding, Weight: 1}, config.Plugin{Name: names.SelectorSpread, Weight: 1}, ) return plugins @@ -218,13 +255,53 @@ leaderElection: "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ - "default-scheduler": defaults.PluginsV1beta3, + "default-scheduler": defaults.ExpandedPluginsV1beta3, + }, + }, + { + name: "component configuration v1beta2", + flags: []string{ + "--config", pluginConfigFilev1beta2, + "--kubeconfig", configKubeconfig, + }, + wantPlugins: map[string]*config.Plugins{ + "default-scheduler": { + Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}}, + Filter: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + }, + }, + PreFilter: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + }, + }, + PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}}, + PreScore: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: "InterPodAffinity"}, + {Name: "TaintToleration"}, + }, + }, + QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}}, + Score: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: "InterPodAffinity", Weight: 1}, + {Name: "TaintToleration", Weight: 1}, + }, + }, + Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}}, + PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}}, + }, }, }, { - name: "component configuration", + name: "component configuration v1beta3", flags: []string{ - "--config", pluginConfigFile, + "--config", pluginConfigFilev1beta3, "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ diff --git a/pkg/scheduler/apis/config/testing/defaults/defaults.go b/pkg/scheduler/apis/config/testing/defaults/defaults.go index b441b8a5c9f1..07e0d3ca696e 100644 --- a/pkg/scheduler/apis/config/testing/defaults/defaults.go +++ b/pkg/scheduler/apis/config/testing/defaults/defaults.go @@ -151,8 +151,36 @@ var PluginConfigsV1beta2 = []config.PluginConfig{ }, } -// PluginsV1beta3 default set of v1beta3 plugins. +// PluginsV1beta3 is the set of default v1beta3 plugins (before MultiPoint expansion) var PluginsV1beta3 = &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.PrioritySort}, + {Name: names.NodeUnschedulable}, + {Name: names.NodeName}, + {Name: names.TaintToleration, Weight: 3}, + {Name: names.NodeAffinity, Weight: 2}, + {Name: names.NodePorts}, + {Name: names.NodeResourcesFit, Weight: 1}, + {Name: names.VolumeRestrictions}, + {Name: names.EBSLimits}, + {Name: names.GCEPDLimits}, + {Name: names.NodeVolumeLimits}, + {Name: names.AzureDiskLimits}, + {Name: names.VolumeBinding}, + {Name: names.VolumeZone}, + {Name: names.PodTopologySpread, Weight: 2}, + {Name: names.InterPodAffinity, Weight: 2}, + {Name: names.DefaultPreemption}, + {Name: names.NodeResourcesBalancedAllocation, Weight: 1}, + {Name: names.ImageLocality, Weight: 1}, + {Name: names.DefaultBinder}, + }, + }, +} + +// ExpandedPluginsV1beta3 default set of v1beta3 plugins after MultiPoint expansion +var ExpandedPluginsV1beta3 = &config.Plugins{ QueueSort: config.PluginSet{ Enabled: []config.Plugin{ {Name: names.PrioritySort}, @@ -160,13 +188,13 @@ var PluginsV1beta3 = &config.Plugins{ }, PreFilter: config.PluginSet{ Enabled: []config.Plugin{ - {Name: names.NodeResourcesFit}, + {Name: names.NodeAffinity}, {Name: names.NodePorts}, + {Name: names.NodeResourcesFit}, {Name: names.VolumeRestrictions}, + {Name: names.VolumeBinding}, {Name: names.PodTopologySpread}, {Name: names.InterPodAffinity}, - {Name: names.VolumeBinding}, - {Name: names.NodeAffinity}, }, }, Filter: config.PluginSet{ @@ -195,32 +223,37 @@ var PluginsV1beta3 = &config.Plugins{ }, PreScore: config.PluginSet{ Enabled: []config.Plugin{ - {Name: names.InterPodAffinity}, - {Name: names.PodTopologySpread}, {Name: names.TaintToleration}, {Name: names.NodeAffinity}, + {Name: names.PodTopologySpread}, + {Name: names.InterPodAffinity}, }, }, Score: config.PluginSet{ Enabled: []config.Plugin{ - {Name: names.NodeResourcesBalancedAllocation, Weight: 1}, - {Name: names.ImageLocality, Weight: 1}, - {Name: names.NodeResourcesFit, Weight: 1}, - // Weight is doubled because: + // Weight is tripled because: // - This is a score coming from user preference. - {Name: names.InterPodAffinity, Weight: 2}, + // - Usage of node tainting to group nodes in the cluster is increasing becoming a use-case + // for many user workloads + {Name: names.TaintToleration, Weight: 3}, // Weight is doubled because: // - This is a score coming from user preference. {Name: names.NodeAffinity, Weight: 2}, + {Name: names.NodeResourcesFit, Weight: 1}, + // Weight is tripled because: + // - This is a score coming from user preference. + // - Usage of node tainting to group nodes in the cluster is increasing becoming a use-case + // for many user workloads + {Name: names.VolumeBinding, Weight: 1}, // Weight is doubled because: // - This is a score coming from user preference. // - It makes its signal comparable to NodeResourcesLeastAllocated. {Name: names.PodTopologySpread, Weight: 2}, - // Weight is tripled because: + // Weight is doubled because: // - This is a score coming from user preference. - // - Usage of node tainting to group nodes in the cluster is increasing becoming a use-case - // for many user workloads - {Name: names.TaintToleration, Weight: 3}, + {Name: names.InterPodAffinity, Weight: 2}, + {Name: names.NodeResourcesBalancedAllocation, Weight: 1}, + {Name: names.ImageLocality, Weight: 1}, }, }, Reserve: config.PluginSet{ diff --git a/pkg/scheduler/apis/config/types.go b/pkg/scheduler/apis/config/types.go index 41fb7d92d418..75fdb1e23d2c 100644 --- a/pkg/scheduler/apis/config/types.go +++ b/pkg/scheduler/apis/config/types.go @@ -161,6 +161,9 @@ type Plugins struct { // PostBind is a list of plugins that should be invoked after a pod is successfully bound. PostBind PluginSet + + // MultiPoint is a simplified config field for enabling plugins for all valid extension points + MultiPoint PluginSet } // PluginSet specifies enabled and disabled plugins for an extension point. diff --git a/pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go b/pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go index 02e0251220d7..07764f9c6d83 100644 --- a/pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go +++ b/pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go @@ -757,6 +757,9 @@ func autoConvert_v1beta2_Plugins_To_config_Plugins(in *v1beta2.Plugins, out *con if err := Convert_v1beta2_PluginSet_To_config_PluginSet(&in.PostBind, &out.PostBind, s); err != nil { return err } + if err := Convert_v1beta2_PluginSet_To_config_PluginSet(&in.MultiPoint, &out.MultiPoint, s); err != nil { + return err + } return nil } @@ -799,6 +802,9 @@ func autoConvert_config_Plugins_To_v1beta2_Plugins(in *config.Plugins, out *v1be if err := Convert_config_PluginSet_To_v1beta2_PluginSet(&in.PostBind, &out.PostBind, s); err != nil { return err } + if err := Convert_config_PluginSet_To_v1beta2_PluginSet(&in.MultiPoint, &out.MultiPoint, s); err != nil { + return err + } return nil } diff --git a/pkg/scheduler/apis/config/v1beta3/default_plugins.go b/pkg/scheduler/apis/config/v1beta3/default_plugins.go index 3d4f35e2f164..9519eb3e6936 100644 --- a/pkg/scheduler/apis/config/v1beta3/default_plugins.go +++ b/pkg/scheduler/apis/config/v1beta3/default_plugins.go @@ -29,30 +29,15 @@ import ( // getDefaultPlugins returns the default set of plugins. func getDefaultPlugins() *v1beta3.Plugins { plugins := &v1beta3.Plugins{ - QueueSort: v1beta3.PluginSet{ + MultiPoint: v1beta3.PluginSet{ Enabled: []v1beta3.Plugin{ {Name: names.PrioritySort}, - }, - }, - PreFilter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.NodeResourcesFit}, - {Name: names.NodePorts}, - {Name: names.VolumeRestrictions}, - {Name: names.PodTopologySpread}, - {Name: names.InterPodAffinity}, - {Name: names.VolumeBinding}, - {Name: names.NodeAffinity}, - }, - }, - Filter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ {Name: names.NodeUnschedulable}, {Name: names.NodeName}, - {Name: names.TaintToleration}, - {Name: names.NodeAffinity}, + {Name: names.TaintToleration, Weight: pointer.Int32(3)}, + {Name: names.NodeAffinity, Weight: pointer.Int32(2)}, {Name: names.NodePorts}, - {Name: names.NodeResourcesFit}, + {Name: names.NodeResourcesFit, Weight: pointer.Int32(1)}, {Name: names.VolumeRestrictions}, {Name: names.EBSLimits}, {Name: names.GCEPDLimits}, @@ -60,57 +45,11 @@ func getDefaultPlugins() *v1beta3.Plugins { {Name: names.AzureDiskLimits}, {Name: names.VolumeBinding}, {Name: names.VolumeZone}, - {Name: names.PodTopologySpread}, - {Name: names.InterPodAffinity}, - }, - }, - PostFilter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ + {Name: names.PodTopologySpread, Weight: pointer.Int32(2)}, + {Name: names.InterPodAffinity, Weight: pointer.Int32(2)}, {Name: names.DefaultPreemption}, - }, - }, - PreScore: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.InterPodAffinity}, - {Name: names.PodTopologySpread}, - {Name: names.TaintToleration}, - {Name: names.NodeAffinity}, - }, - }, - Score: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)}, - {Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)}, - {Name: names.NodeResourcesFit, Weight: pointer.Int32Ptr(1)}, - // Weight is doubled because: - // - This is a score coming from user preference. - {Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(2)}, - // Weight is doubled because: - // - This is a score coming from user preference. - {Name: names.NodeAffinity, Weight: pointer.Int32Ptr(2)}, - // Weight is doubled because: - // - This is a score coming from user preference. - // - It makes its signal comparable to NodeResourcesFit.LeastAllocated. - {Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)}, - // Weight is tripled because: - // - This is a score coming from user preference. - // - Usage of node tainting to group nodes in the cluster is becoming a valid use-case more often - // for many user workloads - {Name: names.TaintToleration, Weight: pointer.Int32Ptr(3)}, - }, - }, - Reserve: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.VolumeBinding}, - }, - }, - PreBind: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.VolumeBinding}, - }, - }, - Bind: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ + {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)}, + {Name: names.ImageLocality, Weight: pointer.Int32(1)}, {Name: names.DefaultBinder}, }, }, @@ -121,18 +60,12 @@ func getDefaultPlugins() *v1beta3.Plugins { } func applyFeatureGates(config *v1beta3.Plugins) { - if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) { - config.Score.Enabled = append(config.Score.Enabled, v1beta3.Plugin{Name: names.VolumeBinding, Weight: pointer.Int32Ptr(1)}) - } - if !utilfeature.DefaultFeatureGate.Enabled(features.DefaultPodTopologySpread) { // When feature is enabled, the default spreading is done by // PodTopologySpread plugin, which is enabled by default. klog.InfoS("Registering SelectorSpread plugin") - s := v1beta3.Plugin{Name: names.SelectorSpread} - config.PreScore.Enabled = append(config.PreScore.Enabled, s) - s.Weight = pointer.Int32Ptr(1) - config.Score.Enabled = append(config.Score.Enabled, s) + s := v1beta3.Plugin{Name: names.SelectorSpread, Weight: pointer.Int32Ptr(1)} + config.MultiPoint.Enabled = append(config.MultiPoint.Enabled, s) } } @@ -142,6 +75,7 @@ func mergePlugins(defaultPlugins, customPlugins *v1beta3.Plugins) *v1beta3.Plugi return defaultPlugins } + defaultPlugins.MultiPoint = mergePluginSet(defaultPlugins.MultiPoint, customPlugins.MultiPoint) defaultPlugins.QueueSort = mergePluginSet(defaultPlugins.QueueSort, customPlugins.QueueSort) defaultPlugins.PreFilter = mergePluginSet(defaultPlugins.PreFilter, customPlugins.PreFilter) defaultPlugins.Filter = mergePluginSet(defaultPlugins.Filter, customPlugins.Filter) @@ -166,9 +100,22 @@ func mergePluginSet(defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3 enabledCustomPlugins := make(map[string]pluginIndex) // replacedPluginIndex is a set of index of plugins, which have replaced the default plugins. replacedPluginIndex := sets.NewInt() + var disabled []v1beta3.Plugin for _, disabledPlugin := range customPluginSet.Disabled { + // if the user is manually disabling any (or all, with "*") default plugins for an extension point, + // we need to track that so that the MultiPoint extension logic in the framework can know to skip + // inserting unspecified default plugins to this point. + disabled = append(disabled, v1beta3.Plugin{Name: disabledPlugin.Name}) disabledPlugins.Insert(disabledPlugin.Name) } + + // With MultiPoint, we may now have some disabledPlugins in the default registry + // For example, we enable PluginX with Filter+Score through MultiPoint but disable its Score plugin by default. + for _, disabledPlugin := range defaultPluginSet.Disabled { + disabled = append(disabled, v1beta3.Plugin{Name: disabledPlugin.Name}) + disabledPlugins.Insert(disabledPlugin.Name) + } + for index, enabledPlugin := range customPluginSet.Enabled { enabledCustomPlugins[enabledPlugin.Name] = pluginIndex{index, enabledPlugin} } @@ -197,5 +144,5 @@ func mergePluginSet(defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3 enabledPlugins = append(enabledPlugins, plugin) } } - return v1beta3.PluginSet{Enabled: enabledPlugins} + return v1beta3.PluginSet{Enabled: enabledPlugins, Disabled: disabled} } diff --git a/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go b/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go index 56cdbe366c30..4b73e21f631f 100644 --- a/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go +++ b/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go @@ -38,30 +38,15 @@ func TestApplyFeatureGates(t *testing.T) { { name: "Feature gates disabled", wantConfig: &v1beta3.Plugins{ - QueueSort: v1beta3.PluginSet{ + MultiPoint: v1beta3.PluginSet{ Enabled: []v1beta3.Plugin{ {Name: names.PrioritySort}, - }, - }, - PreFilter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.NodeResourcesFit}, - {Name: names.NodePorts}, - {Name: names.VolumeRestrictions}, - {Name: names.PodTopologySpread}, - {Name: names.InterPodAffinity}, - {Name: names.VolumeBinding}, - {Name: names.NodeAffinity}, - }, - }, - Filter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ {Name: names.NodeUnschedulable}, {Name: names.NodeName}, - {Name: names.TaintToleration}, - {Name: names.NodeAffinity}, + {Name: names.TaintToleration, Weight: pointer.Int32(3)}, + {Name: names.NodeAffinity, Weight: pointer.Int32(2)}, {Name: names.NodePorts}, - {Name: names.NodeResourcesFit}, + {Name: names.NodeResourcesFit, Weight: pointer.Int32(1)}, {Name: names.VolumeRestrictions}, {Name: names.EBSLimits}, {Name: names.GCEPDLimits}, @@ -69,46 +54,11 @@ func TestApplyFeatureGates(t *testing.T) { {Name: names.AzureDiskLimits}, {Name: names.VolumeBinding}, {Name: names.VolumeZone}, - {Name: names.PodTopologySpread}, - {Name: names.InterPodAffinity}, - }, - }, - PostFilter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ + {Name: names.PodTopologySpread, Weight: pointer.Int32(2)}, + {Name: names.InterPodAffinity, Weight: pointer.Int32(2)}, {Name: names.DefaultPreemption}, - }, - }, - PreScore: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.InterPodAffinity}, - {Name: names.PodTopologySpread}, - {Name: names.TaintToleration}, - {Name: names.NodeAffinity}, - }, - }, - Score: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)}, - {Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)}, - {Name: names.NodeResourcesFit, Weight: pointer.Int32Ptr(1)}, - {Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(2)}, - {Name: names.NodeAffinity, Weight: pointer.Int32Ptr(2)}, - {Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)}, - {Name: names.TaintToleration, Weight: pointer.Int32Ptr(3)}, - }, - }, - Reserve: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.VolumeBinding}, - }, - }, - PreBind: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.VolumeBinding}, - }, - }, - Bind: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ + {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)}, + {Name: names.ImageLocality, Weight: pointer.Int32(1)}, {Name: names.DefaultBinder}, }, }, @@ -120,30 +70,15 @@ func TestApplyFeatureGates(t *testing.T) { features.DefaultPodTopologySpread: false, }, wantConfig: &v1beta3.Plugins{ - QueueSort: v1beta3.PluginSet{ + MultiPoint: v1beta3.PluginSet{ Enabled: []v1beta3.Plugin{ {Name: names.PrioritySort}, - }, - }, - PreFilter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.NodeResourcesFit}, - {Name: names.NodePorts}, - {Name: names.VolumeRestrictions}, - {Name: names.PodTopologySpread}, - {Name: names.InterPodAffinity}, - {Name: names.VolumeBinding}, - {Name: names.NodeAffinity}, - }, - }, - Filter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ {Name: names.NodeUnschedulable}, {Name: names.NodeName}, - {Name: names.TaintToleration}, - {Name: names.NodeAffinity}, + {Name: names.TaintToleration, Weight: pointer.Int32(3)}, + {Name: names.NodeAffinity, Weight: pointer.Int32(2)}, {Name: names.NodePorts}, - {Name: names.NodeResourcesFit}, + {Name: names.NodeResourcesFit, Weight: pointer.Int32(1)}, {Name: names.VolumeRestrictions}, {Name: names.EBSLimits}, {Name: names.GCEPDLimits}, @@ -151,49 +86,13 @@ func TestApplyFeatureGates(t *testing.T) { {Name: names.AzureDiskLimits}, {Name: names.VolumeBinding}, {Name: names.VolumeZone}, - {Name: names.PodTopologySpread}, - {Name: names.InterPodAffinity}, - }, - }, - PostFilter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ + {Name: names.PodTopologySpread, Weight: pointer.Int32(2)}, + {Name: names.InterPodAffinity, Weight: pointer.Int32(2)}, {Name: names.DefaultPreemption}, - }, - }, - PreScore: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.InterPodAffinity}, - {Name: names.PodTopologySpread}, - {Name: names.TaintToleration}, - {Name: names.NodeAffinity}, - {Name: names.SelectorSpread}, - }, - }, - Score: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)}, - {Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)}, - {Name: names.NodeResourcesFit, Weight: pointer.Int32Ptr(1)}, - {Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(2)}, - {Name: names.NodeAffinity, Weight: pointer.Int32Ptr(2)}, - {Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)}, - {Name: names.TaintToleration, Weight: pointer.Int32Ptr(3)}, - {Name: names.SelectorSpread, Weight: pointer.Int32Ptr(1)}, - }, - }, - Reserve: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.VolumeBinding}, - }, - }, - PreBind: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.VolumeBinding}, - }, - }, - Bind: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ + {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)}, + {Name: names.ImageLocality, Weight: pointer.Int32(1)}, {Name: names.DefaultBinder}, + {Name: names.SelectorSpread, Weight: pointer.Int32(1)}, }, }, }, @@ -276,6 +175,9 @@ func TestMergePlugins(t *testing.T) { {Name: "CustomPlugin"}, {Name: "DefaultPlugin2"}, }, + Disabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin2"}, + }, }, }, }, @@ -308,6 +210,9 @@ func TestMergePlugins(t *testing.T) { {Name: "DefaultPlugin1"}, {Name: "DefaultPlugin2"}, }, + Disabled: []v1beta3.Plugin{ + {Name: "*"}, + }, }, }, }, @@ -338,6 +243,9 @@ func TestMergePlugins(t *testing.T) { {Name: "DefaultPlugin2"}, {Name: "DefaultPlugin1"}, }, + Disabled: []v1beta3.Plugin{ + {Name: "*"}, + }, }, }, }, @@ -451,12 +359,197 @@ func TestMergePlugins(t *testing.T) { }, }, }, + { + name: "Append custom MultiPoint plugin", + customPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "CustomPlugin"}, + }, + }, + }, + defaultPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin1"}, + {Name: "DefaultPlugin2"}, + }, + }, + }, + expectedPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin1"}, + {Name: "DefaultPlugin2"}, + {Name: "CustomPlugin"}, + }, + }, + }, + }, + { + name: "Append disabled Multipoint plugins", + customPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "CustomPlugin"}, + {Name: "CustomPlugin2"}, + }, + Disabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin2"}, + }, + }, + Score: v1beta3.PluginSet{ + Disabled: []v1beta3.Plugin{ + {Name: "CustomPlugin2"}, + }, + }, + }, + defaultPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin1"}, + {Name: "DefaultPlugin2"}, + }, + }, + }, + expectedPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin1"}, + {Name: "CustomPlugin"}, + {Name: "CustomPlugin2"}, + }, + Disabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin2"}, + }, + }, + Score: v1beta3.PluginSet{ + Disabled: []v1beta3.Plugin{ + {Name: "CustomPlugin2"}, + }, + }, + }, + }, + { + name: "override default MultiPoint plugins with custom value", + customPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin", Weight: pointer.Int32(5)}, + }, + }, + }, + defaultPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin"}, + }, + }, + }, + expectedPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin", Weight: pointer.Int32(5)}, + }, + }, + }, + }, + { + name: "disabled MultiPoint plugin in default set", + defaultPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin"}, + }, + Disabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin2"}, + }, + }, + }, + customPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "CustomPlugin"}, + }, + }, + }, + expectedPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin"}, + {Name: "CustomPlugin"}, + }, + Disabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin2"}, + }, + }, + }, + }, + { + name: "disabled MultiPoint plugin in default set for specific extension point", + defaultPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin"}, + }, + }, + Score: v1beta3.PluginSet{ + Disabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin2"}, + }, + }, + }, + customPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "CustomPlugin"}, + }, + }, + }, + expectedPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin"}, + {Name: "CustomPlugin"}, + }, + }, + Score: v1beta3.PluginSet{ + Disabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin2"}, + }, + }, + }, + }, + { + name: "multipoint with only disabled gets merged", + defaultPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Enabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin"}, + }, + }, + }, + customPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Disabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin"}, + }, + }, + }, + expectedPlugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Disabled: []v1beta3.Plugin{ + {Name: "DefaultPlugin"}, + }, + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.defaultPlugins = mergePlugins(test.defaultPlugins, test.customPlugins) - if d := cmp.Diff(test.expectedPlugins, test.defaultPlugins); d != "" { + gotPlugins := mergePlugins(test.defaultPlugins, test.customPlugins) + if d := cmp.Diff(test.expectedPlugins, gotPlugins); d != "" { t.Fatalf("plugins mismatch (-want +got):\n%s", d) } }) diff --git a/pkg/scheduler/apis/config/v1beta3/defaults.go b/pkg/scheduler/apis/config/v1beta3/defaults.go index cd2929a541b4..cc333e08667e 100644 --- a/pkg/scheduler/apis/config/v1beta3/defaults.go +++ b/pkg/scheduler/apis/config/v1beta3/defaults.go @@ -42,6 +42,7 @@ func pluginsNames(p *v1beta3.Plugins) []string { return nil } extensions := []v1beta3.PluginSet{ + p.MultiPoint, p.PreFilter, p.Filter, p.PostFilter, @@ -66,7 +67,6 @@ func pluginsNames(p *v1beta3.Plugins) []string { func setDefaults_KubeSchedulerProfile(prof *v1beta3.KubeSchedulerProfile) { // Set default plugins. prof.Plugins = mergePlugins(getDefaultPlugins(), prof.Plugins) - // Set default plugin configs. scheme := GetPluginArgConversionScheme() existingConfigs := sets.NewString() diff --git a/pkg/scheduler/apis/config/v1beta3/defaults_test.go b/pkg/scheduler/apis/config/v1beta3/defaults_test.go index d0cb48b40d58..27b53fcdb925 100644 --- a/pkg/scheduler/apis/config/v1beta3/defaults_test.go +++ b/pkg/scheduler/apis/config/v1beta3/defaults_test.go @@ -325,30 +325,15 @@ func TestSchedulerDefaults(t *testing.T) { { SchedulerName: pointer.StringPtr("custom-scheduler"), Plugins: &v1beta3.Plugins{ - QueueSort: v1beta3.PluginSet{ + MultiPoint: v1beta3.PluginSet{ Enabled: []v1beta3.Plugin{ {Name: names.PrioritySort}, - }, - }, - PreFilter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.NodeResourcesFit}, - {Name: names.NodePorts}, - {Name: names.VolumeRestrictions}, - {Name: names.PodTopologySpread}, - {Name: names.InterPodAffinity}, - {Name: names.VolumeBinding}, - {Name: names.NodeAffinity}, - }, - }, - Filter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ {Name: names.NodeUnschedulable}, {Name: names.NodeName}, - {Name: names.TaintToleration}, - {Name: names.NodeAffinity}, + {Name: names.TaintToleration, Weight: pointer.Int32(3)}, + {Name: names.NodeAffinity, Weight: pointer.Int32(2)}, {Name: names.NodePorts}, - {Name: names.NodeResourcesFit}, + {Name: names.NodeResourcesFit, Weight: pointer.Int32(1)}, {Name: names.VolumeRestrictions}, {Name: names.EBSLimits}, {Name: names.GCEPDLimits}, @@ -356,48 +341,21 @@ func TestSchedulerDefaults(t *testing.T) { {Name: names.AzureDiskLimits}, {Name: names.VolumeBinding}, {Name: names.VolumeZone}, - {Name: names.PodTopologySpread}, - {Name: names.InterPodAffinity}, - }, - }, - PostFilter: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ + {Name: names.PodTopologySpread, Weight: pointer.Int32(2)}, + {Name: names.InterPodAffinity, Weight: pointer.Int32(2)}, {Name: names.DefaultPreemption}, - }, - }, - PreScore: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.InterPodAffinity}, - {Name: names.PodTopologySpread}, - {Name: names.TaintToleration}, - {Name: names.NodeAffinity}, - }, - }, - Score: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)}, - {Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)}, - {Name: names.NodeResourcesFit, Weight: pointer.Int32Ptr(1)}, - {Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(2)}, - {Name: names.NodeAffinity, Weight: pointer.Int32Ptr(2)}, - {Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)}, - {Name: names.TaintToleration, Weight: pointer.Int32Ptr(3)}, - }, - }, - Reserve: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.VolumeBinding}, - }, - }, - PreBind: v1beta3.PluginSet{ - Enabled: []v1beta3.Plugin{ - {Name: names.VolumeBinding}, + {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)}, + {Name: names.ImageLocality, Weight: pointer.Int32(1)}, + {Name: names.DefaultBinder}, }, }, Bind: v1beta3.PluginSet{ Enabled: []v1beta3.Plugin{ {Name: "BarPlugin"}, }, + Disabled: []v1beta3.Plugin{ + {Name: names.DefaultBinder}, + }, }, }, PluginConfig: pluginConfigs, diff --git a/pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go b/pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go index 3bb0346405d9..6a854ad09c2c 100644 --- a/pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go +++ b/pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go @@ -747,6 +747,9 @@ func autoConvert_v1beta3_Plugins_To_config_Plugins(in *v1beta3.Plugins, out *con if err := Convert_v1beta3_PluginSet_To_config_PluginSet(&in.PostBind, &out.PostBind, s); err != nil { return err } + if err := Convert_v1beta3_PluginSet_To_config_PluginSet(&in.MultiPoint, &out.MultiPoint, s); err != nil { + return err + } return nil } @@ -789,6 +792,9 @@ func autoConvert_config_Plugins_To_v1beta3_Plugins(in *config.Plugins, out *v1be if err := Convert_config_PluginSet_To_v1beta3_PluginSet(&in.PostBind, &out.PostBind, s); err != nil { return err } + if err := Convert_config_PluginSet_To_v1beta3_PluginSet(&in.MultiPoint, &out.MultiPoint, s); err != nil { + return err + } return nil } diff --git a/pkg/scheduler/apis/config/zz_generated.deepcopy.go b/pkg/scheduler/apis/config/zz_generated.deepcopy.go index 804a71ed6d62..40ce4da58788 100644 --- a/pkg/scheduler/apis/config/zz_generated.deepcopy.go +++ b/pkg/scheduler/apis/config/zz_generated.deepcopy.go @@ -395,6 +395,7 @@ func (in *Plugins) DeepCopyInto(out *Plugins) { in.PreBind.DeepCopyInto(&out.PreBind) in.Bind.DeepCopyInto(&out.Bind) in.PostBind.DeepCopyInto(&out.PostBind) + in.MultiPoint.DeepCopyInto(&out.MultiPoint) return } diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 3d93aa9260be..477f7e9f70c5 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -289,22 +289,6 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, opts ...Opti return f, nil } - var totalPriority int64 - for _, e := range profile.Plugins.Score.Enabled { - // a weight of zero is not permitted, plugins can be disabled explicitly - // when configured. - f.scorePluginWeight[e.Name] = int(e.Weight) - if f.scorePluginWeight[e.Name] == 0 { - f.scorePluginWeight[e.Name] = 1 - } - - // Checks totalPriority against MaxTotalScore to avoid overflow - if int64(f.scorePluginWeight[e.Name])*framework.MaxNodeScore > framework.MaxTotalScore-totalPriority { - return nil, fmt.Errorf("total score of Score plugins could overflow") - } - totalPriority += int64(f.scorePluginWeight[e.Name]) * framework.MaxNodeScore - } - // get needed plugins from config pg := f.pluginsNeeded(profile.Plugins) @@ -346,12 +330,28 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, opts ...Opti fillEventToPluginMap(p, options.clusterEventMap) } + // initialize plugins per individual extension points for _, e := range f.getExtensionPoints(profile.Plugins) { if err := updatePluginList(e.slicePtr, *e.plugins, pluginsMap); err != nil { return nil, err } } + // initialize multiPoint plugins to their expanded extension points + if len(profile.Plugins.MultiPoint.Enabled) > 0 { + if err := f.expandMultiPointPlugins(profile, pluginsMap); err != nil { + return nil, err + } + } + + if len(f.queueSortPlugins) != 1 { + return nil, fmt.Errorf("one queue sort plugin required for profile with scheduler name %q", profile.SchedulerName) + } + + if err := getScoreWeights(f, pluginsMap, append(profile.Plugins.Score.Enabled, profile.Plugins.MultiPoint.Enabled...)); err != nil { + return nil, err + } + // Verifying the score weights again since Plugin.Name() could return a different // value from the one used in the configuration. for _, scorePlugin := range f.scorePlugins { @@ -384,6 +384,106 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, opts ...Opti return f, nil } +// getScoreWeights makes sure that, between MultiPoint-Score plugin weights and individual Score +// plugin weights there is not an overflow of MaxTotalScore. +func getScoreWeights(f *frameworkImpl, pluginsMap map[string]framework.Plugin, plugins []config.Plugin) error { + var totalPriority int64 + scorePlugins := reflect.ValueOf(&f.scorePlugins).Elem() + pluginType := scorePlugins.Type().Elem() + for _, e := range plugins { + pg := pluginsMap[e.Name] + if !reflect.TypeOf(pg).Implements(pluginType) { + continue + } + + // We append MultiPoint plugins to the list of Score plugins. So if this plugin has already been + // encountered, let the individual Score weight take precedence. + if _, ok := f.scorePluginWeight[e.Name]; ok { + continue + } + // a weight of zero is not permitted, plugins can be disabled explicitly + // when configured. + f.scorePluginWeight[e.Name] = int(e.Weight) + if f.scorePluginWeight[e.Name] == 0 { + f.scorePluginWeight[e.Name] = 1 + } + + // Checks totalPriority against MaxTotalScore to avoid overflow + if int64(f.scorePluginWeight[e.Name])*framework.MaxNodeScore > framework.MaxTotalScore-totalPriority { + return fmt.Errorf("total score of Score plugins could overflow") + } + totalPriority += int64(f.scorePluginWeight[e.Name]) * framework.MaxNodeScore + } + return nil +} + +func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerProfile, pluginsMap map[string]framework.Plugin) error { + // initialize MultiPoint plugins + for _, e := range f.getExtensionPoints(profile.Plugins) { + plugins := reflect.ValueOf(e.slicePtr).Elem() + pluginType := plugins.Type().Elem() + // build enabledSet of plugins already registered via normal extension points + // to check double registration + enabledSet := sets.NewString() + for _, plugin := range e.plugins.Enabled { + enabledSet.Insert(plugin.Name) + } + + disabledSet := sets.NewString() + for _, disabledPlugin := range e.plugins.Disabled { + disabledSet.Insert(disabledPlugin.Name) + } + if disabledSet.Has("*") { + klog.V(4).InfoS("all plugins disabled for extension point, skipping MultiPoint expansion", "extension", pluginType) + continue + } + + // track plugins enabled via multipoint separately from those enabled by specific extensions, + // so that we can distinguish between double-registration and explicit overrides + multiPointEnabled := sets.NewString() + + for _, ep := range profile.Plugins.MultiPoint.Enabled { + pg, ok := pluginsMap[ep.Name] + if !ok { + return fmt.Errorf("%s %q does not exist", pluginType.Name(), ep.Name) + } + + // if this plugin doesn't implement the type for the current extension we're trying to expand, skip + if !reflect.TypeOf(pg).Implements(pluginType) { + continue + } + + // a plugin that's enabled via MultiPoint can still be disabled for specific extension points + if disabledSet.Has(ep.Name) { + klog.V(4).InfoS("plugin disabled for extension point", "plugin", ep.Name, "extension", pluginType) + continue + } + + // if this plugin has already been enabled by the specific extension point, + // the user intent is to override the default plugin or make some other explicit setting. + // Either way, discard the MultiPoint value for this plugin. + // This maintains expected behavior for overriding default plugins (see https://github.com/kubernetes/kubernetes/pull/99582) + if enabledSet.Has(ep.Name) { + klog.InfoS("MultiPoint plugin is explicitly re-configured; overriding", "plugin", ep.Name) + continue + } + + // if this plugin is already registered via MultiPoint, then this is + // a double registration and an error in the config. + if multiPointEnabled.Has(ep.Name) { + return fmt.Errorf("plugin %q already registered as %q", ep.Name, pluginType.Name()) + } + + // we only need to update the multipoint set, since we already have the specific extension set from above + multiPointEnabled.Insert(ep.Name) + + newPlugins := reflect.Append(plugins, reflect.ValueOf(pg)) + plugins.Set(newPlugins) + } + } + return nil +} + func fillEventToPluginMap(p framework.Plugin, eventToPlugins map[framework.ClusterEvent]sets.String) { ext, ok := p.(framework.EnqueueExtensions) if !ok { @@ -1174,6 +1274,9 @@ func (f *frameworkImpl) pluginsNeeded(plugins *config.Plugins) map[string]config for _, e := range f.getExtensionPoints(plugins) { find(e.plugins) } + + // Parse MultiPoint separately since they are not returned by f.getExtensionPoints() + find(&plugins.MultiPoint) return pgMap } diff --git a/pkg/scheduler/framework/runtime/framework_test.go b/pkg/scheduler/framework/runtime/framework_test.go index 5cd301b0327c..755077b2b1c7 100644 --- a/pkg/scheduler/framework/runtime/framework_test.go +++ b/pkg/scheduler/framework/runtime/framework_test.go @@ -119,6 +119,10 @@ func (pl *TestScorePlugin) Name() string { return pl.name } +func (pl *TestScorePlugin) PreScore(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodes []*v1.Node) *framework.Status { + return framework.NewStatus(framework.Code(pl.inj.PreScoreStatus), "injected status") +} + func (pl *TestScorePlugin) Score(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) (int64, *framework.Status) { return setScoreRes(pl.inj) } @@ -490,6 +494,281 @@ func TestNewFrameworkErrors(t *testing.T) { } } +func TestNewFrameworkMultiPointExpansion(t *testing.T) { + tests := []struct { + name string + plugins *config.Plugins + wantPlugins *config.Plugins + wantErr string + }{ + { + name: "plugin expansion", + plugins: &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: testPlugin, Weight: 5}, + }, + }, + }, + wantPlugins: &config.Plugins{ + QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreScore: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Score: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin, Weight: 5}}}, + Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + }, + }, + { + name: "disable MultiPoint plugin at some extension points", + plugins: &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: testPlugin}, + }, + }, + PreScore: config.PluginSet{ + Disabled: []config.Plugin{ + {Name: testPlugin}, + }, + }, + Score: config.PluginSet{ + Disabled: []config.Plugin{ + {Name: testPlugin}, + }, + }, + }, + wantPlugins: &config.Plugins{ + QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + }, + }, + { + name: "Multiple MultiPoint plugins", + plugins: &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: testPlugin}, + {Name: scorePlugin1}, + }, + }, + }, + wantPlugins: &config.Plugins{ + QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreScore: config.PluginSet{Enabled: []config.Plugin{ + {Name: testPlugin}, + {Name: scorePlugin1}, + }}, + Score: config.PluginSet{Enabled: []config.Plugin{ + {Name: testPlugin, Weight: 1}, + {Name: scorePlugin1, Weight: 1}, + }}, + Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + }, + }, + { + name: "disable MultiPoint extension", + plugins: &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: testPlugin}, + {Name: scorePlugin1}, + }, + }, + PreScore: config.PluginSet{ + Disabled: []config.Plugin{ + {Name: "*"}, + }, + }, + }, + wantPlugins: &config.Plugins{ + QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Score: config.PluginSet{Enabled: []config.Plugin{ + {Name: testPlugin, Weight: 1}, + {Name: scorePlugin1, Weight: 1}, + }}, + Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + }, + }, + { + name: "Reorder MultiPoint plugins (specified extension takes precedence)", + plugins: &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: scoreWithNormalizePlugin1}, + {Name: testPlugin}, + {Name: scorePlugin1}, + }, + }, + Score: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: scorePlugin1}, + {Name: testPlugin}, + }, + }, + }, + wantPlugins: &config.Plugins{ + QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreScore: config.PluginSet{Enabled: []config.Plugin{ + {Name: testPlugin}, + {Name: scorePlugin1}, + }}, + Score: config.PluginSet{Enabled: []config.Plugin{ + {Name: scorePlugin1, Weight: 1}, + {Name: testPlugin, Weight: 1}, + {Name: scoreWithNormalizePlugin1, Weight: 1}, + }}, + Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + }, + }, + { + name: "Override MultiPoint plugins weights", + plugins: &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: testPlugin}, + {Name: scorePlugin1}, + }, + }, + Score: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: scorePlugin1, Weight: 5}, + {Name: testPlugin, Weight: 3}, + }, + }, + }, + wantPlugins: &config.Plugins{ + QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreScore: config.PluginSet{Enabled: []config.Plugin{ + {Name: testPlugin}, + {Name: scorePlugin1}, + }}, + Score: config.PluginSet{Enabled: []config.Plugin{ + {Name: scorePlugin1, Weight: 5}, + {Name: testPlugin, Weight: 3}, + }}, + Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}}, + }, + }, + { + name: "disable and enable MultiPoint plugins with '*'", + plugins: &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: queueSortPlugin}, + {Name: bindPlugin}, + {Name: scorePlugin1}, + }, + Disabled: []config.Plugin{ + {Name: "*"}, + }, + }, + }, + wantPlugins: &config.Plugins{ + QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: queueSortPlugin}}}, + PreScore: config.PluginSet{Enabled: []config.Plugin{ + {Name: scorePlugin1}, + }}, + Score: config.PluginSet{Enabled: []config.Plugin{ + {Name: scorePlugin1, Weight: 1}, + }}, + Bind: config.PluginSet{Enabled: []config.Plugin{{Name: bindPlugin}}}, + }, + }, + { + name: "disable and enable MultiPoint plugin by name", + plugins: &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: bindPlugin}, + {Name: queueSortPlugin}, + {Name: scorePlugin1}, + }, + Disabled: []config.Plugin{ + {Name: scorePlugin1}, + }, + }, + }, + wantPlugins: &config.Plugins{ + QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: queueSortPlugin}}}, + PreScore: config.PluginSet{Enabled: []config.Plugin{ + {Name: scorePlugin1}, + }}, + Score: config.PluginSet{Enabled: []config.Plugin{ + {Name: scorePlugin1, Weight: 1}, + }}, + Bind: config.PluginSet{Enabled: []config.Plugin{{Name: bindPlugin}}}, + }, + }, + { + name: "Expect 'already registered' error", + plugins: &config.Plugins{ + MultiPoint: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: testPlugin}, + {Name: testPlugin}, + }, + }, + }, + wantErr: "already registered", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fw, err := NewFramework(registry, &config.KubeSchedulerProfile{Plugins: tc.plugins}) + if (err != nil && tc.wantErr == "") || (err == nil && tc.wantErr != "") || (err != nil && !strings.Contains(err.Error(), tc.wantErr)) { + t.Errorf("Unexpected error, got %v, expect: %s", err, tc.wantErr) + } + if tc.wantErr == "" { + if diff := cmp.Diff(tc.wantPlugins, fw.ListPlugins()); diff != "" { + t.Errorf("Unexpected eventToPlugin map (-want,+got):%s", diff) + } + } + }) + } +} + // fakeNoopPlugin doesn't implement interface framework.EnqueueExtensions. type fakeNoopPlugin struct{} diff --git a/pkg/scheduler/profile/profile.go b/pkg/scheduler/profile/profile.go index 206b1a66c2f3..8fb331cd2a5d 100644 --- a/pkg/scheduler/profile/profile.go +++ b/pkg/scheduler/profile/profile.go @@ -20,7 +20,6 @@ package profile import ( "errors" "fmt" - "github.com/google/go-cmp/cmp" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" @@ -55,13 +54,13 @@ func NewMap(cfgs []config.KubeSchedulerProfile, r frameworkruntime.Registry, rec v := cfgValidator{m: m} for _, cfg := range cfgs { - if err := v.validate(cfg); err != nil { - return nil, err - } p, err := newProfile(cfg, r, recorderFact, opts...) if err != nil { return nil, fmt.Errorf("creating profile for scheduler name %s: %v", cfg.SchedulerName, err) } + if err := v.validate(cfg, p); err != nil { + return nil, err + } m[cfg.SchedulerName] = p } return m, nil @@ -86,20 +85,18 @@ type cfgValidator struct { queueSortArgs runtime.Object } -func (v *cfgValidator) validate(cfg config.KubeSchedulerProfile) error { - if len(cfg.SchedulerName) == 0 { +func (v *cfgValidator) validate(cfg config.KubeSchedulerProfile, f framework.Framework) error { + if len(f.ProfileName()) == 0 { return errors.New("scheduler name is needed") } if cfg.Plugins == nil { - return fmt.Errorf("plugins required for profile with scheduler name %q", cfg.SchedulerName) - } - if v.m[cfg.SchedulerName] != nil { - return fmt.Errorf("duplicate profile with scheduler name %q", cfg.SchedulerName) + return fmt.Errorf("plugins required for profile with scheduler name %q", f.ProfileName()) } - if len(cfg.Plugins.QueueSort.Enabled) != 1 { - return fmt.Errorf("one queue sort plugin required for profile with scheduler name %q", cfg.SchedulerName) + if v.m[f.ProfileName()] != nil { + return fmt.Errorf("duplicate profile with scheduler name %q", f.ProfileName()) } - queueSort := cfg.Plugins.QueueSort.Enabled[0].Name + + queueSort := f.ListPlugins().QueueSort.Enabled[0].Name var queueSortArgs runtime.Object for _, plCfg := range cfg.PluginConfig { if plCfg.Name == queueSort { diff --git a/pkg/scheduler/profile/profile_test.go b/pkg/scheduler/profile/profile_test.go index 1c884f28aea6..6716ec0a2882 100644 --- a/pkg/scheduler/profile/profile_test.go +++ b/pkg/scheduler/profile/profile_test.go @@ -31,10 +31,10 @@ import ( ) var fakeRegistry = frameworkruntime.Registry{ - "QueueSort": newFakePlugin, - "Bind1": newFakePlugin, - "Bind2": newFakePlugin, - "Another": newFakePlugin, + "QueueSort": newFakePlugin("QueueSort"), + "Bind1": newFakePlugin("Bind1"), + "Bind2": newFakePlugin("Bind2"), + "Another": newFakePlugin("Another"), } func TestNewMap(t *testing.T) { @@ -260,10 +260,12 @@ func TestNewMap(t *testing.T) { } } -type fakePlugin struct{} +type fakePlugin struct { + name string +} func (p *fakePlugin) Name() string { - return "" + return p.name } func (p *fakePlugin) Less(*framework.QueuedPodInfo, *framework.QueuedPodInfo) bool { @@ -274,8 +276,10 @@ func (p *fakePlugin) Bind(context.Context, *framework.CycleState, *v1.Pod, strin return nil } -func newFakePlugin(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) { - return &fakePlugin{}, nil +func newFakePlugin(name string) func(object runtime.Object, handle framework.Handle) (framework.Plugin, error) { + return func(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) { + return &fakePlugin{name: name}, nil + } } func nilRecorderFactory(_ string) events.EventRecorder { diff --git a/staging/src/k8s.io/kube-scheduler/config/v1beta2/types.go b/staging/src/k8s.io/kube-scheduler/config/v1beta2/types.go index 2bbd3a1d6eab..d1d1d1a595eb 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1beta2/types.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1beta2/types.go @@ -191,6 +191,9 @@ type Plugins struct { // PostBind is a list of plugins that should be invoked after a pod is successfully bound. PostBind PluginSet `json:"postBind,omitempty"` + + // MultiPoint is a simplified config section to enable plugins for all valid extension points. + MultiPoint PluginSet `json:"multiPoint,omitempty"` } // PluginSet specifies enabled and disabled plugins for an extension point. diff --git a/staging/src/k8s.io/kube-scheduler/config/v1beta2/zz_generated.deepcopy.go b/staging/src/k8s.io/kube-scheduler/config/v1beta2/zz_generated.deepcopy.go index b64b030c42fc..3b787773bf67 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1beta2/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1beta2/zz_generated.deepcopy.go @@ -452,6 +452,7 @@ func (in *Plugins) DeepCopyInto(out *Plugins) { in.PreBind.DeepCopyInto(&out.PreBind) in.Bind.DeepCopyInto(&out.Bind) in.PostBind.DeepCopyInto(&out.PostBind) + in.MultiPoint.DeepCopyInto(&out.MultiPoint) return } diff --git a/staging/src/k8s.io/kube-scheduler/config/v1beta3/types.go b/staging/src/k8s.io/kube-scheduler/config/v1beta3/types.go index f4400ae4d2f5..4ab11007c233 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1beta3/types.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1beta3/types.go @@ -184,6 +184,24 @@ type Plugins struct { // PostBind is a list of plugins that should be invoked after a pod is successfully bound. PostBind PluginSet `json:"postBind,omitempty"` + + // MultiPoint is a simplified config section to enable plugins for all valid extension points. + // Plugins enabled through MultiPoint will automatically register for every individual extension + // point the plugin has implemented. Disabling a plugin through MultiPoint disables that behavior. + // The same is true for disabling "*" through MultiPoint (no default plugins will be automatically registered). + // Plugins can still be disabled through their individual extension points. + // + // In terms of precedence, plugin config follows this basic hierarchy + // 1. Specific extension points + // 2. Explicitly configured MultiPoint plugins + // 3. The set of default plugins, as MultiPoint plugins + // This implies that a higher precedence plugin will run first and overwrite any settings within MultiPoint. + // Explicitly user-configured plugins also take a higher precedence over default plugins. + // Within this hierarchy, an Enabled setting takes precedence over Disabled. For example, if a plugin is + // set in both `multiPoint.Enabled` and `multiPoint.Disabled`, the plugin will be enabled. Similarly, + // including `multiPoint.Disabled = '*'` and `multiPoint.Enabled = pluginA` will still register that specific + // plugin through MultiPoint. This follows the same behavior as all other extension point configurations. + MultiPoint PluginSet `json:"multiPoint,omitempty"` } // PluginSet specifies enabled and disabled plugins for an extension point. diff --git a/staging/src/k8s.io/kube-scheduler/config/v1beta3/zz_generated.deepcopy.go b/staging/src/k8s.io/kube-scheduler/config/v1beta3/zz_generated.deepcopy.go index 4d2f51a1b465..f1a3e7b2991c 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1beta3/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1beta3/zz_generated.deepcopy.go @@ -442,6 +442,7 @@ func (in *Plugins) DeepCopyInto(out *Plugins) { in.PreBind.DeepCopyInto(&out.PreBind) in.Bind.DeepCopyInto(&out.Bind) in.PostBind.DeepCopyInto(&out.PostBind) + in.MultiPoint.DeepCopyInto(&out.MultiPoint) return } diff --git a/test/integration/scheduler/framework_test.go b/test/integration/scheduler/framework_test.go index 07a5b3f2f7a5..af7d14a27960 100644 --- a/test/integration/scheduler/framework_test.go +++ b/test/integration/scheduler/framework_test.go @@ -1259,6 +1259,11 @@ func TestBindPlugin(t *testing.T) { Profiles: []v1beta3.KubeSchedulerProfile{{ SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName), Plugins: &v1beta3.Plugins{ + MultiPoint: v1beta3.PluginSet{ + Disabled: []v1beta3.Plugin{ + {Name: defaultbinder.Name}, + }, + }, Reserve: v1beta3.PluginSet{ Enabled: []v1beta3.Plugin{{Name: reservePlugin.Name()}}, },