Skip to content

Commit

Permalink
Add CrossNamespacePodAffinity quota scope and PodAffinityTerm.Namespa…
Browse files Browse the repository at this point in the history 
…ceSelector APIs, and CrossNamespacePodAffinity quota scope implementation.
  • Loading branch information
@ahg-g
ahg-g committed Mar 4, 2021
1 parent 4f93175 commit 3c5f018
Show file tree
Hide file tree
Showing 85 changed files with 11,347 additions and 3,565 deletions.
6 changes: 5 additions & 1 deletion api/openapi-spec/swagger.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

61 changes: 61 additions & 0 deletions pkg/api/pod/util.go
View file
Expand Up @@ -523,6 +523,7 @@ func dropDisabledFields(
podSpec.SetHostnameAsFQDN = nil
}

dropDisabledPodAffinityTermFields(podSpec, oldPodSpec)
}

// dropDisabledProcMountField removes disabled fields from PodSpec related
Expand Down Expand Up @@ -572,6 +573,66 @@ func dropDisabledEphemeralVolumeSourceAlphaFields(podSpec, oldPodSpec *api.PodSp
}
}

func dropPodAffinityTermNamespaceSelector(terms []api.PodAffinityTerm) {
for i := range terms {
terms[i].NamespaceSelector = nil
}
}

func dropWeightedPodAffinityTermNamespaceSelector(terms []api.WeightedPodAffinityTerm) {
for i := range terms {
terms[i].PodAffinityTerm.NamespaceSelector = nil
}
}

// dropDisabledPodAffinityTermFields removes disabled fields from PodSpec related
// to PodAffinityTerm only if it is not already used by the old spec
func dropDisabledPodAffinityTermFields(podSpec, oldPodSpec *api.PodSpec) {
if !utilfeature.DefaultFeatureGate.Enabled(features.PodAffinityNamespaceSelector) &&
podSpec != nil && podSpec.Affinity != nil &&
!podAffinityNamespaceSelectorInUse(oldPodSpec) {
if podSpec.Affinity.PodAffinity != nil {
dropPodAffinityTermNamespaceSelector(podSpec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
dropWeightedPodAffinityTermNamespaceSelector(podSpec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution)
}
if podSpec.Affinity.PodAntiAffinity != nil {
dropPodAffinityTermNamespaceSelector(podSpec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
dropWeightedPodAffinityTermNamespaceSelector(podSpec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution)
}
}
}

func podAffinityNamespaceSelectorInUse(podSpec *api.PodSpec) bool {
if podSpec == nil || podSpec.Affinity == nil {
return false
}
if podSpec.Affinity.PodAffinity != nil {
for _, t := range podSpec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution {
if t.NamespaceSelector != nil {
return true
}
}
for _, t := range podSpec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
if t.PodAffinityTerm.NamespaceSelector != nil {
return true
}
}
}
if podSpec.Affinity.PodAntiAffinity != nil {
for _, t := range podSpec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution {
if t.NamespaceSelector != nil {
return true
}
}
for _, t := range podSpec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
if t.PodAffinityTerm.NamespaceSelector == nil {
return true
}
}
}
return false
}

func ephemeralContainersInUse(podSpec *api.PodSpec) bool {
if podSpec == nil {
return false
Expand Down
156 changes: 156 additions & 0 deletions pkg/api/pod/util_test.go
View file
Expand Up @@ -1375,7 +1375,163 @@ func TestValidatePodDeletionCostOption(t *testing.T) {
if tc.wantAllowInvalidPodDeletionCost != gotOptions.AllowInvalidPodDeletionCost {
t.Errorf("unexpected diff, want: %v, got: %v", tc.wantAllowInvalidPodDeletionCost, gotOptions.AllowInvalidPodDeletionCost)
}
})
}
}

func TestDropDisabledPodAffinityTermFields(t *testing.T) {
testCases := []struct {
name string
enabled bool
podSpec *api.PodSpec
oldPodSpec *api.PodSpec
wantPodSpec *api.PodSpec
}{
{
name: "nil affinity",
podSpec: &api.PodSpec{},
wantPodSpec: &api.PodSpec{},
},
{
name: "empty affinity",
podSpec: &api.PodSpec{Affinity: &api.Affinity{}},
wantPodSpec: &api.PodSpec{Affinity: &api.Affinity{}},
},
{
name: "NamespaceSelector cleared",
podSpec: &api.PodSpec{Affinity: &api.Affinity{
PodAffinity: &api.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns1"}, TopologyKey: "region1", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns2"}, TopologyKey: "region2", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
PodAntiAffinity: &api.PodAntiAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns3"}, TopologyKey: "region3", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns4"}, TopologyKey: "region4", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
}},
oldPodSpec: &api.PodSpec{Affinity: &api.Affinity{}},
wantPodSpec: &api.PodSpec{Affinity: &api.Affinity{
PodAffinity: &api.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns1"}, TopologyKey: "region1"},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns2"}, TopologyKey: "region2"}},
},
},
PodAntiAffinity: &api.PodAntiAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns3"}, TopologyKey: "region3"},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns4"}, TopologyKey: "region4"}},
},
},
}},
},
{
name: "NamespaceSelector not cleared since old spec already sets it",
podSpec: &api.PodSpec{Affinity: &api.Affinity{
PodAffinity: &api.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns1"}, TopologyKey: "region1", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns2"}, TopologyKey: "region2", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
PodAntiAffinity: &api.PodAntiAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns3"}, TopologyKey: "region3", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns4"}, TopologyKey: "region4", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
}},
oldPodSpec: &api.PodSpec{Affinity: &api.Affinity{
PodAffinity: &api.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns1"}, TopologyKey: "region1", NamespaceSelector: &metav1.LabelSelector{}},
},
},
}},
wantPodSpec: &api.PodSpec{Affinity: &api.Affinity{
PodAffinity: &api.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns1"}, TopologyKey: "region1", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns2"}, TopologyKey: "region2", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
PodAntiAffinity: &api.PodAntiAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns3"}, TopologyKey: "region3", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns4"}, TopologyKey: "region4", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
}},
},
{
name: "NamespaceSelector not cleared since feature is enabled",
enabled: true,
podSpec: &api.PodSpec{Affinity: &api.Affinity{
PodAffinity: &api.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns1"}, TopologyKey: "region1", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns2"}, TopologyKey: "region2", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
PodAntiAffinity: &api.PodAntiAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns3"}, TopologyKey: "region3", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns4"}, TopologyKey: "region4", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
}},
wantPodSpec: &api.PodSpec{Affinity: &api.Affinity{
PodAffinity: &api.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns1"}, TopologyKey: "region1", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns2"}, TopologyKey: "region2", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
PodAntiAffinity: &api.PodAntiAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns3"}, TopologyKey: "region3", NamespaceSelector: &metav1.LabelSelector{}},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{
{PodAffinityTerm: api.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{}, Namespaces: []string{"ns4"}, TopologyKey: "region4", NamespaceSelector: &metav1.LabelSelector{}}},
},
},
}},
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodAffinityNamespaceSelector, tc.enabled)()
dropDisabledPodAffinityTermFields(tc.podSpec, tc.oldPodSpec)
if diff := cmp.Diff(tc.wantPodSpec, tc.podSpec); diff != "" {
t.Errorf("unexpected pod spec (-want, +got):\n%s", diff)
}
})
}
}
8 changes: 5 additions & 3 deletions pkg/apis/core/helper/helpers.go
View file
Expand Up @@ -108,8 +108,9 @@ var standardResourceQuotaScopes = sets.NewString(
)

// IsStandardResourceQuotaScope returns true if the scope is a standard value
func IsStandardResourceQuotaScope(str string) bool {
return standardResourceQuotaScopes.Has(str)
func IsStandardResourceQuotaScope(str string, allowNamespaceAffinityScope bool) bool {
return standardResourceQuotaScopes.Has(str) ||
(allowNamespaceAffinityScope && str == string(core.ResourceQuotaScopeCrossNamespacePodAffinity))
}

var podObjectCountQuotaResources = sets.NewString(
Expand All @@ -128,7 +129,8 @@ var podComputeQuotaResources = sets.NewString(
// IsResourceQuotaScopeValidForResource returns true if the resource applies to the specified scope
func IsResourceQuotaScopeValidForResource(scope core.ResourceQuotaScope, resource string) bool {
switch scope {
case core.ResourceQuotaScopeTerminating, core.ResourceQuotaScopeNotTerminating, core.ResourceQuotaScopeNotBestEffort, core.ResourceQuotaScopePriorityClass:
case core.ResourceQuotaScopeTerminating, core.ResourceQuotaScopeNotTerminating, core.ResourceQuotaScopeNotBestEffort,
core.ResourceQuotaScopePriorityClass, core.ResourceQuotaScopeCrossNamespacePodAffinity:
return podObjectCountQuotaResources.Has(resource) || podComputeQuotaResources.Has(resource)
case core.ResourceQuotaScopeBestEffort:
return podObjectCountQuotaResources.Has(resource)
Expand Down
17 changes: 15 additions & 2 deletions pkg/apis/core/types.go
View file
Expand Up @@ -2553,8 +2553,10 @@ type PodAffinityTerm struct {
// A label query over a set of resources, in this case pods.
// +optional
LabelSelector *metav1.LabelSelector
// namespaces specifies which namespaces the labelSelector applies to (matches against);
// null or empty list means "this pod's namespace"
// namespaces specifies a static list of namespace names that the term applies to.
// The term is applied to the union of the namespaces listed in this field
// and the ones selected by namespaceSelector.
// null or empty namespaces list and null namespaceSelector means "this pod's namespace"
// +optional
Namespaces []string
// This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching
Expand All @@ -2563,6 +2565,14 @@ type PodAffinityTerm struct {
// selected pods is running.
// Empty topologyKey is not allowed.
TopologyKey string
// A label query over the set of namespaces that the term applies to.
// The term is applied to the union of the namespaces selected by this field
// and the ones listed in the namespaces field.
// null selector and null or empty namespaces list means "this pod's namespace".
// An empty selector ({}) matches all namespaces.
// This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled.
// +optional
NamespaceSelector *metav1.LabelSelector
}

// NodeAffinity is a group of node affinity scheduling rules.
Expand Down Expand Up @@ -4842,6 +4852,9 @@ const (
ResourceQuotaScopeNotBestEffort ResourceQuotaScope = "NotBestEffort"
// Match all pod objects that have priority class mentioned
ResourceQuotaScopePriorityClass ResourceQuotaScope = "PriorityClass"
// Match all pod objects that have cross-namespace pod (anti)affinity mentioned
// This is an alpha feature enabled by the PodAffinityNamespaceSelector feature flag.
ResourceQuotaScopeCrossNamespacePodAffinity ResourceQuotaScope = "CrossNamespacePodAffinity"
)

// ResourceQuotaSpec defines the desired hard limits to enforce for Quota
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/core/v1/zz_generated.conversion.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 3c5f018

Please sign in to comment.