From 2e020fcdcbad487881905301944d18e6136d4a97 Mon Sep 17 00:00:00 2001 From: celestialorb Date: Sat, 22 Oct 2022 13:24:19 -0700 Subject: [PATCH 1/2] adding topology spread constraints to Grafana CR --- api/integreatly/v1alpha1/grafana_types.go | 19 ++++---- controllers/model/grafanaDeployment.go | 8 ++++ controllers/model/grafanaDeployment_test.go | 53 +++++++++++++++++++++ 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/api/integreatly/v1alpha1/grafana_types.go b/api/integreatly/v1alpha1/grafana_types.go index 3231c804a..334639c65 100644 --- a/api/integreatly/v1alpha1/grafana_types.go +++ b/api/integreatly/v1alpha1/grafana_types.go @@ -107,15 +107,16 @@ type GrafanaDeployment struct { Annotations map[string]string `json:"annotations,omitempty"` Labels map[string]string `json:"labels,omitempty"` // +nullable - Replicas *int32 `json:"replicas,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Tolerations []v1.Toleration `json:"tolerations,omitempty"` - Affinity *v1.Affinity `json:"affinity,omitempty"` - SecurityContext *v1.PodSecurityContext `json:"securityContext,omitempty"` - ContainerSecurityContext *v1.SecurityContext `json:"containerSecurityContext,omitempty"` - TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` - EnvFrom []v1.EnvFromSource `json:"envFrom,omitempty"` - Env []v1.EnvVar `json:"env,omitempty"` + Replicas *int32 `json:"replicas,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + Affinity *v1.Affinity `json:"affinity,omitempty"` + TopologySpreadConstraints []v1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` + SecurityContext *v1.PodSecurityContext `json:"securityContext,omitempty"` + ContainerSecurityContext *v1.SecurityContext `json:"containerSecurityContext,omitempty"` + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` + EnvFrom []v1.EnvFromSource `json:"envFrom,omitempty"` + Env []v1.EnvVar `json:"env,omitempty"` // +nullable SkipCreateAdminAccount *bool `json:"skipCreateAdminAccount,omitempty"` PriorityClassName string `json:"priorityClassName,omitempty"` diff --git a/controllers/model/grafanaDeployment.go b/controllers/model/grafanaDeployment.go index 749f6e00f..6e7b1fe63 100644 --- a/controllers/model/grafanaDeployment.go +++ b/controllers/model/grafanaDeployment.go @@ -195,6 +195,13 @@ func getPodPriorityClassName(cr *v1alpha1.Grafana) string { return "" } +func getTopologySpreadConstraints(cr *v1alpha1.Grafana) []v13.TopologySpreadConstraint { + if cr.Spec.Deployment != nil && cr.Spec.Deployment.TopologySpreadConstraints != nil { + return cr.Spec.Deployment.TopologySpreadConstraints + } + return []v13.TopologySpreadConstraint{} +} + func getTolerations(cr *v1alpha1.Grafana) []v13.Toleration { tolerations := []v13.Toleration{} @@ -700,6 +707,7 @@ func getDeploymentSpec(cr *v1alpha1.Grafana, annotations map[string]string, conf ServiceAccountName: constants.GrafanaServiceAccountName, TerminationGracePeriodSeconds: getTerminationGracePeriod(cr), PriorityClassName: getPodPriorityClassName(cr), + TopologySpreadConstraints: getTopologySpreadConstraints(cr), }, }, Strategy: getDeploymentStrategy(cr), diff --git a/controllers/model/grafanaDeployment_test.go b/controllers/model/grafanaDeployment_test.go index ff5c21d01..f4fbd81b4 100644 --- a/controllers/model/grafanaDeployment_test.go +++ b/controllers/model/grafanaDeployment_test.go @@ -202,3 +202,56 @@ func Test_getReadinessProbe(t *testing.T) { assert.Equal(t, want, got) }) } + +func Test_getTopologySpreadConstraints(t *testing.T) { + t.Run("Default empty topology constraints", func(t *testing.T) { + cr := &v1alpha1.Grafana{ + Spec: v1alpha1.GrafanaSpec{}, + } + + got := getTopologySpreadConstraints(cr) + want := []v1.TopologySpreadConstraint{} + + assert.Equal(t, want, got) + }) + + t.Run("Specified empty constraints", func(t *testing.T) { + cr := &v1alpha1.Grafana{ + Spec: v1alpha1.GrafanaSpec{ + Deployment: &v1alpha1.GrafanaDeployment{ + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, + }, + }, + } + + got := getTopologySpreadConstraints(cr) + want := []v1.TopologySpreadConstraint{} + + assert.Equal(t, want, got) + }) + + t.Run("Specified non-empty constraints", func(t *testing.T) { + cr := &v1alpha1.Grafana{ + Spec: v1alpha1.GrafanaSpec{ + Deployment: &v1alpha1.GrafanaDeployment{ + TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "topology.kubernetes.io/zone", + }, + }, + }, + }, + } + + got := getTopologySpreadConstraints(cr) + want := []v1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "topology.kubernetes.io/zone", + }, + } + + assert.Equal(t, want, got) + }) +} From 4ee4648ec2c9ada523a778c4f85f72f243b0605c Mon Sep 17 00:00:00 2001 From: Edvin Norling Date: Thu, 17 Nov 2022 20:05:00 +0100 Subject: [PATCH 2/2] run make test to generate all files --- .../v1alpha1/zz_generated.deepcopy.go | 7 + .../crd/bases/integreatly.org_grafanas.yaml | 137 +++++++++++++++++ deploy/manifests/latest/crds.yaml | 137 +++++++++++++++++ documentation/api.md | 143 ++++++++++++++++++ 4 files changed, 424 insertions(+) diff --git a/api/integreatly/v1alpha1/zz_generated.deepcopy.go b/api/integreatly/v1alpha1/zz_generated.deepcopy.go index 90b12bd96..29ba46053 100644 --- a/api/integreatly/v1alpha1/zz_generated.deepcopy.go +++ b/api/integreatly/v1alpha1/zz_generated.deepcopy.go @@ -1863,6 +1863,13 @@ func (in *GrafanaDeployment) DeepCopyInto(out *GrafanaDeployment) { *out = new(v1.Affinity) (*in).DeepCopyInto(*out) } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.SecurityContext != nil { in, out := &in.SecurityContext, &out.SecurityContext *out = new(v1.PodSecurityContext) diff --git a/config/crd/bases/integreatly.org_grafanas.yaml b/config/crd/bases/integreatly.org_grafanas.yaml index 9083bfc0a..aae894be2 100644 --- a/config/crd/bases/integreatly.org_grafanas.yaml +++ b/config/crd/bases/integreatly.org_grafanas.yaml @@ -5201,6 +5201,143 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. The global minimum is the minimum number of matching + pods in an eligible domain or zero if the number of eligible + domains is less than MinDomains. For example, in a 3-zone + cluster, MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | | P P | P P | P | - + if MaxSkew is 1, incoming pod can only be scheduled to + zone3 to become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation + of Skew is performed. And when the number of eligible + domains with matching topology keys equals or greater + than minDomains, this value has no effect on scheduling. + As a result, when the number of eligible domains is less + than minDomains, scheduler won't schedule more than maxSkew + Pods to those domains. If value is nil, the constraint + behaves as if MinDomains is equal to 1. Valid values are + integers greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a 3-zone cluster, + MaxSkew is set to 2, MinDomains is set to 5 and pods with + the same labelSelector spread as 2/2/2: | zone1 | zone2 + | zone3 | | P P | P P | P P | The number of domains + is less than 5(MinDomains), so \"global minimum\" is treated + as 0. In this situation, new pod with the same labelSelector + cannot be scheduled, because computed skew will be 3(3 + - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. \n This is an alpha field and + requires enabling MinDomainsInPodTopologySpread feature + gate." + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. We define a domain as a particular + instance of a topology. Also, we define an eligible domain + as a domain whose nodes match the node selector. e.g. + If TopologyKey is "kubernetes.io/hostname", each Node + is a domain of that topology. And, if TopologyKey is "topology.kubernetes.io/zone", + each zone is a domain of that topology. It's a required + field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to + topologies that would help reduce the skew. A constraint + is considered "Unsatisfiable" for an incoming pod if and + only if every possible node assignment for that pod would + violate "MaxSkew" on some topology. For example, in a + 3-zone cluster, MaxSkew is set to 1, and pods with the + same labelSelector spread as 3/1/1: | zone1 | zone2 | + zone3 | | P P P | P | P | If WhenUnsatisfiable + is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) + on zone2(zone3) satisfies MaxSkew(1). In other words, + the cluster can still be imbalanced, but scheduler won''t + make it *more* imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array type: object ingress: description: GrafanaIngress provides a means to configure the ingress diff --git a/deploy/manifests/latest/crds.yaml b/deploy/manifests/latest/crds.yaml index cd92fa3a1..238e328fb 100644 --- a/deploy/manifests/latest/crds.yaml +++ b/deploy/manifests/latest/crds.yaml @@ -5942,6 +5942,143 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. The global minimum is the minimum number of matching + pods in an eligible domain or zero if the number of eligible + domains is less than MinDomains. For example, in a 3-zone + cluster, MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | | P P | P P | P | - + if MaxSkew is 1, incoming pod can only be scheduled to + zone3 to become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation + of Skew is performed. And when the number of eligible + domains with matching topology keys equals or greater + than minDomains, this value has no effect on scheduling. + As a result, when the number of eligible domains is less + than minDomains, scheduler won't schedule more than maxSkew + Pods to those domains. If value is nil, the constraint + behaves as if MinDomains is equal to 1. Valid values are + integers greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a 3-zone cluster, + MaxSkew is set to 2, MinDomains is set to 5 and pods with + the same labelSelector spread as 2/2/2: | zone1 | zone2 + | zone3 | | P P | P P | P P | The number of domains + is less than 5(MinDomains), so \"global minimum\" is treated + as 0. In this situation, new pod with the same labelSelector + cannot be scheduled, because computed skew will be 3(3 + - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. \n This is an alpha field and + requires enabling MinDomainsInPodTopologySpread feature + gate." + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. We define a domain as a particular + instance of a topology. Also, we define an eligible domain + as a domain whose nodes match the node selector. e.g. + If TopologyKey is "kubernetes.io/hostname", each Node + is a domain of that topology. And, if TopologyKey is "topology.kubernetes.io/zone", + each zone is a domain of that topology. It's a required + field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to + topologies that would help reduce the skew. A constraint + is considered "Unsatisfiable" for an incoming pod if and + only if every possible node assignment for that pod would + violate "MaxSkew" on some topology. For example, in a + 3-zone cluster, MaxSkew is set to 1, and pods with the + same labelSelector spread as 3/1/1: | zone1 | zone2 | + zone3 | | P P P | P | P | If WhenUnsatisfiable + is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) + on zone2(zone3) satisfies MaxSkew(1). In other words, + the cluster can still be imbalanced, but scheduler won''t + make it *more* imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array type: object ingress: description: GrafanaIngress provides a means to configure the ingress diff --git a/documentation/api.md b/documentation/api.md index bb072edf2..f493f276e 100644 --- a/documentation/api.md +++ b/documentation/api.md @@ -7819,6 +7819,13 @@ GrafanaDeployment provides a means to configure the deployment
false + + topologySpreadConstraints + []object + +
+ + false @@ -13070,6 +13077,142 @@ The pod this Toleration is attached to tolerates any taint that matches the trip +### Grafana.spec.deployment.topologySpreadConstraints[index] +[↩ Parent](#grafanaspecdeployment) + + + +TopologySpreadConstraint specifies how to spread matching pods among the given topology. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
maxSkewinteger + MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. The global minimum is the minimum number of matching pods in an eligible domain or zero if the number of eligible domains is less than MinDomains. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 2/2/1: In this case, the global minimum is 1. | zone1 | zone2 | zone3 | | P P | P P | P | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It's a required field. Default value is 1 and 0 is not allowed.
+
+ Format: int32
+ 		true
topologyKeystring + TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain whose nodes match the node selector. e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. It's a required field.
+ 		true
whenUnsatisfiablestring + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location, but giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for an incoming pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won't make it *more* imbalanced. It's a required field.
+ 		true
labelSelectorobject + LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.
+ 		false
minDomainsinteger + MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | | P P | P P | P P | The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. + This is an alpha field and requires enabling MinDomainsInPodTopologySpread feature gate.
+
+ Format: int32
+ 		false
+ + +### Grafana.spec.deployment.topologySpreadConstraints[index].labelSelector +[↩ Parent](#grafanaspecdeploymenttopologyspreadconstraintsindex) + + + +LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ 		false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ 		false
+ + +### Grafana.spec.deployment.topologySpreadConstraints[index].labelSelector.matchExpressions[index] +[↩ Parent](#grafanaspecdeploymenttopologyspreadconstraintsindexlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+ 		true
operatorstring + operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ 		true
values[]string + values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ 		false
+ + ### Grafana.spec.ingress [↩ Parent](#grafanaspec)