Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding Topology Spread Constraints to Grafana CR #839

Merged
merged 2 commits into from Nov 17, 2022
Merged

Adding Topology Spread Constraints to Grafana CR #839

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e020fc
adding topology spread constraints to Grafana CR
celestialorb Oct 22, 2022
4ee4648
run make test to generate all files
Nov 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 10 additions & 9 deletions api/integreatly/v1alpha1/grafana_types.go
View file
Open in desktop
Expand Up @@ -107,15 +107,16 @@ type GrafanaDeployment struct {
Annotations map[string]string `json:"annotations,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
// +nullable
Replicas *int32 `json:"replicas,omitempty"`
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
Tolerations []v1.Toleration `json:"tolerations,omitempty"`
Affinity *v1.Affinity `json:"affinity,omitempty"`
SecurityContext *v1.PodSecurityContext `json:"securityContext,omitempty"`
ContainerSecurityContext *v1.SecurityContext `json:"containerSecurityContext,omitempty"`
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
EnvFrom []v1.EnvFromSource `json:"envFrom,omitempty"`
Env []v1.EnvVar `json:"env,omitempty"`
Replicas *int32 `json:"replicas,omitempty"`
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
Tolerations []v1.Toleration `json:"tolerations,omitempty"`
Affinity *v1.Affinity `json:"affinity,omitempty"`
TopologySpreadConstraints []v1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
SecurityContext *v1.PodSecurityContext `json:"securityContext,omitempty"`
ContainerSecurityContext *v1.SecurityContext `json:"containerSecurityContext,omitempty"`
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
EnvFrom []v1.EnvFromSource `json:"envFrom,omitempty"`
Env []v1.EnvVar `json:"env,omitempty"`
// +nullable
SkipCreateAdminAccount *bool `json:"skipCreateAdminAccount,omitempty"`
PriorityClassName string `json:"priorityClassName,omitempty"`
Expand Down
7 changes: 7 additions & 0 deletions api/integreatly/v1alpha1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

137 changes: 137 additions & 0 deletions config/crd/bases/integreatly.org_grafanas.yaml
View file
Open in desktop
Expand Up @@ -5201,6 +5201,143 @@ spec:
type: string
type: object
type: array
topologySpreadConstraints:
items:
description: TopologySpreadConstraint specifies how to spread
matching pods among the given topology.
properties:
labelSelector:
description: LabelSelector is used to find matching pods.
Pods that match this label selector are counted to determine
the number of pods in their corresponding topology domain.
properties:
matchExpressions:
description: matchExpressions is a list of label selector
requirements. The requirements are ANDed.
items:
description: A label selector requirement is a selector
that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector
applies to.
type: string
operator:
description: operator represents a key's relationship
to a set of values. Valid operators are In,
NotIn, Exists and DoesNotExist.
type: string
values:
description: values is an array of string values.
If the operator is In or NotIn, the values array
must be non-empty. If the operator is Exists
or DoesNotExist, the values array must be empty.
This array is replaced during a strategic merge
patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
matchLabels:
additionalProperties:
type: string
description: matchLabels is a map of {key,value} pairs.
A single {key,value} in the matchLabels map is equivalent
to an element of matchExpressions, whose key field
is "key", the operator is "In", and the values array
contains only "value". The requirements are ANDed.
type: object
type: object
maxSkew:
description: 'MaxSkew describes the degree to which pods
may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`,
it is the maximum permitted difference between the number
of matching pods in the target topology and the global
minimum. The global minimum is the minimum number of matching
pods in an eligible domain or zero if the number of eligible
domains is less than MinDomains. For example, in a 3-zone
cluster, MaxSkew is set to 1, and pods with the same labelSelector
spread as 2/2/1: In this case, the global minimum is 1.
| zone1 | zone2 | zone3 | | P P | P P | P | -
if MaxSkew is 1, incoming pod can only be scheduled to
zone3 to become 2/2/2; scheduling it onto zone1(zone2)
would make the ActualSkew(3-1) on zone1(zone2) violate
MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled
onto any zone. When `whenUnsatisfiable=ScheduleAnyway`,
it is used to give higher precedence to topologies that
satisfy it. It''s a required field. Default value is 1
and 0 is not allowed.'
format: int32
type: integer
minDomains:
description: "MinDomains indicates a minimum number of eligible
domains. When the number of eligible domains with matching
topology keys is less than minDomains, Pod Topology Spread
treats \"global minimum\" as 0, and then the calculation
of Skew is performed. And when the number of eligible
domains with matching topology keys equals or greater
than minDomains, this value has no effect on scheduling.
As a result, when the number of eligible domains is less
than minDomains, scheduler won't schedule more than maxSkew
Pods to those domains. If value is nil, the constraint
behaves as if MinDomains is equal to 1. Valid values are
integers greater than 0. When value is not nil, WhenUnsatisfiable
must be DoNotSchedule. \n For example, in a 3-zone cluster,
MaxSkew is set to 2, MinDomains is set to 5 and pods with
the same labelSelector spread as 2/2/2: | zone1 | zone2
| zone3 | | P P | P P | P P | The number of domains
is less than 5(MinDomains), so \"global minimum\" is treated
as 0. In this situation, new pod with the same labelSelector
cannot be scheduled, because computed skew will be 3(3
- 0) if new Pod is scheduled to any of the three zones,
it will violate MaxSkew. \n This is an alpha field and
requires enabling MinDomainsInPodTopologySpread feature
gate."
format: int32
type: integer
topologyKey:
description: TopologyKey is the key of node labels. Nodes
that have a label with this key and identical values are
considered to be in the same topology. We consider each
<key, value> as a "bucket", and try to put balanced number
of pods into each bucket. We define a domain as a particular
instance of a topology. Also, we define an eligible domain
as a domain whose nodes match the node selector. e.g.
If TopologyKey is "kubernetes.io/hostname", each Node
is a domain of that topology. And, if TopologyKey is "topology.kubernetes.io/zone",
each zone is a domain of that topology. It's a required
field.
type: string
whenUnsatisfiable:
description: 'WhenUnsatisfiable indicates how to deal with
a pod if it doesn''t satisfy the spread constraint. -
DoNotSchedule (default) tells the scheduler not to schedule
it. - ScheduleAnyway tells the scheduler to schedule the
pod in any location, but giving higher precedence to
topologies that would help reduce the skew. A constraint
is considered "Unsatisfiable" for an incoming pod if and
only if every possible node assignment for that pod would
violate "MaxSkew" on some topology. For example, in a
3-zone cluster, MaxSkew is set to 1, and pods with the
same labelSelector spread as 3/1/1: | zone1 | zone2 |
zone3 | | P P P | P | P | If WhenUnsatisfiable
is set to DoNotSchedule, incoming pod can only be scheduled
to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1)
on zone2(zone3) satisfies MaxSkew(1). In other words,
the cluster can still be imbalanced, but scheduler won''t
make it *more* imbalanced. It''s a required field.'
type: string
required:
- maxSkew
- topologyKey
- whenUnsatisfiable
type: object
type: array
type: object
ingress:
description: GrafanaIngress provides a means to configure the ingress
Expand Down
8 changes: 8 additions & 0 deletions controllers/model/grafanaDeployment.go
View file
Open in desktop
Expand Up @@ -195,6 +195,13 @@ func getPodPriorityClassName(cr *v1alpha1.Grafana) string {
return ""
}

func getTopologySpreadConstraints(cr *v1alpha1.Grafana) []v13.TopologySpreadConstraint {
if cr.Spec.Deployment != nil && cr.Spec.Deployment.TopologySpreadConstraints != nil {
return cr.Spec.Deployment.TopologySpreadConstraints
}
return []v13.TopologySpreadConstraint{}
}

func getTolerations(cr *v1alpha1.Grafana) []v13.Toleration {
tolerations := []v13.Toleration{}

Expand Down Expand Up @@ -700,6 +707,7 @@ func getDeploymentSpec(cr *v1alpha1.Grafana, annotations map[string]string, conf
ServiceAccountName: constants.GrafanaServiceAccountName,
TerminationGracePeriodSeconds: getTerminationGracePeriod(cr),
PriorityClassName: getPodPriorityClassName(cr),
TopologySpreadConstraints: getTopologySpreadConstraints(cr),
},
},
Strategy: getDeploymentStrategy(cr),
Expand Down
53 changes: 53 additions & 0 deletions controllers/model/grafanaDeployment_test.go
View file
Open in desktop
Expand Up @@ -202,3 +202,56 @@ func Test_getReadinessProbe(t *testing.T) {
assert.Equal(t, want, got)
})
}

func Test_getTopologySpreadConstraints(t *testing.T) {
t.Run("Default empty topology constraints", func(t *testing.T) {
cr := &v1alpha1.Grafana{
Spec: v1alpha1.GrafanaSpec{},
}

got := getTopologySpreadConstraints(cr)
want := []v1.TopologySpreadConstraint{}

assert.Equal(t, want, got)
})

t.Run("Specified empty constraints", func(t *testing.T) {
cr := &v1alpha1.Grafana{
Spec: v1alpha1.GrafanaSpec{
Deployment: &v1alpha1.GrafanaDeployment{
TopologySpreadConstraints: []v1.TopologySpreadConstraint{},
},
},
}

got := getTopologySpreadConstraints(cr)
want := []v1.TopologySpreadConstraint{}

assert.Equal(t, want, got)
})

t.Run("Specified non-empty constraints", func(t *testing.T) {
cr := &v1alpha1.Grafana{
Spec: v1alpha1.GrafanaSpec{
Deployment: &v1alpha1.GrafanaDeployment{
TopologySpreadConstraints: []v1.TopologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "topology.kubernetes.io/zone",
},
},
},
},
}

got := getTopologySpreadConstraints(cr)
want := []v1.TopologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "topology.kubernetes.io/zone",
},
}

assert.Equal(t, want, got)
})
}