-
Notifications
You must be signed in to change notification settings - Fork 177
/
placement.go
140 lines (124 loc) · 5.36 KB
/
placement.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
package storagecluster
import (
ocsv1 "github.com/red-hat-storage/ocs-operator/api/v1"
"github.com/red-hat-storage/ocs-operator/controllers/defaults"
rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// getPlacement returns placement configuration for ceph components with appropriate topology
func getPlacement(sc *ocsv1.StorageCluster, component string) rookCephv1.Placement {
placement := rookCephv1.Placement{}
in, ok := sc.Spec.Placement[rookCephv1.KeyType(component)]
if ok {
(&in).DeepCopyInto(&placement)
} else {
in := defaults.DaemonPlacements[component]
(&in).DeepCopyInto(&placement)
}
// ignore default PodAntiAffinity mon placement when arbiter is enabled
if component == "mon" && arbiterEnabled(sc) {
placement.PodAntiAffinity = &corev1.PodAntiAffinity{}
}
if component == "arbiter" {
if !sc.Spec.Arbiter.DisableMasterNodeToleration {
placement.Tolerations = append(placement.Tolerations, corev1.Toleration{
Key: "node-role.kubernetes.io/master",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoSchedule,
})
}
return placement
}
// If no placement is specified for the given component and the
// StorageCluster has no label selector, set the default node
// affinity.
if placement.NodeAffinity == nil && sc.Spec.LabelSelector == nil {
placement.NodeAffinity = defaults.DefaultNodeAffinity
}
// If the StorageCluster specifies a label selector, append it to the
// node affinity, creating it if it doesn't exist.
if sc.Spec.LabelSelector != nil {
reqs := convertLabelToNodeSelectorRequirements(*sc.Spec.LabelSelector)
if len(reqs) != 0 {
appendNodeRequirements(&placement, reqs...)
}
}
topologyMap := sc.Status.NodeTopologies
if topologyMap == nil {
return placement
}
topologyKey := getFailureDomain(sc)
topologyKey, _ = topologyMap.GetKeyValues(topologyKey)
if component == "mon" || component == "mds" || (component == "rgw" && getCephObjectStoreGatewayInstances(sc) > 1) {
if placement.PodAntiAffinity != nil {
if placement.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution != nil {
for i := range placement.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
placement.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[i].PodAffinityTerm.TopologyKey = topologyKey
}
}
if placement.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
for i := range placement.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution {
placement.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[i].TopologyKey = topologyKey
}
}
}
}
return placement
}
// convertLabelToNodeSelectorRequirements returns a NodeSelectorRequirement list from a given LabelSelector
func convertLabelToNodeSelectorRequirements(labelSelector metav1.LabelSelector) []corev1.NodeSelectorRequirement {
reqs := []corev1.NodeSelectorRequirement{}
for key, value := range labelSelector.MatchLabels {
req := corev1.NodeSelectorRequirement{}
req.Key = key
req.Operator = corev1.NodeSelectorOpIn
req.Values = append(req.Values, value)
reqs = append(reqs, req)
}
numIter := len(labelSelector.MatchExpressions)
for i := 0; i < numIter; i++ {
req := corev1.NodeSelectorRequirement{}
req.Key = labelSelector.MatchExpressions[i].Key
req.Operator = corev1.NodeSelectorOperator(labelSelector.MatchExpressions[i].Operator)
req.Values = labelSelector.MatchExpressions[i].Values
reqs = append(reqs, req)
}
return reqs
}
func appendNodeRequirements(placement *rookCephv1.Placement, reqs ...corev1.NodeSelectorRequirement) {
if placement.NodeAffinity == nil {
placement.NodeAffinity = &corev1.NodeAffinity{}
}
if placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{}
}
nodeSelector := placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution
if len(nodeSelector.NodeSelectorTerms) == 0 {
nodeSelector.NodeSelectorTerms = append(nodeSelector.NodeSelectorTerms, corev1.NodeSelectorTerm{})
}
nodeSelector.NodeSelectorTerms[0].MatchExpressions = append(nodeSelector.NodeSelectorTerms[0].MatchExpressions, reqs...)
}
// MatchingLabelsSelector filters the list/delete operation on the given label
// selector (or index in the case of cached lists). A struct is used because
// labels.Selector is an interface, which cannot be aliased.
type MatchingLabelsSelector struct {
labels.Selector
}
// ApplyToList applies this configuration to the given list options.
// This is implemented by MatchingLabelsSelector which implements ListOption interface.
func (m MatchingLabelsSelector) ApplyToList(opts *client.ListOptions) {
opts.LabelSelector = m
}
// setTopologyForAffinity assigns topology related values to the affinity placements
func setTopologyForAffinity(placement *rookCephv1.Placement, selectorValue string, topologyKey string) {
placement.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.TopologyKey = topologyKey
nodeZoneSelector := corev1.NodeSelectorRequirement{
Key: topologyKey,
Operator: corev1.NodeSelectorOpIn,
Values: []string{selectorValue},
}
appendNodeRequirements(placement, nodeZoneSelector)
}