forked from red-hat-storage/ocs-operator
-
Notifications
You must be signed in to change notification settings - Fork 0
/
uninstall_reconciler.go
382 lines (328 loc) · 14.5 KB
/
uninstall_reconciler.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
package storagecluster
import (
"context"
"encoding/json"
"fmt"
nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1"
ocsv1 "github.com/red-hat-storage/ocs-operator/api/v1"
ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v1alpha1"
"github.com/red-hat-storage/ocs-operator/controllers/defaults"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
// CleanupPolicyType is a string representing cleanup policy
type CleanupPolicyType string
// UninstallModeType is a string representing cleanup mode, it decides whether the deletion is graceful or forced
type UninstallModeType string
const (
// CleanupPolicyAnnotation defines the cleanup policy for data and metadata during uninstall
CleanupPolicyAnnotation = "uninstall.ocs.openshift.io/cleanup-policy"
// CleanupPolicyDelete when set, modifies the cleanup policy for Rook to delete the DataDirHostPath on uninstall
CleanupPolicyDelete CleanupPolicyType = "delete"
// CleanupPolicyRetain when set, modifies the cleanup policy for Rook to not cleanup the DataDirHostPath and the disks on uninstall
CleanupPolicyRetain CleanupPolicyType = "retain"
// UninstallModeAnnotation defines the uninstall mode
UninstallModeAnnotation = "uninstall.ocs.openshift.io/mode"
// UninstallModeForced when set, sets the uninstall mode for Rook and Noobaa to forced.
UninstallModeForced UninstallModeType = "forced"
// UninstallModeGraceful when set, sets the uninstall mode for Rook and Noobaa to graceful.
UninstallModeGraceful UninstallModeType = "graceful"
)
// deleteNodeAffinityKeyFromNodes deletes the default NodeAffinityKey from the OCS nodes
// This is not used, yet.
//
// nolint:unused
func (r *StorageClusterReconciler) deleteNodeAffinityKeyFromNodes(sc *ocsv1.StorageCluster) (err error) {
// We should delete the label only when the StorageCluster is using the default NodeAffinityKey
if sc.Spec.LabelSelector == nil {
nodes, err := r.getStorageClusterEligibleNodes(sc)
if err != nil {
r.Log.Error(err, "Uninstall: Unable to obtain the list of nodes eligible for the Storage Cluster.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) //nolint:gosimple
return nil
}
for _, node := range nodes.Items {
r.Log.Info("Uninstall: Deleting OCS label from Node.", "Node", node.Name)
new := node.DeepCopy()
delete(new.ObjectMeta.Labels, defaults.NodeAffinityKey)
oldJSON, err := json.Marshal(node)
if err != nil {
r.Log.Error(err, "Uninstall: Unable to remove the NodeAffinityKey from the Node.", "Node", node.Name)
continue
}
newJSON, err := json.Marshal(new)
if err != nil {
r.Log.Error(err, "Uninstall: Unable to remove the NodeAffinityKey from the Node.", "Node", node.Name)
continue
}
patch, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, node)
if err != nil {
r.Log.Error(err, "Uninstall: Unable to remove the NodeAffinityKey from the Node.", "Node", node.Name)
continue
}
err = r.Client.Patch(context.TODO(), &node, client.RawPatch(types.StrategicMergePatchType, patch))
if err != nil {
r.Log.Error(err, "Uninstall: Unable to remove the NodeAffinityKey from the Node.", "Node", node.Name)
continue
}
}
}
return nil
}
// deleteNodeTaint deletes the default NodeTolerationKey from the OCS nodes
func (r *StorageClusterReconciler) deleteNodeTaint(sc *ocsv1.StorageCluster) (err error) {
nodes, err := r.getStorageClusterEligibleNodes(sc)
if err != nil {
r.Log.Error(err, "Uninstall: Unable to obtain the list of nodes eligible for the Storage Cluster.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) //nolint:gosimple
return nil
}
for _, node := range nodes.Items {
r.Log.Info("Uninstall: Deleting OCS NodeTolerationKey from the Node.", "Node", node.Name)
new := node.DeepCopy()
new.Spec.Taints = make([]corev1.Taint, 0)
for _, taint := range node.Spec.Taints {
if defaults.NodeTolerationKey == taint.Key {
continue
}
new.Spec.Taints = append(new.Spec.Taints, taint)
}
oldJSON, err := json.Marshal(node)
if err != nil {
r.Log.Error(err, "Uninstall: Unable to remove the NodeTolerationKey from the Node.", "Node", node.Name)
continue
}
newJSON, err := json.Marshal(new)
if err != nil {
r.Log.Error(err, "Uninstall: Unable to remove the NodeTolerationKey from the Node.", "Node", node.Name)
continue
}
patch, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, node)
if err != nil {
r.Log.Error(err, "Uninstall: Unable to remove the NodeTolerationKey from the Node.", "Node", node.Name)
continue
}
err = r.Client.Patch(context.TODO(), &node, client.RawPatch(types.StrategicMergePatchType, patch))
if err != nil {
r.Log.Error(err, "Uninstall: Unable to remove the NodeTolerationKey from the Node.", "Node", node.Name)
continue
}
}
return nil
}
// setRookUninstallandCleanupPolicy sets the uninstall mode and cleanup policy for rook based on the annotation on the StorageCluster
func (r *StorageClusterReconciler) setRookUninstallandCleanupPolicy(instance *ocsv1.StorageCluster, cephCluster *cephv1.CephCluster) (err error) {
var updateRequired bool
if v, found := instance.ObjectMeta.Annotations[CleanupPolicyAnnotation]; found {
if (v == string(CleanupPolicyDelete)) && (cephCluster.Spec.CleanupPolicy.Confirmation != cephv1.DeleteDataDirOnHostsConfirmation) {
cephCluster.Spec.CleanupPolicy.Confirmation = cephv1.DeleteDataDirOnHostsConfirmation
updateRequired = true
} else if (v == string(CleanupPolicyRetain)) && (cephCluster.Spec.CleanupPolicy.Confirmation != "") {
cephCluster.Spec.CleanupPolicy.Confirmation = ""
updateRequired = true
}
}
if v, found := instance.ObjectMeta.Annotations[UninstallModeAnnotation]; found {
if (v == string(UninstallModeForced)) && (!cephCluster.Spec.CleanupPolicy.AllowUninstallWithVolumes) {
cephCluster.Spec.CleanupPolicy.AllowUninstallWithVolumes = true
updateRequired = true
} else if (v == string(UninstallModeGraceful)) && (cephCluster.Spec.CleanupPolicy.AllowUninstallWithVolumes) {
cephCluster.Spec.CleanupPolicy.AllowUninstallWithVolumes = false
updateRequired = true
}
}
if updateRequired {
err := r.Client.Update(context.TODO(), cephCluster)
if err != nil {
return fmt.Errorf("Uninstall: Unable to update the cephCluster to set uninstall mode and/or cleanup policy: %v", err)
}
r.Log.Info("Uninstall: CephCluster uninstall mode and cleanup policy has been set.", "CephCluser", klog.KRef(cephCluster.Namespace, cephCluster.Name))
}
return nil
}
// setNoobaaUninstallMode sets the uninstall mode for Noobaa based on the annotation on the StorageCluster
func (r *StorageClusterReconciler) setNoobaaUninstallMode(sc *ocsv1.StorageCluster) error {
// Do this if Noobaa is being managed by the OCS operator
if sc.Spec.MultiCloudGateway != nil {
reconcileStrategy := ReconcileStrategy(sc.Spec.MultiCloudGateway.ReconcileStrategy)
if reconcileStrategy == ReconcileStrategyIgnore {
return nil
}
}
noobaa := &nbv1.NooBaa{}
var updateRequired bool
err := r.Client.Get(context.TODO(), types.NamespacedName{Name: "noobaa", Namespace: sc.Namespace}, noobaa)
if err != nil {
if errors.IsNotFound(err) {
r.Log.Info("Uninstall: NooBaa not found, can't set uninstall mode.", "Noobaa", klog.KRef(sc.Namespace, "noobaa"))
return nil
}
return fmt.Errorf("Uninstall: Error while getting NooBaa %v", err)
}
// Explicitly allow deletion of NooBaa CR
if !noobaa.Spec.CleanupPolicy.AllowNoobaaDeletion {
noobaa.Spec.CleanupPolicy.AllowNoobaaDeletion = true
updateRequired = true
}
// The CleanupPolicy attribute in the Noobaa spec decides the uninstall mode.
// Unlike the Rook CleanupPolicy which decides whether the data needs to be erased.
if v, found := sc.ObjectMeta.Annotations[UninstallModeAnnotation]; found {
if (v == string(UninstallModeForced)) && (noobaa.Spec.CleanupPolicy.Confirmation != nbv1.DeleteOBCConfirmation) {
noobaa.Spec.CleanupPolicy.Confirmation = nbv1.DeleteOBCConfirmation
updateRequired = true
} else if (v == string(UninstallModeGraceful)) && (noobaa.Spec.CleanupPolicy.Confirmation != "") {
noobaa.Spec.CleanupPolicy.Confirmation = ""
updateRequired = true
}
}
if updateRequired {
err = r.Client.Update(context.TODO(), noobaa)
if err != nil {
return fmt.Errorf("Uninstall: Unable to update NooBaa uninstall mode: %v", err)
}
r.Log.Info("Uninstall: NooBaa uninstall mode has been set.", "NooBaa", klog.KRef(noobaa.Namespace, noobaa.Name))
}
return nil
}
// reconcileUninstallAnnotations looks at the current uninstall annotations on the StorageCluster and sets defaults if none or unrecognized ones are set.
// It returns error and bool (true if storagecluster gets updated else false).
func (r *StorageClusterReconciler) reconcileUninstallAnnotations(sc *ocsv1.StorageCluster) (bool, error) {
var updateRequired bool
if v, found := sc.ObjectMeta.Annotations[UninstallModeAnnotation]; !found {
metav1.SetMetaDataAnnotation(&sc.ObjectMeta, string(UninstallModeAnnotation), string(UninstallModeGraceful))
r.Log.Info("Uninstall: Setting uninstall mode annotation to default.", "UninstallMode", UninstallModeGraceful)
updateRequired = true
} else if found && v != string(UninstallModeGraceful) && v != string(UninstallModeForced) {
// if wrong value found
metav1.SetMetaDataAnnotation(&sc.ObjectMeta, string(UninstallModeAnnotation), string(UninstallModeGraceful))
r.Log.Info("Uninstall: Found unrecognized uninstall mode annotation. Changing it to default.",
"CurrentUninstallMode", v, "DefaultUninstallMode", UninstallModeGraceful)
updateRequired = true
}
if v, found := sc.ObjectMeta.Annotations[CleanupPolicyAnnotation]; !found {
metav1.SetMetaDataAnnotation(&sc.ObjectMeta, string(CleanupPolicyAnnotation), string(CleanupPolicyDelete))
r.Log.Info("Uninstall: Setting uninstall cleanup policy annotation to default.", "CleanupPolicy", CleanupPolicyDelete)
updateRequired = true
} else if found && v != string(CleanupPolicyDelete) && v != string(CleanupPolicyRetain) {
// if wrong value found
metav1.SetMetaDataAnnotation(&sc.ObjectMeta, string(CleanupPolicyAnnotation), string(CleanupPolicyDelete))
r.Log.Info("Uninstall: Found unrecognized uninstall cleanup policy annotation.Changing it to default.",
"CurrentCleanupPolicy", v, "DefaultCleanupPolicy", CleanupPolicyDelete)
updateRequired = true
}
if updateRequired {
oldSc := ocsv1.StorageCluster{}
err := r.Client.Get(context.TODO(), types.NamespacedName{Name: sc.Name, Namespace: sc.Namespace}, &oldSc)
if err != nil {
r.Log.Error(err, "Uninstall: Failed to get StorageCluster.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name))
return false, err
}
sc.ObjectMeta.ResourceVersion = oldSc.ObjectMeta.ResourceVersion
if err := r.Client.Update(context.TODO(), sc); err != nil {
r.Log.Error(err, "Uninstall: Failed to update the StorageCluster with uninstall defaults.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name))
return false, err
}
r.Log.Info("Uninstall: Default uninstall annotations has been set on StorageCluster", "StorageCluster", klog.KRef(sc.Namespace, sc.Name))
return true, nil
}
return false, nil
}
// verifyNoStorageConsumerExist verifies there are no storageConsumers on the same namespace
func (r *StorageClusterReconciler) verifyNoStorageConsumerExist(instance *ocsv1.StorageCluster) error {
storageConsumers := &ocsv1alpha1.StorageConsumerList{}
err := r.Client.List(context.TODO(), storageConsumers, &client.ListOptions{Namespace: instance.Namespace})
if err != nil {
return err
}
if len(storageConsumers.Items) != 0 {
err = fmt.Errorf("Failed to cleanup provider resources. StorageConsumers are present in the %s namespace. "+
"Offboard all consumer clusters for the provider cleanup to proceed", instance.Namespace)
r.recorder.ReportIfNotPresent(instance, corev1.EventTypeWarning, "ProviderCleanup", err.Error())
r.Log.Error(err, "Waiting for all consumer clusters to offboard.")
return err
}
return nil
}
// deleteResources is the function where the storageClusterFinalizer is handled
// Every function that is called within this function should be idempotent
func (r *StorageClusterReconciler) deleteResources(sc *ocsv1.StorageCluster) (reconcile.Result, error) {
if IsOCSConsumerMode(sc) {
err := r.verifyNoStorageClassClaimsExist(sc)
if err != nil {
return reconcile.Result{}, err
}
}
// we do not check if it is a provider before checking storageConsumers CR because of corner case
// where user can mark instance.Spec.AllowRemoteStorageConsumers as false and mark CR for deletion immediately.
// Which will trigger a deletion without having instance.Spec.AllowRemoteStorageConsumers as true.
err := r.verifyNoStorageConsumerExist(sc)
if err != nil {
return reconcile.Result{}, err
}
cephCluster := &cephv1.CephCluster{}
err = r.Client.Get(context.TODO(), types.NamespacedName{Name: generateNameForCephCluster(sc), Namespace: sc.Namespace}, cephCluster)
if err != nil && !errors.IsNotFound(err) {
return reconcile.Result{}, err
}
if !errors.IsNotFound(err) && cephCluster.GetDeletionTimestamp().IsZero() {
err = r.setRookUninstallandCleanupPolicy(sc, cephCluster)
if err != nil {
return reconcile.Result{}, err
}
err = r.Client.Delete(context.TODO(), cephCluster)
if err != nil && !errors.IsNotFound(err) {
return reconcile.Result{}, err
}
}
err = r.setNoobaaUninstallMode(sc)
if err != nil {
return reconcile.Result{}, err
}
objs := []resourceManager{
&ocsExternalResources{},
&ocsProviderServer{},
&ocsNoobaaSystem{},
&ocsCephRGWRoutes{},
&ocsCephObjectStoreUsers{},
&ocsCephObjectStores{},
&ocsCephRbdMirrors{},
&ocsCephNFS{},
&ocsCephNFSService{},
&ocsCephFilesystems{},
&ocsCephBlockPools{},
&ocsSnapshotClass{},
&ocsStorageQuota{},
&ocsStorageClass{},
&ocsCephCluster{},
&ocsClusterClaim{},
&backingStorageClasses{},
}
for _, obj := range objs {
res, err := obj.ensureDeleted(r, sc)
if err != nil {
return reconcile.Result{}, err
} else if !res.IsZero() {
return res, nil
}
}
err = r.deleteNodeTaint(sc)
if err != nil {
return reconcile.Result{}, err
}
err = deleteKMSResources(r, sc)
if err != nil {
return reconcile.Result{}, err
}
// TODO: skip the deletion of these labels till we figure out a way to wait
// for the cleanup jobs
//err = r.deleteNodeAffinityKeyFromNodes(sc)
//if err != nil {
// return err
//}
return reconcile.Result{}, nil
}