From a935e72e83b221d6596cc4b4282a33db41984325 Mon Sep 17 00:00:00 2001 From: bhatfiel Date: Wed, 4 Aug 2021 22:47:31 -0700 Subject: [PATCH] drpolicy controller define - vrg and pv cluster roles manifestworks unify - drpolicy create/update: - finalizer add if absent - cluster roles manifestwork create, if absent, for each cluster in drpolicy - drpolicy delete: - cluster roles manifestwork delete for each cluster not in any other drpolicy - finalizer remove if present - ginkgo version increase from 1.16.1 to 1.16.4 - gomega version increase from 1.11.0 to 1.15.0 Signed-off-by: bhatfiel --- config/rbac/role.yaml | 14 ++ config/samples/kustomization.yaml | 4 + controllers/drplacementcontrol_controller.go | 14 -- .../drplacementcontrol_controller_test.go | 79 ++++++----- controllers/drpolicy_controller.go | 124 +++++++++++++++++ controllers/drpolicy_controller_test.go | 117 ++++++++++++++++ controllers/suite_test.go | 6 + controllers/util/drpolicy_util.go | 9 ++ controllers/util/mw_util.go | 131 ++++++++++++------ go.mod | 4 +- go.sum | 26 ++-- main.go | 8 ++ 12 files changed, 431 insertions(+), 105 deletions(-) create mode 100644 config/samples/kustomization.yaml create mode 100644 controllers/drpolicy_controller.go create mode 100644 controllers/drpolicy_controller_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7b881390c..62a1cfaad 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -108,6 +108,20 @@ rules: - patch - update - watch +- apiGroups: + - ramendr.openshift.io + resources: + - drpolicies/finalizers + verbs: + - update +- apiGroups: + - ramendr.openshift.io + resources: + - drpolicies/status + verbs: + - get + - patch + - update - apiGroups: - ramendr.openshift.io resources: diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml new file mode 100644 index 000000000..e1f5e5abf --- /dev/null +++ b/config/samples/kustomization.yaml @@ -0,0 +1,4 @@ +## Append samples you want in your CSV to this file as resources ## +resources: +- ramendr_v1alpha1_drpolicy.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/controllers/drplacementcontrol_controller.go b/controllers/drplacementcontrol_controller.go index 46814391d..80ba9042f 100644 --- a/controllers/drplacementcontrol_controller.go +++ b/controllers/drplacementcontrol_controller.go @@ -1114,13 +1114,6 @@ func (d *DRPCInstance) updateUserPlRuleAndCreateVRGMW(homeCluster, homeClusterNa } func (d *DRPCInstance) createPVManifestWorkForRestore(newPrimary string) error { - if err := d.mwu.CreateOrUpdatePVRolesManifestWork(newPrimary); err != nil { - d.log.Error(err, "failed to create or update PersistentVolume Roles manifest") - - return fmt.Errorf("failed to create or update PersistentVolume Roles manifest in namespace %s (%w)", - newPrimary, err) - } - pvMWName := d.mwu.BuildManifestWorkName(rmnutil.MWTypePV) existAndApplied, err := d.mwu.ManifestExistAndApplied(pvMWName, newPrimary) @@ -1518,13 +1511,6 @@ func (d *DRPCInstance) restorePVFromBackup(homeCluster string) error { func (d *DRPCInstance) processVRGManifestWork(homeCluster string) error { d.log.Info("Processing VRG ManifestWork", "cluster", homeCluster) - if err := d.mwu.CreateOrUpdateVRGRolesManifestWork(homeCluster); err != nil { - d.log.Error(err, "failed to create or update VolumeReplicationGroup Roles manifest") - - return fmt.Errorf("failed to create or update VolumeReplicationGroup Roles manifest in namespace %s (%w)", - homeCluster, err) - } - if err := d.mwu.CreateOrUpdateVRGManifestWork( d.instance.Name, d.instance.Namespace, homeCluster, d.drPolicy, d.instance.Spec.PVCSelector); err != nil { diff --git a/controllers/drplacementcontrol_controller_test.go b/controllers/drplacementcontrol_controller_test.go index 477de890f..bc8d0abe7 100644 --- a/controllers/drplacementcontrol_controller_test.go +++ b/controllers/drplacementcontrol_controller_test.go @@ -87,6 +87,25 @@ var ( } schedulingInterval = "1h" + + drPolicy = &rmn.DRPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: DRPolicyName, + }, + Spec: rmn.DRPolicySpec{ + DRClusterSet: []rmn.ManagedCluster{ + { + Name: EastManagedCluster, + S3ProfileName: "fakeS3Profile", + }, + { + Name: WestManagedCluster, + S3ProfileName: "fakeS3Profile", + }, + }, + SchedulingInterval: schedulingInterval, + }, + } ) var safeToProceed bool @@ -474,22 +493,15 @@ func createManagedClusters() { } } -func createDRPolicy(name, namespace string, drClusterSet []rmn.ManagedCluster) { - drPolicy := &rmn.DRPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: rmn.DRPolicySpec{ - DRClusterSet: drClusterSet, - SchedulingInterval: schedulingInterval, - }, - } - +func createDRPolicy() { err := k8sClient.Create(context.TODO(), drPolicy) Expect(err).NotTo(HaveOccurred()) } +func deleteDRPolicy() { + Expect(k8sClient.Delete(context.TODO(), drPolicy)).To(Succeed()) +} + func updateManifestWorkStatus(clusterNamespace, mwType, workType string) { manifestLookupKey := types.NamespacedName{ Name: rmnutil.ManifestWorkName(DRPCName, DRPCNamespaceName, mwType), @@ -564,17 +576,7 @@ func InitialDeployment(namespace, placementName, homeCluster string) (*plrv1.Pla createNamespaces() createManagedClusters() - createDRPolicy(DRPolicyName, DRPCNamespaceName, - []rmn.ManagedCluster{ - { - Name: EastManagedCluster, - S3ProfileName: "fakeS3Profile", - }, - { - Name: WestManagedCluster, - S3ProfileName: "fakeS3Profile", - }, - }) + createDRPolicy() placementRule := createPlacementRule(placementName, namespace) drpc := createDRPC(DRPCName, DRPCNamespaceName) @@ -584,7 +586,7 @@ func InitialDeployment(namespace, placementName, homeCluster string) (*plrv1.Pla func verifyVRGManifestWorkCreatedAsPrimary(managedCluster string) { vrgManifestLookupKey := types.NamespacedName{ - Name: "ramendr-vrg-roles", + Name: rmnutil.ClusterRolesManifestWorkName, Namespace: managedCluster, } createdVRGRolesManifest := &ocmworkv1.ManifestWork{} @@ -595,7 +597,7 @@ func verifyVRGManifestWorkCreatedAsPrimary(managedCluster string) { return err == nil }, timeout, interval).Should(BeTrue()) - Expect(len(createdVRGRolesManifest.Spec.Workload.Manifests)).To(Equal(2)) + Expect(len(createdVRGRolesManifest.Spec.Workload.Manifests)).To(BeNumerically(">=", 2)) vrgClusterRoleManifest := createdVRGRolesManifest.Spec.Workload.Manifests[0] Expect(vrgClusterRoleManifest).ToNot(BeNil()) @@ -760,13 +762,13 @@ var _ = Describe("DRPlacementControl Reconciler", func() { verifyUserPlacementRuleDecision(userPlacementRule.Name, userPlacementRule.Namespace, WestManagedCluster) verifyDRPCStatusPreferredClusterExpectation(rmn.FailedOver) verifyVRGManifestWorkCreatedAsPrimary(WestManagedCluster) - Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(4)) // MWs for VRG+ROLES+PVs + Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(3)) // MWs for VRG+ROLES+PVs waitForVRGMWDeletion(EastManagedCluster) updateManagedClusterViewStatusAsNotFound(mcvEast) // tickle the DRPC reconciler, should be removed once we watch for MCV resource updates touchDRPCToForceReconcile(drpc) setDRPCSpecExpectationTo(drpc, rmn.ActionFailover, "") - Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(1)) // MW for VRG ROLE only + Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(1)) // Roles MW waitForCompletion() val, err := rmnutil.GetMetricValueSingle("ramen_failover_time", dto.MetricType_GAUGE) @@ -786,10 +788,10 @@ var _ = Describe("DRPlacementControl Reconciler", func() { updateClonedPlacementRuleStatus(userPlacementRule, drpc, WestManagedCluster) verifyUserPlacementRuleDecision(userPlacementRule.Name, userPlacementRule.Namespace, WestManagedCluster) verifyDRPCStatusPreferredClusterExpectation(rmn.FailedOver) - Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(4)) // MWs for VRG+ROLES+PVs + Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(3)) // MWs for VRG+ROLES+PVs waitForVRGMWDeletion(EastManagedCluster) updateManagedClusterViewStatusAsNotFound(mcvEast) - Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(1)) // MWs for VRG ROLE only + Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(1)) // Roles MW touchDRPCToForceReconcile(drpc) waitForCompletion() }) @@ -820,11 +822,11 @@ var _ = Describe("DRPlacementControl Reconciler", func() { verifyDRPCStatusPreferredClusterExpectation(rmn.FailedBack) verifyVRGManifestWorkCreatedAsPrimary(EastManagedCluster) - Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(4)) // MWs for VRG+ROLES+PVs + Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(3)) // MWs for VRG+ROLES+PVs waitForVRGMWDeletion(WestManagedCluster) updateManagedClusterViewStatusAsNotFound(mcvWest) - Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(2)) // Roles MWs + Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(1)) // Roles MW waitForCompletion() val, err := rmnutil.GetMetricValueSingle("ramen_failback_time", dto.MetricType_GAUGE) @@ -848,8 +850,8 @@ var _ = Describe("DRPlacementControl Reconciler", func() { verifyDRPCStatusPreferredClusterExpectation(rmn.FailedBack) waitForVRGMWDeletion(WestManagedCluster) updateManagedClusterViewStatusAsNotFound(mcvWest) - Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(4)) // MWs for VRG+ROLES+PVs - Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(2)) // Roles MWs + Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(3)) // MWs for VRG+ROLES+PVs + Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(1)) // Roles MW waitForCompletion() }) }) @@ -874,10 +876,10 @@ var _ = Describe("DRPlacementControl Reconciler", func() { verifyDRPCStatusPreferredClusterExpectation(rmn.FailedOver) verifyVRGManifestWorkCreatedAsPrimary(WestManagedCluster) - Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(4)) // MW for VRG+ROLES+PVs + Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(3)) // MW for VRG+ROLES+PVs waitForVRGMWDeletion(EastManagedCluster) updateManagedClusterViewStatusAsNotFound(mcvEast) - Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(2)) // Roles MWs + Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(1)) // Roles MW waitForCompletion() }) }) @@ -904,10 +906,10 @@ var _ = Describe("DRPlacementControl Reconciler", func() { verifyDRPCStatusPreferredClusterExpectation(rmn.Relocated) verifyVRGManifestWorkCreatedAsPrimary(EastManagedCluster) - Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(4)) // MWs for VRG+ROLES+PVs + Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(3)) // MWs for VRG+ROLES+PVs waitForVRGMWDeletion(WestManagedCluster) updateManagedClusterViewStatusAsNotFound(mcvWest) - Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(2)) // Roles MWs + Expect(getManifestWorkCount(WestManagedCluster)).Should(Equal(1)) // Roles MW waitForCompletion() val, err := rmnutil.GetMetricValueSingle("ramen_relocate_time", dto.MetricType_GAUGE) @@ -921,7 +923,8 @@ var _ = Describe("DRPlacementControl Reconciler", func() { safeToProceed = false deleteDRPC() waitForCompletion() - Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(2)) // Roles MWs + Expect(getManifestWorkCount(EastManagedCluster)).Should(Equal(1)) // Roles MW + deleteDRPolicy() }) }) }) diff --git a/controllers/drpolicy_controller.go b/controllers/drpolicy_controller.go new file mode 100644 index 000000000..dc93eceab --- /dev/null +++ b/controllers/drpolicy_controller.go @@ -0,0 +1,124 @@ +/* +Copyright 2021 The RamenDR authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + ramen "github.com/ramendr/ramen/api/v1alpha1" + "github.com/ramendr/ramen/controllers/util" +) + +// DRPolicyReconciler reconciles a DRPolicy object +type DRPolicyReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=ramendr.openshift.io,resources=drpolicies,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=ramendr.openshift.io,resources=drpolicies/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=ramendr.openshift.io,resources=drpolicies/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the DRPolicy object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.9.2/pkg/reconcile +func (r *DRPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.Log.WithName("controllers").WithName("drpolicy").WithValues("name", req.NamespacedName.Name) + log.Info("reconcile enter") + + defer log.Info("reconcile exit") + + drpolicy := &ramen.DRPolicy{} + if err := r.Client.Get(ctx, req.NamespacedName, drpolicy); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(fmt.Errorf("get: %w", err)) + } + + manifestWorkUtil := util.MWUtil{Client: r.Client, Ctx: ctx, Log: log, InstName: "", InstNamespace: ""} + + switch drpolicy.ObjectMeta.DeletionTimestamp.IsZero() { + case true: + log.Info("create/update") + + if err := finalizerAdd(ctx, drpolicy, r.Client, log); err != nil { + return ctrl.Result{}, fmt.Errorf("finalizer add update: %w", err) + } + + if err := manifestWorkUtil.ClusterRolesCreate(drpolicy); err != nil { + return ctrl.Result{}, fmt.Errorf("cluster roles create: %w", err) + } + default: + log.Info("delete") + + if err := manifestWorkUtil.ClusterRolesDelete(drpolicy); err != nil { + return ctrl.Result{}, fmt.Errorf("cluster roles delete: %w", err) + } + + if err := finalizerRemove(ctx, drpolicy, r.Client, log); err != nil { + return ctrl.Result{}, fmt.Errorf("finalizer remove update: %w", err) + } + } + + return ctrl.Result{}, nil +} + +const finalizerName = "drpolicies.ramendr.openshift.io/ramen" + +func finalizerAdd(ctx context.Context, drpolicy *ramen.DRPolicy, client client.Client, log logr.Logger) error { + finalizerCount := len(drpolicy.ObjectMeta.Finalizers) + controllerutil.AddFinalizer(drpolicy, finalizerName) + + if len(drpolicy.ObjectMeta.Finalizers) != finalizerCount { + log.Info("finalizer add") + + return client.Update(ctx, drpolicy) + } + + return nil +} + +func finalizerRemove(ctx context.Context, drpolicy *ramen.DRPolicy, client client.Client, log logr.Logger) error { + finalizerCount := len(drpolicy.ObjectMeta.Finalizers) + controllerutil.RemoveFinalizer(drpolicy, finalizerName) + + if len(drpolicy.ObjectMeta.Finalizers) != finalizerCount { + log.Info("finalizer remove") + + return client.Update(ctx, drpolicy) + } + + return nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *DRPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&ramen.DRPolicy{}). + Complete(r) +} diff --git a/controllers/drpolicy_controller_test.go b/controllers/drpolicy_controller_test.go new file mode 100644 index 000000000..e600ed0e9 --- /dev/null +++ b/controllers/drpolicy_controller_test.go @@ -0,0 +1,117 @@ +package controllers_test + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + + ramen "github.com/ramendr/ramen/api/v1alpha1" + "github.com/ramendr/ramen/controllers/util" +) + +var _ = Describe("DrpolicyController", func() { + clusterRolesExpect := func(clusterNamesExpected *sets.String) { + Eventually( + func(g Gomega) { + clusterNames := sets.String{} + g.Expect(util.MWUtil{ + Client: k8sClient, + Ctx: context.TODO(), + Log: nil, + InstName: "", + InstNamespace: "", + }.ClusterRolesList(&clusterNames)).To(Succeed()) + fmt.Fprintf( + GinkgoWriter, + "expect: %v\nactual: %v\n", + *clusterNamesExpected, + clusterNames, + ) + g.Expect(clusterNamesExpected.Equal(clusterNames)).To(BeTrue()) + }, + 10, + 0.25, + ).Should(Succeed()) + } + drpolicyCreate := func(drpolicy *ramen.DRPolicy, clusterNames *sets.String) { + for _, clusterName := range sets.NewString(util.DrpolicyClusterNames(drpolicy)...).Difference(*clusterNames). + UnsortedList() { + Expect(k8sClient.Create( + context.TODO(), + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: clusterName}}, + )).To(Succeed()) + *clusterNames = clusterNames.Insert(clusterName) + } + Expect(k8sClient.Create(context.TODO(), drpolicy)).To(Succeed()) + clusterRolesExpect(clusterNames) + } + drpolicyDelete := func(drpolicy *ramen.DRPolicy, clusterNames *sets.String) { + Expect(k8sClient.Delete(context.TODO(), drpolicy)).To(Succeed()) + for _, clusterName := range sets.NewString(util.DrpolicyClusterNames(drpolicy)...).Difference(*clusterNames). + UnsortedList() { + Expect(k8sClient.Delete( + context.TODO(), + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: clusterName}}, + )).To(Succeed()) + *clusterNames = clusterNames.Delete(clusterName) + } + clusterRolesExpect(clusterNames) + } + drpolicies := [...]ramen.DRPolicy{ + { + ObjectMeta: metav1.ObjectMeta{Name: "drpolicy0"}, + Spec: ramen.DRPolicySpec{ + DRClusterSet: []ramen.ManagedCluster{ + {Name: "cluster-a", S3ProfileName: ""}, + {Name: "cluster-b", S3ProfileName: ""}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "drpolicy1"}, + Spec: ramen.DRPolicySpec{ + DRClusterSet: []ramen.ManagedCluster{ + {Name: "cluster-b", S3ProfileName: ""}, + {Name: "cluster-c", S3ProfileName: ""}, + }, + }, + }, + } + clusterNamesExpected := &sets.String{} + When("a 1st drpolicy is created", func() { + It("should create a cluster roles manifest work for each cluster specified in a 1st drpolicy", func() { + drpolicyCreate(&drpolicies[0], clusterNamesExpected) + }) + }) + When("a 1st drpolicy is updated to add some clusters and remove some other clusters", func() { + It("should create a cluster roles manifest work for each cluster added", func() { + }) + It("should delete a cluster roles manifest work for each cluster removed", func() { + }) + }) + When("a 2nd drpolicy is created specifying some clusters in a 1st drpolicy and some not", func() { + It("should create a cluster roles manifest work for each cluster specified in a 2nd drpolicy but not a 1st drpolicy", + func() { + drpolicyCreate(&drpolicies[1], clusterNamesExpected) + }, + ) + }) + When("a 1st drpolicy is deleted", func() { + It("should delete a cluster roles manifest work for each cluster specified in a 1st drpolicy but not a 2nd drpolicy", + func() { + drpolicyDelete(&drpolicies[0], clusterNamesExpected) + }, + ) + }) + When("a 2nd drpolicy is deleted", func() { + It("should delete a cluster roles manifest work for each cluster specified in a 2nd drpolicy", func() { + drpolicyDelete(&drpolicies[1], clusterNamesExpected) + }) + }) +}) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index cba35062f..54385285e 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -104,6 +104,12 @@ var _ = BeforeSuite(func() { }) Expect(err).ToNot(HaveOccurred()) + Expect((&ramencontrollers.DRPolicyReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + // Log: ctrl.Log.WithName("controllers").WithName("DRPolicy"), + }).SetupWithManager(k8sManager)).To(Succeed()) + err = (&ramencontrollers.VolumeReplicationGroupReconciler{ Client: k8sManager.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("VolumeReplicationGroup"), diff --git a/controllers/util/drpolicy_util.go b/controllers/util/drpolicy_util.go index 257f6c98a..50683c884 100644 --- a/controllers/util/drpolicy_util.go +++ b/controllers/util/drpolicy_util.go @@ -20,6 +20,15 @@ import ( rmn "github.com/ramendr/ramen/api/v1alpha1" ) +func DrpolicyClusterNames(drpolicy *rmn.DRPolicy) []string { + clusterNames := make([]string, len(drpolicy.Spec.DRClusterSet)) + for i := range drpolicy.Spec.DRClusterSet { + clusterNames[i] = drpolicy.Spec.DRClusterSet[i].Name + } + + return clusterNames +} + // Return a list of unique S3 profiles to upload the relevant cluster state of // the given home cluster func s3UploadProfileList(drPolicy rmn.DRPolicy, homeCluster string) (s3ProfileList []string) { diff --git a/controllers/util/mw_util.go b/controllers/util/mw_util.go index d9bc4ed9d..8b30c8f10 100644 --- a/controllers/util/mw_util.go +++ b/controllers/util/mw_util.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "reflect" + "sync" "github.com/go-logr/logr" ocmworkv1 "github.com/open-cluster-management/api/work/v1" @@ -31,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" dto "github.com/prometheus/client_model/go" @@ -40,11 +42,13 @@ import ( ) const ( + ClusterRolesManifestWorkName = "ramendr-roles" + // ManifestWorkNameFormat is a formated a string used to generate the manifest name // The format is name-namespace-type-mw where: // - name is the DRPC name // - namespace is the DRPC namespace - // - type is either "vrg", "pv", or "roles" + // - type is either "vrg" or "pv" ManifestWorkNameFormat string = "%s-%s-%s-mw" // ManifestWork VRG Type @@ -53,9 +57,6 @@ const ( // ManifestWork PV Type MWTypePV string = "pv" - // ManifestWork Roles Type - MWTypeRoles string = "roles" - // Annotations for MW and PlacementRule DRPCNameAnnotation = "drplacementcontrol.ramendr.openshift.io/drpc-name" DRPCNamespaceAnnotation = "drplacementcontrol.ramendr.openshift.io/drpc-namespace" @@ -205,8 +206,68 @@ func (mwu *MWUtil) generateVRGManifest( }) } -func (mwu *MWUtil) CreateOrUpdateVRGRolesManifestWork(namespace string) error { - manifestWork, err := mwu.generateVRGRolesManifestWork(namespace) +func (mwu MWUtil) ClusterRolesList(clusterNames *sets.String) error { + manifestworks := &ocmworkv1.ManifestWorkList{} + if err := mwu.Client.List(mwu.Ctx, manifestworks); err != nil { + return fmt.Errorf("manifestworks list: %w", err) + } + + for i := range manifestworks.Items { + manifestwork := &manifestworks.Items[i] + if manifestwork.ObjectMeta.Name == ClusterRolesManifestWorkName { + *clusterNames = clusterNames.Insert(manifestwork.ObjectMeta.Namespace) + } + } + + return nil +} + +var clusterRolesMutex sync.Mutex + +func (mwu *MWUtil) ClusterRolesCreate(drpolicy *rmn.DRPolicy) error { + clusterRolesMutex.Lock() + defer clusterRolesMutex.Unlock() + + for _, clusterName := range DrpolicyClusterNames(drpolicy) { + if err := mwu.createOrUpdateClusterRolesManifestWork(clusterName); err != nil { + return err + } + } + + return nil +} + +func (mwu *MWUtil) ClusterRolesDelete(drpolicy *rmn.DRPolicy) error { + drpolicies := rmn.DRPolicyList{} + clusterNames := sets.String{} + + clusterRolesMutex.Lock() + defer clusterRolesMutex.Unlock() + + if err := mwu.Client.List(mwu.Ctx, &drpolicies); err != nil { + return fmt.Errorf("drpolicies list: %w", err) + } + + for i := range drpolicies.Items { + drpolicy1 := &drpolicies.Items[i] + if drpolicy1.ObjectMeta.Name != drpolicy.ObjectMeta.Name { + clusterNames = clusterNames.Insert(DrpolicyClusterNames(drpolicy1)...) + } + } + + for _, clusterName := range DrpolicyClusterNames(drpolicy) { + if !clusterNames.Has(clusterName) { + if err := mwu.deleteManifestWork(ClusterRolesManifestWorkName, clusterName); err != nil { + return err + } + } + } + + return nil +} + +func (mwu *MWUtil) createOrUpdateClusterRolesManifestWork(namespace string) error { + manifestWork, err := mwu.generateClusterRolesManifestWork(namespace) if err != nil { return err } @@ -214,7 +275,7 @@ func (mwu *MWUtil) CreateOrUpdateVRGRolesManifestWork(namespace string) error { return mwu.createOrUpdateManifestWork(manifestWork, namespace) } -func (mwu *MWUtil) generateVRGRolesManifestWork(namespace string) (*ocmworkv1.ManifestWork, error) { +func (mwu *MWUtil) generateClusterRolesManifestWork(namespace string) (*ocmworkv1.ManifestWork, error) { vrgClusterRole, err := mwu.generateVRGClusterRoleManifest() if err != nil { mwu.Log.Error(err, "failed to generate VolumeReplicationGroup ClusterRole manifest") @@ -229,10 +290,29 @@ func (mwu *MWUtil) generateVRGRolesManifestWork(namespace string) (*ocmworkv1.Ma return nil, err } - manifests := []ocmworkv1.Manifest{*vrgClusterRole, *vrgClusterRoleBinding} + pvClusterRole, err := mwu.generatePVClusterRoleManifest() + if err != nil { + mwu.Log.Error(err, "failed to generate PersistentVolume ClusterRole manifest") + + return nil, err + } + + pvClusterRoleBinding, err := mwu.generatePVClusterRoleBindingManifest() + if err != nil { + mwu.Log.Error(err, "failed to generate PersistentVolume ClusterRoleBinding manifest") + + return nil, err + } + + manifests := []ocmworkv1.Manifest{ + *vrgClusterRole, + *vrgClusterRoleBinding, + *pvClusterRole, + *pvClusterRoleBinding, + } return mwu.newManifestWork( - "ramendr-vrg-roles", + ClusterRolesManifestWorkName, namespace, map[string]string{}, manifests), nil @@ -271,39 +351,6 @@ func (mwu *MWUtil) generateVRGClusterRoleBindingManifest() (*ocmworkv1.Manifest, }) } -func (mwu *MWUtil) CreateOrUpdatePVRolesManifestWork(namespace string) error { - manifestWork, err := mwu.generatePVRolesManifestWork(namespace) - if err != nil { - return err - } - - return mwu.createOrUpdateManifestWork(manifestWork, namespace) -} - -func (mwu *MWUtil) generatePVRolesManifestWork(namespace string) (*ocmworkv1.ManifestWork, error) { - pvClusterRole, err := mwu.generatePVClusterRoleManifest() - if err != nil { - mwu.Log.Error(err, "failed to generate PersistentVolume ClusterRole manifest") - - return nil, err - } - - pvClusterRoleBinding, err := mwu.generatePVClusterRoleBindingManifest() - if err != nil { - mwu.Log.Error(err, "failed to generate PersistentVolume ClusterRoleBinding manifest") - - return nil, err - } - - manifests := []ocmworkv1.Manifest{*pvClusterRole, *pvClusterRoleBinding} - - return mwu.newManifestWork( - "ramendr-pv-roles", - namespace, - map[string]string{}, - manifests), nil -} - func (mwu *MWUtil) generatePVClusterRoleManifest() (*ocmworkv1.Manifest, error) { return mwu.GenerateManifest(&rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, diff --git a/go.mod b/go.mod index bf9ea9a53..5ccfe2027 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/csi-addons/volume-replication-operator v0.1.0 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-logr/logr v0.4.0 - github.com/onsi/ginkgo v1.16.1 - github.com/onsi/gomega v1.11.0 + github.com/onsi/ginkgo v1.16.4 + github.com/onsi/gomega v1.15.0 github.com/open-cluster-management/api v0.0.0-20210527013639-a6845f2ebcb1 github.com/open-cluster-management/multicloud-operators-placementrule v1.0.1-2020-06-08-14-28-27.0.20201118195339-05a8c4c89c12 github.com/open-cluster-management/multicloud-operators-subscription v1.2.2-2-20201130-59f96 diff --git a/go.sum b/go.sum index bc6240290..40da2de84 100644 --- a/go.sum +++ b/go.sum @@ -759,8 +759,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -809,8 +811,9 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-metrics-stackdriver v0.0.0-20190816035513-b52628e82e2a/go.mod h1:o93WzqysX0jP/10Y13hfL6aq9RoUvGaVdkrH5awMksE= @@ -1428,8 +1431,8 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54= -github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1441,8 +1444,8 @@ github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug= -github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= +github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-cluster-management/addon-framework v0.0.0-20210423114544-8e08479e80b6/go.mod h1:mcpd6pc0j/L+WLFwV2MXHVMr+86ri2iUdTK2M8RHJ7U= github.com/open-cluster-management/ansiblejob-go-lib v0.1.12/go.mod h1:dDL15yXQUna2cVZwjgnQ/HZ95deXgI4Uj8/djY4A5o4= @@ -2178,8 +2181,9 @@ golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2306,6 +2310,7 @@ golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2320,8 +2325,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2604,8 +2610,10 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/AlecAivazis/survey.v1 v1.8.9-0.20200217094205-6773bdf39b7f/go.mod h1:CaHjv79TCgAvXMSFJSVgonHXYWxnhzI3eoHtnX5UgUo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= diff --git a/main.go b/main.go index d36013092..c72179fc1 100644 --- a/main.go +++ b/main.go @@ -105,6 +105,14 @@ func newManager() (ctrl.Manager, error) { func setupReconcilers(mgr ctrl.Manager) { if controllerType == ramendrv1alpha1.DRHub { + if err := (&controllers.DRPolicyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "DRPolicy") + os.Exit(1) + } + drpcReconciler := (&controllers.DRPlacementControlReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("DRPlacementControl"),