Merge pull request #270 from red-hat-storage/sync_us--main

Syncing latest changes from upstream main for ramen

Makefile

@@ -189,7 +189,7 @@ test-ramenctl: ## Run ramenctl tests.
 	$(MAKE) -C ramenctl
 
 e2e-rdr: generate manifests ## Run rdr-e2e tests.
-	./e2e/rdr-e2e.sh
+	cd e2e && ./e2e-rdr.sh
 
 coverage:
 	go tool cover -html=cover.out

e2e/config.yaml

@@ -0,0 +1,4 @@
+---
+channelname: "ramen-gitops"
+channelnamespace: "ramen-samples"
+giturl: "https://github.com/RamenDR/ocm-ramen-samples.git"

e2e/deployers/applicationset.go

@@ -10,8 +10,8 @@ import (
 
 type ApplicationSet struct{}
 
-func (a *ApplicationSet) Init() {
-}
+// func (a *ApplicationSet) Init() {
+// }
 
 func (a ApplicationSet) Deploy(w workloads.Workload) error {
 	util.Ctx.Log.Info("enter Deploy " + w.GetName() + "/Appset")

e2e/deployers/crud.go

@@ -3,34 +3,202 @@
 
 package deployers
 
-func createNamespace() error {
+import (
+	"context"
+	"strings"
+
+	"github.com/ramendr/ramen/e2e/util"
+	"github.com/ramendr/ramen/e2e/workloads"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ocmv1b1 "open-cluster-management.io/api/cluster/v1beta1"
+	ocmv1b2 "open-cluster-management.io/api/cluster/v1beta2"
+	placementrulev1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/placementrule/v1"
+	subscriptionv1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/v1"
+)
+
+const (
+	AppLabelKey    = "app"
+	ClusterSetName = "default"
+)
+
+func createManagedClusterSetBinding(name, namespace string) error {
+	labels := make(map[string]string)
+	labels[AppLabelKey] = namespace
+	mcsb := &ocmv1b2.ManagedClusterSetBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Labels:    labels,
+		},
+		Spec: ocmv1b2.ManagedClusterSetBindingSpec{
+			ClusterSet: ClusterSetName,
+		},
+	}
+
+	err := util.Ctx.Hub.CtrlClient.Create(context.Background(), mcsb)
+	if err != nil {
+		if !errors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+
 	return nil
 }
 
-func createManagedClusterSetBinding() error {
+func deleteManagedClusterSetBinding(name, namespace string) error {
+	mcsb := &ocmv1b2.ManagedClusterSetBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+
+	err := util.Ctx.Hub.CtrlClient.Delete(context.Background(), mcsb)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return err
+		}
+
+		util.Ctx.Log.Info("managedClusterSetBinding " + name + " not found")
+	}
+
 	return nil
 }
 
-func createPlacement() error {
+func createPlacement(name, namespace string) error {
+	labels := make(map[string]string)
+	labels[AppLabelKey] = name
+	clusterSet := []string{"default"}
+
+	var numClusters int32 = 1
+	placement := &ocmv1b1.Placement{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Labels:    labels,
+		},
+		Spec: ocmv1b1.PlacementSpec{
+			ClusterSets:      clusterSet,
+			NumberOfClusters: &numClusters,
+		},
+	}
+
+	err := util.Ctx.Hub.CtrlClient.Create(context.Background(), placement)
+	if err != nil {
+		if !errors.IsAlreadyExists(err) {
+			return err
+		}
+
+		util.Ctx.Log.Info("placement " + placement.Name + " already Exists")
+	}
+
 	return nil
 }
 
-func createSubscription() error {
+func deletePlacement(name, namespace string) error {
+	placement := &ocmv1b1.Placement{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+
+	err := util.Ctx.Hub.CtrlClient.Delete(context.Background(), placement)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return err
+		}
+
+		util.Ctx.Log.Info("placement " + name + " not found")
+	}
+
 	return nil
 }
 
-func deleteNamespace() error {
+func createSubscription(s Subscription, w workloads.Workload) error {
+	name := GetCombinedName(s, w)
+	namespace := name
+
+	labels := make(map[string]string)
+	labels[AppLabelKey] = name
+
+	annotations := make(map[string]string)
+	annotations["apps.open-cluster-management.io/github-branch"] = w.GetRevision()
+	annotations["apps.open-cluster-management.io/github-path"] = w.GetPath()
+
+	placementRef := corev1.ObjectReference{
+		Kind: "Placement",
+		Name: name,
+	}
+
+	placementRulePlacement := &placementrulev1.Placement{}
+	placementRulePlacement.PlacementRef = &placementRef
+
+	subscription := &subscriptionv1.Subscription{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Namespace:   namespace,
+			Labels:      labels,
+			Annotations: annotations,
+		},
+		Spec: subscriptionv1.SubscriptionSpec{
+			Channel:   util.GetChannelNamespace() + "/" + util.GetChannelName(),
+			Placement: placementRulePlacement,
+		},
+	}
+
+	err := util.Ctx.Hub.CtrlClient.Create(context.Background(), subscription)
+	if err != nil {
+		if !errors.IsAlreadyExists(err) {
+			return err
+		}
+
+		util.Ctx.Log.Info("placement " + subscription.Name + " already Exists")
+	}
+
 	return nil
 }
 
-func deleteManagedClusterSetBinding() error {
+func deleteSubscription(s Subscription, w workloads.Workload) error {
+	name := GetCombinedName(s, w)
+	namespace := name
+
+	subscription := &subscriptionv1.Subscription{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+
+	err := util.Ctx.Hub.CtrlClient.Delete(context.Background(), subscription)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return err
+		}
+
+		util.Ctx.Log.Info("subscription " + name + " not found")
+	}
+
 	return nil
 }
 
-func deletePlacement() error {
-	return nil
+func GetCombinedName(d Deployer, w workloads.Workload) string {
+	return strings.ToLower(d.GetName() + "-" + w.GetName() + "-" + w.GetAppName())
 }
 
-func deleteSubscription() error {
-	return nil
+func getSubscription(ctrlClient client.Client, namespace, name string) (*subscriptionv1.Subscription, error) {
+	subscription := &subscriptionv1.Subscription{}
+	key := types.NamespacedName{Name: name, Namespace: namespace}
+
+	err := ctrlClient.Get(context.Background(), key, subscription)
+	if err != nil {
+		return nil, err
+	}
+
+	return subscription, nil
 }

e2e/deployers/deployer.go

@@ -7,7 +7,6 @@ import "github.com/ramendr/ramen/e2e/workloads"
 
 // Deployer interface has methods to deploy a workload to a cluster
 type Deployer interface {
-	Init()
 	Deploy(workloads.Workload) error
 	Undeploy(workloads.Workload) error
 	// Scale(Workload) for adding/removing PVCs; in Deployer even though scaling is a Workload interface

e2e/deployers/retry.go

@@ -0,0 +1,48 @@
+// SPDX-FileCopyrightText: The RamenDR authors
+// SPDX-License-Identifier: Apache-2.0
+
+package deployers
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/ramendr/ramen/e2e/util"
+	subscriptionv1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/v1"
+)
+
+const FiveSecondsDuration = 5 * time.Second
+
+func waitSubscriptionPhase(namespace, name string, phase subscriptionv1.SubscriptionPhase) error {
+	// sleep to wait for subscription is processed
+	time.Sleep(FiveSecondsDuration)
+
+	startTime := time.Now()
+
+	for {
+		sub, err := getSubscription(util.Ctx.Hub.CtrlClient, namespace, name)
+		if err != nil {
+			return err
+		}
+
+		currentPhase := sub.Status.Phase
+		if currentPhase == phase {
+			util.Ctx.Log.Info(fmt.Sprintf("subscription %s phase is %s", name, phase))
+
+			return nil
+		}
+
+		if time.Since(startTime) > time.Second*time.Duration(util.Timeout) {
+			return fmt.Errorf(fmt.Sprintf("subscription %s status is not %s yet before timeout of %v",
+				name, phase, util.Timeout))
+		}
+
+		if currentPhase == "" {
+			currentPhase = "empty"
+		}
+
+		util.Ctx.Log.Info(fmt.Sprintf("current subscription %s phase is %s, expecting %s, retry in %v seconds",
+			name, currentPhase, phase, util.TimeInterval))
+		time.Sleep(time.Second * time.Duration(util.TimeInterval))
+	}
+}

e2e/deployers/subscription.go

@@ -6,21 +6,13 @@ package deployers
 import (
 	"github.com/ramendr/ramen/e2e/util"
 	"github.com/ramendr/ramen/e2e/workloads"
+	subscriptionv1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/v1"
 )
 
-type Subscription struct {
-	// NamePrefix helps when the same workload needs to run in parallel with different deployers.
-	// In the future we potentially would add a resource suffix that is either randomly generated
-	// or a hash of the full test name, to handle cases where we want to run the "same" combination
-	// of deployer+workload for various reasons.
-	NamePrefix string
-	McsbName   string
-}
+// mcsb name must be same as the target ManagedClusterSet
+const McsbName = ClusterSetName
 
-func (s *Subscription) Init() {
-	s.NamePrefix = "sub-"
-	s.McsbName = "default"
-}
+type Subscription struct{}
 
 func (s Subscription) GetName() string {
 	return "Subscription"
@@ -33,25 +25,33 @@ func (s Subscription) Deploy(w workloads.Workload) error {
 	// Generate a Subscription for the Workload
 	// - Kustomize the Workload; call Workload.Kustomize(StorageType)
 	// Address namespace/label/suffix as needed for various resources
-	util.Ctx.Log.Info("enter Deploy " + w.GetName() + "/Subscription")
+	util.Ctx.Log.Info("enter Deploy " + w.GetName() + "/" + s.GetName())
+
+	name := GetCombinedName(s, w)
+	namespace := name
+
+	// create subscription namespace
+	err := util.CreateNamespace(util.Ctx.Hub.CtrlClient, namespace)
+	if err != nil {
+		return err
+	}
 
-	// w.Kustomize()
-	err := createNamespace()
+	err = createManagedClusterSetBinding(McsbName, namespace)
 	if err != nil {
 		return err
 	}
 
-	err = createManagedClusterSetBinding()
+	err = createPlacement(name, namespace)
 	if err != nil {
 		return err
 	}
 
-	err = createPlacement()
+	err = createSubscription(s, w)
 	if err != nil {
 		return err
 	}
 
-	err = createSubscription()
+	err = waitSubscriptionPhase(namespace, name, subscriptionv1.SubscriptionPropagated)
 	if err != nil {
 		return err
 	}
@@ -61,24 +61,27 @@ func (s Subscription) Deploy(w workloads.Workload) error {
 
 func (s Subscription) Undeploy(w workloads.Workload) error {
 	// Delete Subscription, Placement, Binding
-	util.Ctx.Log.Info("enter Undeploy " + w.GetName() + "/Subscription")
+	util.Ctx.Log.Info("enter Undeploy " + w.GetName() + s.GetName())
+
+	name := GetCombinedName(s, w)
+	namespace := name
 
-	err := deleteSubscription()
+	err := deleteSubscription(s, w)
 	if err != nil {
 		return err
 	}
 
-	err = deletePlacement()
+	err = deletePlacement(name, namespace)
 	if err != nil {
 		return err
 	}
 
-	err = deleteManagedClusterSetBinding()
+	err = deleteManagedClusterSetBinding(McsbName, namespace)
 	if err != nil {
 		return err
 	}
 
-	err = deleteNamespace()
+	err = util.DeleteNamespace(util.Ctx.Hub.CtrlClient, namespace)
 	if err != nil {
 		return err
 	}