feat: implement roleRefs API for RootSyncs

e2e/nomostest/nt.go

@@ -28,6 +28,7 @@ import (
 	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"kpt.dev/configsync/e2e/nomostest/gitproviders"
 	"kpt.dev/configsync/e2e/nomostest/ntopts"
 	"kpt.dev/configsync/e2e/nomostest/portforwarder"
@@ -38,6 +39,7 @@ import (
 	"kpt.dev/configsync/e2e/nomostest/testshell"
 	"kpt.dev/configsync/e2e/nomostest/testwatcher"
 	"kpt.dev/configsync/pkg/core"
+	"kpt.dev/configsync/pkg/reconcilermanager/controllers"
 	"kpt.dev/configsync/pkg/testing/fake"
 	"kpt.dev/configsync/pkg/util"
 	"kpt.dev/configsync/pkg/util/log"
@@ -927,3 +929,31 @@ func cloneCloudSourceRepo(nt *NT, repo string) (string, error) {
 	}
 	return cloneDir, nil
 }
+
+// ListReconcilerRoleBindings is a convenience method for listing all RoleBindings
+// associated with a reconciler.
+func (nt *NT) ListReconcilerRoleBindings(syncKind string, rsRef types.NamespacedName) ([]rbacv1.RoleBinding, error) {
+	opts := &client.ListOptions{}
+	opts.LabelSelector = client.MatchingLabelsSelector{
+		Selector: labels.SelectorFromSet(controllers.ManagedObjectLabelMap(syncKind, rsRef)),
+	}
+	rbList := rbacv1.RoleBindingList{}
+	if err := nt.KubeClient.List(&rbList, opts); err != nil {
+		return nil, errors.Wrap(err, "listing RoleBindings")
+	}
+	return rbList.Items, nil
+}
+
+// ListReconcilerClusterRoleBindings is a convenience method for listing all
+// ClusterRoleBindings associated with a reconciler.
+func (nt *NT) ListReconcilerClusterRoleBindings(syncKind string, rsRef types.NamespacedName) ([]rbacv1.ClusterRoleBinding, error) {
+	opts := &client.ListOptions{}
+	opts.LabelSelector = client.MatchingLabelsSelector{
+		Selector: labels.SelectorFromSet(controllers.ManagedObjectLabelMap(syncKind, rsRef)),
+	}
+	crbList := rbacv1.ClusterRoleBindingList{}
+	if err := nt.KubeClient.List(&crbList, opts); err != nil {
+		return nil, errors.Wrap(err, "listing ClusterRoleBindings")
+	}
+	return crbList.Items, nil
+}

e2e/nomostest/testpredicates/predicates.go

@@ -1240,3 +1240,36 @@ func ResourceGroupStatusEquals(expected v1alpha1.ResourceGroupStatus) Predicate
 		return nil
 	}
 }
+
+func subjectNamesEqual(want []string, got []rbacv1.Subject) error {
+	if len(got) != len(want) {
+		return errors.Errorf("want %v subjects; got %v", want, got)
+	}
+
+	found := make(map[string]bool)
+	for _, subj := range got {
+		found[subj.Name] = true
+	}
+	for _, name := range want {
+		if !found[name] {
+			return errors.Errorf("missing subject %q", name)
+		}
+	}
+
+	return nil
+}
+
+// ClusterRoleBindingSubjectNamesEqual checks that the ClusterRoleBinding has a list
+// of subjects whose names match the specified list of names.
+func ClusterRoleBindingSubjectNamesEqual(subjects ...string) func(o client.Object) error {
+	return func(o client.Object) error {
+		if o == nil {
+			return ErrObjectNotFound
+		}
+		r, ok := o.(*rbacv1.ClusterRoleBinding)
+		if !ok {
+			return WrongTypeErr(o, r)
+		}
+		return subjectNamesEqual(subjects, r.Subjects)
+	}
+}

e2e/testcases/namespace_repo_test.go

@@ -328,7 +328,7 @@ func checkRepoSyncResourcesNotPresent(nt *nomostest.NT, namespace string, secret
 		return nt.Watcher.WatchForNotFound(kinds.ServiceAccount(), core.NsReconcilerName(namespace, configsync.RepoSyncName), configsync.ControllerNamespace)
 	})
 	tg.Go(func() error {
-		return nt.Watcher.WatchForNotFound(kinds.ServiceAccount(), controllers.RepoSyncPermissionsName(), configsync.ControllerNamespace)
+		return nt.Watcher.WatchForNotFound(kinds.ServiceAccount(), controllers.RepoSyncBaseClusterRoleName, configsync.ControllerNamespace)
 	})
 	for _, sName := range secretNames {
 		nn := types.NamespacedName{Name: sName, Namespace: configsync.ControllerNamespace}

e2e/testcases/override_role_refs_test.go

@@ -0,0 +1,223 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/pkg/errors"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"kpt.dev/configsync/e2e/nomostest"
+	"kpt.dev/configsync/e2e/nomostest/ntopts"
+	"kpt.dev/configsync/e2e/nomostest/taskgroup"
+	nomostesting "kpt.dev/configsync/e2e/nomostest/testing"
+	"kpt.dev/configsync/e2e/nomostest/testpredicates"
+	"kpt.dev/configsync/pkg/api/configsync"
+	"kpt.dev/configsync/pkg/api/configsync/v1beta1"
+	"kpt.dev/configsync/pkg/core"
+	"kpt.dev/configsync/pkg/kinds"
+	"kpt.dev/configsync/pkg/reconcilermanager/controllers"
+	"kpt.dev/configsync/pkg/testing/fake"
+)
+
+func TestRootSyncRoleRefs(t *testing.T) {
+	nt := nomostest.New(t, nomostesting.OverrideAPI, ntopts.Unstructured,
+		ntopts.RootRepo("sync-a"),
+	)
+	rootSyncA := nomostest.RootSyncObjectV1Beta1FromRootRepo(nt, "sync-a")
+	syncAReconcilerName := core.RootReconcilerName(rootSyncA.Name)
+	syncANN := types.NamespacedName{
+		Name:      rootSyncA.Name,
+		Namespace: rootSyncA.Namespace,
+	}
+	if err := nt.Validate(controllers.RootSyncLegacyClusterRoleBindingName, "", &rbacv1.ClusterRoleBinding{},
+		testpredicates.ClusterRoleBindingSubjectNamesEqual(nomostest.DefaultRootReconcilerName, syncAReconcilerName)); err != nil {
+		nt.T.Fatal(err)
+	}
+	nt.T.Logf("Set custom roleRef overrides on RootSync %s", syncANN.Name)
+	rootSyncA.Spec.SafeOverride().RoleRefs = []v1beta1.RootSyncRoleRef{
+		{
+			Kind:      "Role",
+			Name:      "foo-role",
+			Namespace: "foo",
+		},
+		{
+			Kind: "ClusterRole",
+			Name: "foo-role",
+		},
+		{
+			Kind: "ClusterRole",
+			Name: "bar-role",
+		},
+		{
+			Kind:      "ClusterRole",
+			Name:      "foo-role",
+			Namespace: "foo",
+		},
+	}
+	roleObject := fake.RoleObject(core.Name("foo-role"), core.Namespace("foo"))
+	clusterRoleObject := fake.ClusterRoleObject(core.Name("foo-role"))
+	clusterRoleObject.Rules = []rbacv1.PolicyRule{
+		{ // permission to manage the "safety clusterrole"
+			Verbs:     []string{"*"},
+			APIGroups: []string{"rbac.authorization.k8s.io"},
+			Resources: []string{"clusterroles"},
+		},
+		{ // permission to manage the "safety namespace"
+			Verbs:     []string{"*"},
+			APIGroups: []string{""},
+			Resources: []string{"namespaces"},
+		},
+	}
+	clusterRoleObject2 := fake.ClusterRoleObject(core.Name("bar-role"))
+	nt.Must(nt.RootRepos[configsync.RootSyncName].Add(
+		nomostest.StructuredNSPath(rootSyncA.Namespace, rootSyncA.Name),
+		rootSyncA,
+	))
+	nt.Must(nt.RootRepos[configsync.RootSyncName].Add(
+		fmt.Sprintf("acme/namespaces/%s/%s.yaml", roleObject.Namespace, roleObject.Name),
+		roleObject,
+	))
+	nt.Must(nt.RootRepos[configsync.RootSyncName].Add(
+		fmt.Sprintf("acme/namespaces/%s.yaml", clusterRoleObject.Name),
+		clusterRoleObject,
+	))
+	nt.Must(nt.RootRepos[configsync.RootSyncName].Add(
+		fmt.Sprintf("acme/namespaces/%s.yaml", clusterRoleObject2.Name),
+		clusterRoleObject2,
+	))
+	nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Add Roles and RoleRefs"))
+	if err := nt.WatchForAllSyncs(); err != nil {
+		nt.T.Fatal(err)
+	}
+	tg := taskgroup.New()
+	tg.Go(func() error {
+		return validateRoleRefs(nt, configsync.RootSyncKind, syncANN, rootSyncA.Spec.SafeOverride().RoleRefs)
+	})
+	tg.Go(func() error {
+		return nt.Validate(controllers.RootSyncLegacyClusterRoleBindingName, "", &rbacv1.ClusterRoleBinding{},
+			testpredicates.ClusterRoleBindingSubjectNamesEqual(nomostest.DefaultRootReconcilerName))
+	})
+	tg.Go(func() error {
+		return nt.Validate(controllers.RootSyncBaseClusterRoleBindingName, "", &rbacv1.ClusterRoleBinding{},
+			testpredicates.ClusterRoleBindingSubjectNamesEqual(syncAReconcilerName))
+	})
+	if err := tg.Wait(); err != nil {
+		nt.T.Fatal(err)
+	}
+
+	nt.T.Logf("Remove some but not all roleRefs from %s to verify garbage collection", syncANN.Name)
+	rootSyncA.Spec.SafeOverride().RoleRefs = []v1beta1.RootSyncRoleRef{
+		{
+			Kind: "ClusterRole",
+			Name: "foo-role",
+		},
+	}
+	nt.Must(nt.RootRepos[configsync.RootSyncName].Add(
+		fmt.Sprintf("acme/namespaces/%s/%s.yaml", configsync.ControllerNamespace, rootSyncA.Name),
+		rootSyncA,
+	))
+	nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Reduce RoleRefs"))
+	if err := nt.WatchForAllSyncs(); err != nil {
+		nt.T.Fatal(err)
+	}
+	tg = taskgroup.New()
+	tg.Go(func() error {
+		return validateRoleRefs(nt, configsync.RootSyncKind, syncANN, rootSyncA.Spec.SafeOverride().RoleRefs)
+	})
+	tg.Go(func() error {
+		return nt.Validate(controllers.RootSyncLegacyClusterRoleBindingName, "", &rbacv1.ClusterRoleBinding{},
+			testpredicates.ClusterRoleBindingSubjectNamesEqual(nomostest.DefaultRootReconcilerName))
+	})
+	tg.Go(func() error {
+		return nt.Validate(controllers.RootSyncBaseClusterRoleBindingName, "", &rbacv1.ClusterRoleBinding{},
+			testpredicates.ClusterRoleBindingSubjectNamesEqual(syncAReconcilerName))
+	})
+	if err := tg.Wait(); err != nil {
+		nt.T.Fatal(err)
+	}
+
+	nt.T.Logf("Delete the RootSync %s to verify garbage collection", syncANN.Name)
+	nt.Must(nt.RootRepos[configsync.RootSyncName].Remove(
+		nomostest.StructuredNSPath(rootSyncA.Namespace, rootSyncA.Name),
+	))
+	nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Prune RootSync"))
+	if err := nt.WatchForSync(kinds.RootSyncV1Beta1(), configsync.RootSyncName, configsync.ControllerNamespace,
+		nomostest.DefaultRootSha1Fn, nomostest.RootSyncHasStatusSyncCommit, nil); err != nil {
+		nt.T.Fatal(err)
+	}
+	tg = taskgroup.New()
+	tg.Go(func() error {
+		return nt.ValidateNotFound(syncANN.Name, syncANN.Namespace, &v1beta1.RootSync{})
+	})
+	tg.Go(func() error {
+		return validateRoleRefs(nt, configsync.RootSyncKind, syncANN, []v1beta1.RootSyncRoleRef{})
+	})
+	tg.Go(func() error {
+		return nt.Validate(controllers.RootSyncLegacyClusterRoleBindingName, "", &rbacv1.ClusterRoleBinding{},
+			testpredicates.ClusterRoleBindingSubjectNamesEqual(nomostest.DefaultRootReconcilerName))
+	})
+	tg.Go(func() error {
+		return nt.ValidateNotFound(controllers.RootSyncBaseClusterRoleBindingName, "", &rbacv1.ClusterRoleBinding{})
+	})
+	if err := tg.Wait(); err != nil {
+		nt.T.Fatal(err)
+	}
+}
+
+// This helper function verifies that the specified role refs are mapped to
+// bindings on the cluster. The bindings are looked up using labels based on the
+// RSync kind/name/namespace. Returns an error if what is found on the cluster
+// is not an exact match.
+func validateRoleRefs(nt *nomostest.NT, syncKind string, rsRef types.NamespacedName, expected []v1beta1.RootSyncRoleRef) error {
+	roleBindings, err := nt.ListReconcilerRoleBindings(syncKind, rsRef)
+	if err != nil {
+		return err
+	}
+	actualRoleRefCount := make(map[v1beta1.RootSyncRoleRef]int)
+	for _, rb := range roleBindings {
+		roleRef := v1beta1.RootSyncRoleRef{
+			Kind:      rb.RoleRef.Kind,
+			Name:      rb.RoleRef.Name,
+			Namespace: rb.Namespace,
+		}
+		actualRoleRefCount[roleRef]++
+	}
+	clusterRoleBindings, err := nt.ListReconcilerClusterRoleBindings(syncKind, rsRef)
+	if err != nil {
+		return err
+	}
+	for _, crb := range clusterRoleBindings {
+		roleRef := v1beta1.RootSyncRoleRef{
+			Kind: crb.RoleRef.Kind,
+			Name: crb.RoleRef.Name,
+		}
+		actualRoleRefCount[roleRef]++
+	}
+	totalBindings := len(roleBindings) + len(clusterRoleBindings)
+	if len(expected) != totalBindings {
+		return errors.Errorf("expected %d bindings but found %d",
+			len(expected), totalBindings)
+	}
+	for _, roleRef := range expected {
+		if actualRoleRefCount[roleRef] != 1 {
+			return errors.Errorf("expected to find one binding mapping to %s, found %d",
+				roleRef, actualRoleRefCount[roleRef])
+		}
+	}
+	return nil
+}

e2e/testcases/reconciler_manager_test.go

@@ -207,7 +207,7 @@ func validateRootSyncDependencies(nt *nomostest.NT, rsName string) []client.Obje
 	// only happens when upgrading from a very old unsupported version.
 
 	rootSyncCRB := &rbacv1.ClusterRoleBinding{}
-	setNN(rootSyncCRB, client.ObjectKey{Name: controllers.RootSyncPermissionsName(rootSyncReconciler.Name)})
+	rootSyncCRB.Name = controllers.RootSyncLegacyClusterRoleBindingName
 	rootSyncDependencies = append(rootSyncDependencies, rootSyncCRB)
 
 	rootSyncSA := &corev1.ServiceAccount{}
@@ -233,10 +233,8 @@ func validateRepoSyncDependencies(nt *nomostest.NT, ns, rsName string) []client.
 	// only happens when upgrading from a very old unsupported version.
 
 	repoSyncRB := &rbacv1.RoleBinding{}
-	setNN(repoSyncRB, client.ObjectKey{
-		Name:      controllers.RepoSyncPermissionsName(),
-		Namespace: ns,
-	})
+	repoSyncRB.Name = controllers.RepoSyncBaseRoleBindingName
+	repoSyncRB.Namespace = ns
 	repoSyncDependencies = append(repoSyncDependencies, repoSyncRB)
 
 	repoSyncSA := &corev1.ServiceAccount{}
@@ -268,6 +266,7 @@ func validateRepoSyncDependencies(nt *nomostest.NT, ns, rsName string) []client.
 		err := nt.Validate(obj.GetName(), obj.GetNamespace(), obj)
 		require.NoError(nt.T, err)
 	}
+
 	return repoSyncDependencies
 }
 

e2e/testcases/root_sync_test.go

@@ -69,7 +69,7 @@ func TestDeleteRootSyncAndRootSyncV1Alpha1(t *testing.T) {
 		saName := core.RootReconcilerName(rs.Name)
 		errs = multierr.Append(errs, nt.ValidateNotFound(saName, configsync.ControllerNamespace, fake.ServiceAccountObject(saName)))
 		// validate Root Reconciler ClusterRoleBinding is no longer present.
-		errs = multierr.Append(errs, nt.ValidateNotFound(controllers.RootSyncPermissionsName(nomostest.DefaultRootReconcilerName), configsync.ControllerNamespace, fake.ClusterRoleBindingObject()))
+		errs = multierr.Append(errs, nt.ValidateNotFound(controllers.RootSyncLegacyClusterRoleBindingName, configsync.ControllerNamespace, fake.ClusterRoleBindingObject()))
 		return errs
 	})
 	if err != nil {

manifests/base/kustomization.yaml

@@ -17,7 +17,8 @@ resources:
 - ../cluster-registry-crd.yaml
 - ../container-default-limits.yaml
 - ../namespace-selector-crd.yaml
-- ../ns-reconciler-cluster-role.yaml
+- ../ns-reconciler-base-cluster-role.yaml
+- ../root-reconciler-base-cluster-role.yaml
 - ../otel-agent-cm.yaml
 - ../reconciler-manager-service-account.yaml
 - ../reposync-crd.yaml