From 3811640aff04c76028259859796e2deb8589be5b Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 23 Apr 2024 15:18:45 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Cluster=20Provider=20and=20cluster-?= =?UTF-8?q?aware=20controllers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vince Prignano Signed-off-by: Vince Prignano Signed-off-by: Dr. Stefan Schimanski --- examples/fleet-namespace/cache.go | 204 ++++++++++++++ examples/fleet-namespace/client.go | 91 +++++++ examples/fleet-namespace/cluster.go | 55 ++++ examples/fleet-namespace/go.mod | 69 +++++ examples/fleet-namespace/go.sum | 192 +++++++++++++ examples/fleet-namespace/main.go | 253 ++++++++++++++++++ examples/fleet/go.mod | 76 ++++++ examples/fleet/go.sum | 218 +++++++++++++++ examples/fleet/main.go | 210 +++++++++++++++ pkg/builder/controller.go | 179 +++++++++---- pkg/builder/controller_test.go | 173 +++++++++++- pkg/cache/cache.go | 18 +- pkg/cache/cache_test.go | 7 +- pkg/cache/informer_cache.go | 38 +-- pkg/cache/internal/informers.go | 101 +++++-- pkg/client/interfaces.go | 3 + pkg/cluster/cluster.go | 49 +++- pkg/cluster/internal.go | 9 + pkg/cluster/provider.go | 62 +++++ pkg/config/controller.go | 12 + pkg/controller/controller.go | 11 + pkg/controller/controller_test.go | 8 +- pkg/controller/example_test.go | 5 + pkg/controller/multicluster.go | 106 ++++++++ pkg/handler/cluster.go | 82 ++++++ pkg/internal/controller/controller.go | 3 + .../testing/controlplane/apiserver.go | 3 + pkg/internal/testing/process/process_test.go | 3 +- pkg/manager/internal.go | 242 ++++++++++++++++- pkg/manager/manager.go | 34 ++- pkg/manager/manager_test.go | 148 +++++++++- pkg/manager/runnable_group.go | 9 + pkg/reconcile/reconcile.go | 13 + 33 files changed, 2530 insertions(+), 156 deletions(-) create mode 100644 examples/fleet-namespace/cache.go create mode 100644 examples/fleet-namespace/client.go create mode 100644 examples/fleet-namespace/cluster.go create mode 100644 examples/fleet-namespace/go.mod create mode 100644 examples/fleet-namespace/go.sum create mode 100644 examples/fleet-namespace/main.go create mode 100644 examples/fleet/go.mod create mode 100644 examples/fleet/go.sum create mode 100644 examples/fleet/main.go create mode 100644 pkg/cluster/provider.go create mode 100644 pkg/controller/multicluster.go create mode 100644 pkg/handler/cluster.go diff --git a/examples/fleet-namespace/cache.go b/examples/fleet-namespace/cache.go new file mode 100644 index 0000000000..5c3eb0b877 --- /dev/null +++ b/examples/fleet-namespace/cache.go @@ -0,0 +1,204 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + toolscache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ClusterNameIndex = "cluster/name" + ClusterIndex = "cluster" +) + +var _ cache.Cache = &NamespacedCache{} + +type NamespacedCache struct { + clusterName string + cache.Cache +} + +func (c *NamespacedCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Namespace != "default" { + return apierrors.NewNotFound(schema.GroupResource{}, key.Name) + } + key.Namespace = c.clusterName + if err := c.Cache.Get(ctx, key, obj, opts...); err != nil { + return err + } + obj.SetNamespace("default") + return nil +} + +func (c *NamespacedCache) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + var listOpts client.ListOptions + for _, o := range opts { + o.ApplyToList(&listOpts) + } + + switch { + case listOpts.FieldSelector != nil: + reqs := listOpts.FieldSelector.Requirements() + flds := make(map[string]string, len(reqs)) + for i := range reqs { + flds[fmt.Sprintf("cluster/%s", reqs[i].Field)] = fmt.Sprintf("%s/%s", c.clusterName, reqs[i].Value) + } + opts = append(opts, client.MatchingFields(flds)) + case listOpts.Namespace != "": + opts = append(opts, client.MatchingFields(map[string]string{ClusterIndex: c.clusterName})) + if c.clusterName == "*" { + listOpts.Namespace = "" + } else if listOpts.Namespace == "default" { + listOpts.Namespace = c.clusterName + } + default: + opts = append(opts, client.MatchingFields(map[string]string{ClusterIndex: c.clusterName})) + } + + if err := c.Cache.List(ctx, list, opts...); err != nil { + return err + } + + return meta.EachListItem(list, func(obj runtime.Object) error { + obj.(client.Object).SetNamespace("default") + return nil + }) +} + +func (c *NamespacedCache) GetInformer(ctx context.Context, obj client.Object, opts ...cache.InformerGetOption) (cache.Informer, error) { + inf, err := c.Cache.GetInformer(ctx, obj, opts...) + if err != nil { + return nil, err + } + return &ScopedInformer{clusterName: c.clusterName, Informer: inf}, nil +} + +func (c *NamespacedCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...cache.InformerGetOption) (cache.Informer, error) { + inf, err := c.Cache.GetInformerForKind(ctx, gvk, opts...) + if err != nil { + return nil, err + } + return &ScopedInformer{clusterName: c.clusterName, Informer: inf}, nil +} + +func (c *NamespacedCache) RemoveInformer(ctx context.Context, obj client.Object) error { + return errors.New("informer cannot be removed from scoped cache") +} + +type ScopedInformer struct { + clusterName string + cache.Informer +} + +func (i *ScopedInformer) AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) { + return i.Informer.AddEventHandler(toolscache.ResourceEventHandlerDetailedFuncs{ + AddFunc: func(obj interface{}, isInInitialList bool) { + cobj := obj.(client.Object) + if cobj.GetNamespace() == i.clusterName { + cobj := cobj.DeepCopyObject().(client.Object) + cobj.SetNamespace("default") + handler.OnAdd(cobj, isInInitialList) + } + }, + UpdateFunc: func(oldObj, newObj interface{}) { + cobj := newObj.(client.Object) + if cobj.GetNamespace() == i.clusterName { + cobj := cobj.DeepCopyObject().(client.Object) + cobj.SetNamespace("default") + handler.OnUpdate(oldObj, cobj) + } + }, + DeleteFunc: func(obj interface{}) { + if tombStone, ok := obj.(toolscache.DeletedFinalStateUnknown); ok { + obj = tombStone.Obj + } + cobj := obj.(client.Object) + if cobj.GetNamespace() == i.clusterName { + cobj := cobj.DeepCopyObject().(client.Object) + cobj.SetNamespace("default") + handler.OnDelete(cobj) + } + }, + }) +} + +func (i *ScopedInformer) AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error) { + return i.Informer.AddEventHandlerWithResyncPeriod(toolscache.ResourceEventHandlerDetailedFuncs{ + AddFunc: func(obj interface{}, isInInitialList bool) { + cobj := obj.(client.Object) + if cobj.GetNamespace() == i.clusterName { + cobj := cobj.DeepCopyObject().(client.Object) + cobj.SetNamespace("default") + handler.OnAdd(cobj, isInInitialList) + } + }, + UpdateFunc: func(oldObj, newObj interface{}) { + cobj := newObj.(client.Object) + if cobj.GetNamespace() == i.clusterName { + cobj := cobj.DeepCopyObject().(client.Object) + cobj.SetNamespace("default") + handler.OnUpdate(oldObj, cobj) + } + }, + DeleteFunc: func(obj interface{}) { + if tombStone, ok := obj.(toolscache.DeletedFinalStateUnknown); ok { + obj = tombStone.Obj + } + cobj := obj.(client.Object) + if cobj.GetNamespace() == i.clusterName { + cobj := cobj.DeepCopyObject().(client.Object) + cobj.SetNamespace("default") + handler.OnDelete(cobj) + } + }, + }, resyncPeriod) +} + +func (i *ScopedInformer) AddIndexers(indexers toolscache.Indexers) error { + return errors.New("indexes cannot be added to scoped informers") +} + +type NamespaceScopeableCache struct { + cache.Cache +} + +func (f *NamespaceScopeableCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + return f.IndexField(ctx, obj, "cluster/"+field, func(obj client.Object) []string { + keys := extractValue(obj) + withCluster := make([]string, len(keys)*2) + for i, key := range keys { + withCluster[i] = fmt.Sprintf("%s/%s", obj.GetNamespace(), key) + withCluster[i+len(keys)] = fmt.Sprintf("*/%s", key) + } + return withCluster + }) +} + +func (f *NamespaceScopeableCache) Start(ctx context.Context) error { + return nil // no-op as this is shared +} diff --git a/examples/fleet-namespace/client.go b/examples/fleet-namespace/client.go new file mode 100644 index 0000000000..143ae6627f --- /dev/null +++ b/examples/fleet-namespace/client.go @@ -0,0 +1,91 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ client.Client = &NamespacedClient{} + +type NamespacedClient struct { + clusterName string + client.Client +} + +func (n *NamespacedClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Namespace != "default" { + return apierrors.NewNotFound(schema.GroupResource{}, key.Name) + } + key.Namespace = n.clusterName + if err := n.Client.Get(ctx, key, obj, opts...); err != nil { + return err + } + obj.SetNamespace("default") + return nil +} + +func (n *NamespacedClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + var copts client.ListOptions + for _, o := range opts { + o.ApplyToList(&copts) + } + if copts.Namespace != "default" { + return apierrors.NewNotFound(schema.GroupResource{}, copts.Namespace) + } + if err := n.Client.List(ctx, list, append(opts, client.InNamespace(n.clusterName))...); err != nil { + return err + } + return meta.EachListItem(list, func(obj runtime.Object) error { + obj.(client.Object).SetNamespace("default") + return nil + }) +} + +func (n *NamespacedClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + panic("implement me") +} + +func (n *NamespacedClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + panic("implement me") +} + +func (n *NamespacedClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + panic("implement me") +} + +func (n *NamespacedClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + panic("implement me") +} + +func (n *NamespacedClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + panic("implement me") +} + +func (n *NamespacedClient) Status() client.SubResourceWriter { + panic("implement me") +} + +func (n *NamespacedClient) SubResource(subResource string) client.SubResourceClient { + panic("implement me") +} diff --git a/examples/fleet-namespace/cluster.go b/examples/fleet-namespace/cluster.go new file mode 100644 index 0000000000..ce012cfeff --- /dev/null +++ b/examples/fleet-namespace/cluster.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" +) + +type NamespacedCluster struct { + clusterName string + cluster.Cluster +} + +func (c *NamespacedCluster) Name() string { + return c.clusterName +} + +func (c *NamespacedCluster) GetCache() cache.Cache { + return &NamespacedCache{clusterName: c.clusterName, Cache: c.Cluster.GetCache()} +} + +func (c *NamespacedCluster) GetClient() client.Client { + return &NamespacedClient{clusterName: c.clusterName, Client: c.Cluster.GetClient()} +} + +func (c *NamespacedCluster) GetEventRecorderFor(name string) record.EventRecorder { + panic("implement me") +} + +func (c *NamespacedCluster) GetAPIReader() client.Reader { + return c.GetClient() +} + +func (c *NamespacedCluster) Start(ctx context.Context) error { + return nil // no-op as this is shared +} diff --git a/examples/fleet-namespace/go.mod b/examples/fleet-namespace/go.mod new file mode 100644 index 0000000000..df1b54da0a --- /dev/null +++ b/examples/fleet-namespace/go.mod @@ -0,0 +1,69 @@ +module sigs.k8s.io/controller-runtime/examples/fleet + +go 1.22.0 + +toolchain go1.22.1 + +replace sigs.k8s.io/controller-runtime => ../.. + +require ( + golang.org/x/sync v0.6.0 + k8s.io/api v0.30.0-beta.0 + k8s.io/apimachinery v0.30.0-beta.0 + k8s.io/client-go v0.30.0-beta.0 + k8s.io/klog/v2 v2.120.1 + sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.30.0-beta.0 // indirect + k8s.io/component-base v0.30.0-beta.0 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/examples/fleet-namespace/go.sum b/examples/fleet-namespace/go.sum new file mode 100644 index 0000000000..560ea954c4 --- /dev/null +++ b/examples/fleet-namespace/go.sum @@ -0,0 +1,192 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.17.0 h1:kdnunFXpBjbzN56hcJHrXZ8M+LOkenKA7NnBzTNigTI= +github.com/onsi/ginkgo/v2 v2.17.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.30.0-beta.0 h1:5nsH5CjCcgbHxWigNtvMu3rulycLWPqrlDe5tSonVQI= +k8s.io/api v0.30.0-beta.0/go.mod h1:A74Wh+vOyYXQS7SbhLXPF3rW0+CQP078Sqn+yuHQV8Y= +k8s.io/apiextensions-apiserver v0.30.0-beta.0 h1:jQQjaL42qHYqpERMmgWMk6R6mTle3DV9k2RweaFT1Uw= +k8s.io/apiextensions-apiserver v0.30.0-beta.0/go.mod h1:vQzYpGPmkfiBCjHoZii1qlmJXkTEA2oi/Q/HFD/JJRA= +k8s.io/apimachinery v0.30.0-beta.0 h1:/gaNLWP5ynEG0ExJ+4w2YCj5/L4MU66RsWEAKciy0/g= +k8s.io/apimachinery v0.30.0-beta.0/go.mod h1:wEJvNDlfxMRaMhyv38SIHIEC9hah/xuzqUUhxIyUv7Y= +k8s.io/client-go v0.30.0-beta.0 h1:9K7+KFX7IuacC0lvMyxRBAx6rEiFfhWYo+AN919XWp4= +k8s.io/client-go v0.30.0-beta.0/go.mod h1:HFM/neoED2h1OCm5AERy1PmCb3etIgrfIbcDbUqfDQ8= +k8s.io/component-base v0.30.0-beta.0 h1:BrmAW/HLVhLUi9lpLJKKsb0Xqn8KX/5ez56/lVzgWXg= +k8s.io/component-base v0.30.0-beta.0/go.mod h1:jz7Tz00FFNpjMBCJ1X4CTLIGAwmHjlmTKB25vPcnvuI= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/examples/fleet-namespace/main.go b/examples/fleet-namespace/main.go new file mode 100644 index 0000000000..06facec1c7 --- /dev/null +++ b/examples/fleet-namespace/main.go @@ -0,0 +1,253 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + + "golang.org/x/sync/errgroup" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/rest" + toolscache "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + cache "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func init() { + ctrl.SetLogger(klog.Background()) +} + +func main() { + entryLog := log.Log.WithName("entrypoint") + ctx := signals.SetupSignalHandler() + + testEnv := &envtest.Environment{} + cfg, err := testEnv.Start() + if err != nil { + entryLog.Error(err, "failed to start local environment") + os.Exit(1) + } + defer func() { + if testEnv == nil { + return + } + if err := testEnv.Stop(); err != nil { + entryLog.Error(err, "failed to stop local environment") + os.Exit(1) + } + }() + + // Test fixtures + cli, err := client.New(cfg, client.Options{}) + if err != nil { + entryLog.Error(err, "failed to create client") + os.Exit(1) + } + runtime.Must(cli.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "zoo"}})) + runtime.Must(cli.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "zoo", Name: "elephant"}})) + runtime.Must(cli.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "zoo", Name: "lion"}})) + runtime.Must(cli.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "jungle"}})) + runtime.Must(cli.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "jungle", Name: "monkey"}})) + runtime.Must(cli.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "island"}})) + runtime.Must(cli.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "island", Name: "bird"}})) + + entryLog.Info("Setting up provider") + cl, err := cluster.New(cfg, func(options *cluster.Options) { + options.Cache.AdditionalDefaultIndexes = map[string]client.IndexerFunc{ + ClusterNameIndex: func(obj client.Object) []string { + return []string{ + fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName()), + fmt.Sprintf("%s/%s", "*", obj.GetName()), + } + }, + ClusterIndex: func(obj client.Object) []string { + return []string{obj.GetNamespace()} + }, + } + }) + if err != nil { + entryLog.Error(err, "unable to set up provider") + os.Exit(1) + } + provider := &NamespacedClusterProvider{Cluster: cl} + + // Setup a cluster-aware Manager, watching the clusters (= namespaces) through + // the cluster provider. + entryLog.Info("Setting up cluster-aware manager") + mgr, err := manager.New(cfg, manager.Options{ + NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) { + // wrap cache to turn IndexField calls into cluster-scoped indexes. + return &NamespaceScopeableCache{Cache: cl.GetCache()}, nil + }, + ExperimentalClusterProvider: provider, + }) + if err != nil { + entryLog.Error(err, "unable to set up overall controller manager") + os.Exit(1) + } + + if err := builder.ControllerManagedBy(mgr). + For(&corev1.ConfigMap{}).Complete(reconcile.Func( + func(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx) + + cl, err := mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return reconcile.Result{}, err + } + cli := cl.GetClient() + + // Retrieve the service account from the namespace. + cm := &corev1.ConfigMap{} + if err := cli.Get(ctx, req.NamespacedName, cm); err != nil { + return reconcile.Result{}, err + } + log.Info("Reconciling configmap", "cluster", req.ClusterName, "ns", req.Namespace, "name", cm.Name, "uuid", cm.UID) + + return ctrl.Result{}, nil + }, + )); err != nil { + entryLog.Error(err, "unable to set up controller") + os.Exit(1) + } + + entryLog.Info("Starting provider") + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + if err := ignoreCanceled(provider.Start(ctx)); err != nil { + return fmt.Errorf("failed to start provider: %w", err) + } + return nil + }) + + entryLog.Info("Starting cluster-aware manager") + g.Go(func() error { + if err := ignoreCanceled(mgr.Start(ctx)); err != nil { + return fmt.Errorf("unable to run cluster-aware manager: %w", err) + } + return nil + }) + + if err := g.Wait(); err != nil { + entryLog.Error(err, "failed to run managers") + os.Exit(1) + } +} + +// NamespacedClusterProvider is a cluster provider that represents each namespace +// as a dedicated cluster with only a "default" namespace. It maps each namespace +// to "default" and vice versa, simulating a multi-cluster setup. It uses one +// informer to watch objects for all namespaces. +type NamespacedClusterProvider struct { + cluster.Cluster +} + +func (p *NamespacedClusterProvider) Get(ctx context.Context, clusterName string, opts ...cluster.Option) (cluster.Cluster, error) { + ns := &corev1.Namespace{} + if err := p.Cluster.GetCache().Get(ctx, client.ObjectKey{Name: clusterName}, ns); err != nil { + return nil, err + } + + return &NamespacedCluster{clusterName: clusterName, Cluster: p.Cluster}, nil +} + +func (p *NamespacedClusterProvider) List(ctx context.Context) ([]string, error) { + nss := &corev1.NamespaceList{} + if err := p.Cluster.GetCache().List(ctx, nss); err != nil { + return nil, err + } + + res := make([]string, 0, len(nss.Items)) + for _, ns := range nss.Items { + res = append(res, ns.Name) + } + return res, nil +} + +func (p *NamespacedClusterProvider) Watch(ctx context.Context) (cluster.Watcher, error) { + inf, err := p.Cluster.GetCache().GetInformer(ctx, &corev1.Namespace{}) + if err != nil { + return nil, err + } + return &NamespaceWatcher{inf: inf, ch: make(chan cluster.WatchEvent)}, nil +} + +type NamespaceWatcher struct { + inf cache.Informer + init sync.Once + ch chan cluster.WatchEvent + reg toolscache.ResourceEventHandlerRegistration +} + +func (w *NamespaceWatcher) Stop() { + if w.reg != nil { + _ = w.inf.RemoveEventHandler(w.reg) + } + close(w.ch) +} + +func (w *NamespaceWatcher) ResultChan() <-chan cluster.WatchEvent { + w.init.Do(func() { + w.reg, _ = w.inf.AddEventHandler(toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + ns := obj.(*corev1.Namespace) + w.ch <- cluster.WatchEvent{ + Type: watch.Added, + ClusterName: ns.Name, + } + }, + DeleteFunc: func(obj interface{}) { + ns := obj.(*corev1.Namespace) + w.ch <- cluster.WatchEvent{ + Type: watch.Deleted, + ClusterName: ns.Name, + } + }, + UpdateFunc: func(oldObj, newObj interface{}) { + ns := newObj.(*corev1.Namespace) + w.ch <- cluster.WatchEvent{ + Type: watch.Modified, + ClusterName: ns.Name, + } + }, + }) + }) + return w.ch +} + +func ignoreCanceled(err error) error { + if errors.Is(err, context.Canceled) { + return nil + } + return err +} diff --git a/examples/fleet/go.mod b/examples/fleet/go.mod new file mode 100644 index 0000000000..7c3d8ca221 --- /dev/null +++ b/examples/fleet/go.mod @@ -0,0 +1,76 @@ +module sigs.k8s.io/controller-runtime/examples/fleet + +go 1.22.0 + +toolchain go1.22.1 + +replace sigs.k8s.io/controller-runtime => ../.. + +require ( + k8s.io/api v0.30.0-beta.0 + k8s.io/apimachinery v0.30.0-beta.0 + k8s.io/client-go v0.30.0-beta.0 + k8s.io/klog/v2 v2.120.1 + sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 + sigs.k8s.io/kind v0.17.0 +) + +require ( + github.com/BurntSushi/toml v1.0.0 // indirect + github.com/alessio/shellescape v1.4.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.30.0-beta.0 // indirect + k8s.io/component-base v0.30.0-beta.0 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/examples/fleet/go.sum b/examples/fleet/go.sum new file mode 100644 index 0000000000..78ba33a163 --- /dev/null +++ b/examples/fleet/go.sum @@ -0,0 +1,218 @@ +github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/ginkgo/v2 v2.17.0 h1:kdnunFXpBjbzN56hcJHrXZ8M+LOkenKA7NnBzTNigTI= +github.com/onsi/ginkgo/v2 v2.17.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.30.0-beta.0 h1:5nsH5CjCcgbHxWigNtvMu3rulycLWPqrlDe5tSonVQI= +k8s.io/api v0.30.0-beta.0/go.mod h1:A74Wh+vOyYXQS7SbhLXPF3rW0+CQP078Sqn+yuHQV8Y= +k8s.io/apiextensions-apiserver v0.30.0-beta.0 h1:jQQjaL42qHYqpERMmgWMk6R6mTle3DV9k2RweaFT1Uw= +k8s.io/apiextensions-apiserver v0.30.0-beta.0/go.mod h1:vQzYpGPmkfiBCjHoZii1qlmJXkTEA2oi/Q/HFD/JJRA= +k8s.io/apimachinery v0.30.0-beta.0 h1:/gaNLWP5ynEG0ExJ+4w2YCj5/L4MU66RsWEAKciy0/g= +k8s.io/apimachinery v0.30.0-beta.0/go.mod h1:wEJvNDlfxMRaMhyv38SIHIEC9hah/xuzqUUhxIyUv7Y= +k8s.io/client-go v0.30.0-beta.0 h1:9K7+KFX7IuacC0lvMyxRBAx6rEiFfhWYo+AN919XWp4= +k8s.io/client-go v0.30.0-beta.0/go.mod h1:HFM/neoED2h1OCm5AERy1PmCb3etIgrfIbcDbUqfDQ8= +k8s.io/component-base v0.30.0-beta.0 h1:BrmAW/HLVhLUi9lpLJKKsb0Xqn8KX/5ez56/lVzgWXg= +k8s.io/component-base v0.30.0-beta.0/go.mod h1:jz7Tz00FFNpjMBCJ1X4CTLIGAwmHjlmTKB25vPcnvuI= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kind v0.17.0 h1:CScmGz/wX66puA06Gj8OZb76Wmk7JIjgWf5JDvY7msM= +sigs.k8s.io/kind v0.17.0/go.mod h1:Qqp8AiwOlMZmJWs37Hgs31xcbiYXjtXlRBSftcnZXQk= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/examples/fleet/main.go b/examples/fleet/main.go new file mode 100644 index 0000000000..4e950969b7 --- /dev/null +++ b/examples/fleet/main.go @@ -0,0 +1,210 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "os" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + kind "sigs.k8s.io/kind/pkg/cluster" +) + +func init() { + ctrl.SetLogger(klog.Background()) +} + +func main() { + entryLog := log.Log.WithName("entrypoint") + + testEnv := &envtest.Environment{} + cfg, err := testEnv.Start() + if err != nil { + entryLog.Error(err, "failed to start local environment") + os.Exit(1) + } + defer func() { + if testEnv == nil { + return + } + if err := testEnv.Stop(); err != nil { + entryLog.Error(err, "failed to stop local environment") + os.Exit(1) + } + }() + + // Setup a Manager + entryLog.Info("Setting up manager") + mgr, err := manager.New( + cfg, + manager.Options{ExperimentalClusterProvider: &KindClusterProvider{}}, + ) + if err != nil { + entryLog.Error(err, "unable to set up overall controller manager") + os.Exit(1) + } + + builder.ControllerManagedBy(mgr). + For(&corev1.Pod{}).Complete(reconcile.Func( + func(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx) + + cluster, err := mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return reconcile.Result{}, err + } + client := cluster.GetClient() + + // Retrieve the pod from the cluster. + pod := &corev1.Pod{} + if err := client.Get(ctx, req.NamespacedName, pod); err != nil { + return reconcile.Result{}, err + } + log.Info("Reconciling pod", "name", pod.Name, "uuid", pod.UID) + + // Print any annotations that start with fleet. + for k, v := range pod.Labels { + if strings.HasPrefix(k, "fleet-") { + log.Info("Detected fleet annotation!", "key", k, "value", v) + } + } + + return ctrl.Result{}, nil + }, + )) + + entryLog.Info("Starting manager") + if err := mgr.Start(signals.SetupSignalHandler()); err != nil { + entryLog.Error(err, "unable to run manager") + os.Exit(1) + } +} + +// KindClusterProvider is a cluster provider that works with a local Kind instance. +type KindClusterProvider struct{} + +func (k *KindClusterProvider) Get(ctx context.Context, clusterName string, opts ...cluster.Option) (cluster.Cluster, error) { + provider := kind.NewProvider() + kubeconfig, err := provider.KubeConfig(clusterName, false) + if err != nil { + return nil, err + } + // Parse the kubeconfig into a rest.Config. + cfg, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeconfig)) + if err != nil { + return nil, err + } + return cluster.New(cfg, opts...) +} + +func (k *KindClusterProvider) List(ctx context.Context) ([]string, error) { + provider := kind.NewProvider() + list, err := provider.List() + if err != nil { + return nil, err + } + res := make([]string, 0, len(list)) + for _, cluster := range list { + if !strings.HasPrefix(cluster, "fleet-") { + continue + } + res = append(res, cluster) + } + return res, nil +} + +func (k *KindClusterProvider) Watch(_ context.Context) (cluster.Watcher, error) { + return &KindWatcher{ch: make(chan cluster.WatchEvent)}, nil +} + +type KindWatcher struct { + init sync.Once + wg sync.WaitGroup + ch chan cluster.WatchEvent + cancel context.CancelFunc +} + +func (k *KindWatcher) Stop() { + if k.cancel != nil { + k.cancel() + } + k.wg.Wait() + close(k.ch) +} + +func (k *KindWatcher) ResultChan() <-chan cluster.WatchEvent { + k.init.Do(func() { + ctx, cancel := context.WithCancel(context.Background()) + k.cancel = cancel + set := sets.New[string]() + k.wg.Add(1) + go func() { + defer k.wg.Done() + for { + select { + case <-time.After(2 * time.Second): + provider := kind.NewProvider() + list, err := provider.List() + if err != nil { + klog.Error(err) + continue + } + newSet := sets.New(list...) + // Check for new clusters. + for _, cl := range newSet.Difference(set).UnsortedList() { + if !strings.HasPrefix(cl, "fleet-") { + continue + } + k.ch <- cluster.WatchEvent{ + Type: watch.Added, + ClusterName: cl, + } + } + // Check for deleted clusters. + for _, cl := range set.Difference(newSet).UnsortedList() { + if !strings.HasPrefix(cl, "fleet-") { + continue + } + k.ch <- cluster.WatchEvent{ + Type: watch.Deleted, + ClusterName: cl, + } + } + set = newSet + case <-ctx.Done(): + return + } + } + }() + }) + return k.ch +} diff --git a/pkg/builder/controller.go b/pkg/builder/controller.go index 2c0063a837..a60c997304 100644 --- a/pkg/builder/controller.go +++ b/pkg/builder/controller.go @@ -17,6 +17,7 @@ limitations under the License. package builder import ( + "context" "errors" "fmt" "strings" @@ -24,7 +25,9 @@ import ( "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -37,7 +40,7 @@ import ( ) // Supporting mocking out functions for testing. -var newController = controller.New +var newController = controller.NewUnmanaged var getGvk = apiutil.GVKForObject // project represents other forms that we can use to @@ -51,17 +54,24 @@ const ( projectAsMetadata ) -// Builder builds a Controller. -type Builder struct { +var _ controller.ClusterWatcher = &clusterWatcher{} + +// clusterWatcher sets up watches between a cluster and a controller. +type clusterWatcher struct { + ctrl controller.Controller forInput ForInput ownsInput []OwnsInput - rawSources []source.Source watchesInput []WatchesInput - mgr manager.Manager globalPredicates []predicate.Predicate - ctrl controller.Controller - ctrlOptions controller.Options - name string +} + +// Builder builds a Controller. +type Builder struct { + clusterWatcher + rawSources []source.Source + mgr manager.Manager + ctrlOptions controller.Options + name string } // ControllerManagedBy returns a new controller builder that will be started by the provided Manager. @@ -144,7 +154,6 @@ func (blder *Builder) Watches(object client.Object, eventHandler handler.EventHa } blder.watchesInput = append(blder.watchesInput, input) - return blder } @@ -252,16 +261,25 @@ func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, erro return nil, err } + ctrl := blder.ctrl + if *blder.ctrlOptions.EngageWithProviderClusters { + // wrap as cluster.AwareRunnable to be engaged with provider clusters on demand + ctrl = controller.NewMultiClusterController(ctrl, &blder.clusterWatcher) + } + if err := blder.mgr.Add(ctrl); err != nil { + return nil, err + } + return blder.ctrl, nil } -func (blder *Builder) project(obj client.Object, proj objectProjection) (client.Object, error) { +func project(cl cluster.Cluster, obj client.Object, proj objectProjection) (client.Object, error) { switch proj { case projectAsNormal: return obj, nil case projectAsMetadata: metaObj := &metav1.PartialObjectMetadata{} - gvk, err := getGvk(obj, blder.mgr.GetScheme()) + gvk, err := getGvk(obj, cl.GetScheme()) if err != nil { return nil, fmt.Errorf("unable to determine GVK of %T for a metadata-only watch: %w", obj, err) } @@ -272,28 +290,25 @@ func (blder *Builder) project(obj client.Object, proj objectProjection) (client. } } -func (blder *Builder) doWatch() error { +func (cc *clusterWatcher) Watch(ctx context.Context, cl cluster.Cluster) error { // Reconcile type - if blder.forInput.object != nil { - obj, err := blder.project(blder.forInput.object, blder.forInput.objectProjection) + if cc.forInput.object != nil { + obj, err := project(cl, cc.forInput.object, cc.forInput.objectProjection) if err != nil { return err } hdler := &handler.EnqueueRequestForObject{} - allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...) - allPredicates = append(allPredicates, blder.forInput.predicates...) - src := source.Kind(blder.mgr.GetCache(), obj, hdler, allPredicates...) - if err := blder.ctrl.Watch(src); err != nil { + allPredicates := append([]predicate.Predicate(nil), cc.globalPredicates...) + allPredicates = append(allPredicates, cc.forInput.predicates...) + src := &ctxBoundedSyncingSource{ctx: ctx, src: source.Kind(cl.GetCache(), obj, handler.ForCluster(cl.Name(), hdler), allPredicates...)} + if err := cc.ctrl.Watch(src); err != nil { return err } } // Watches the managed types - if len(blder.ownsInput) > 0 && blder.forInput.object == nil { - return errors.New("Owns() can only be used together with For()") - } - for _, own := range blder.ownsInput { - obj, err := blder.project(own.object, own.objectProjection) + for _, own := range cc.ownsInput { + obj, err := project(cl, own.object, own.objectProjection) if err != nil { return err } @@ -302,33 +317,54 @@ func (blder *Builder) doWatch() error { opts = append(opts, handler.OnlyControllerOwner()) } hdler := handler.EnqueueRequestForOwner( - blder.mgr.GetScheme(), blder.mgr.GetRESTMapper(), - blder.forInput.object, + cl.GetScheme(), cl.GetRESTMapper(), + cc.forInput.object, opts..., ) - allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...) + + allPredicates := append([]predicate.Predicate(nil), cc.globalPredicates...) allPredicates = append(allPredicates, own.predicates...) - src := source.Kind(blder.mgr.GetCache(), obj, hdler, allPredicates...) - if err := blder.ctrl.Watch(src); err != nil { + src := &ctxBoundedSyncingSource{ctx: ctx, src: source.Kind(cl.GetCache(), obj, handler.ForCluster(cl.Name(), hdler), allPredicates...)} + if err := cc.ctrl.Watch(src); err != nil { return err } } - // Do the watch requests - if len(blder.watchesInput) == 0 && blder.forInput.object == nil && len(blder.rawSources) == 0 { - return errors.New("there are no watches configured, controller will never get triggered. Use For(), Owns(), Watches() or WatchesRawSource() to set them up") - } - for _, w := range blder.watchesInput { - projected, err := blder.project(w.obj, w.objectProjection) + // Watches extra types + for _, w := range cc.watchesInput { + projected, err := project(cl, w.obj, w.objectProjection) if err != nil { return fmt.Errorf("failed to project for %T: %w", w.obj, err) } - allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...) + allPredicates := append([]predicate.Predicate(nil), cc.globalPredicates...) allPredicates = append(allPredicates, w.predicates...) - if err := blder.ctrl.Watch(source.Kind(blder.mgr.GetCache(), projected, w.handler, allPredicates...)); err != nil { + src := &ctxBoundedSyncingSource{ctx: ctx, src: source.Kind(cl.GetCache(), projected, handler.ForCluster(cl.Name(), w.handler), allPredicates...)} + if err := cc.ctrl.Watch(src); err != nil { + return err + } + } + + return nil +} + +func (blder *Builder) doWatch() error { + // Pre-checks for a valid configuration + if len(blder.ownsInput) > 0 && blder.forInput.object == nil { + return errors.New("Owns() can only be used together with For()") + } + if len(blder.watchesInput) == 0 && blder.forInput.object == nil && len(blder.rawSources) == 0 { + return errors.New("there are no watches configured, controller will never get triggered. Use For(), Owns(), Watches() or WatchesRawSource() to set them up") + } + if *blder.ctrlOptions.EngageWithProviderClusters && len(blder.rawSources) > 0 { + return errors.New("when using a cluster adapter, custom raw watches are not allowed") + } + + if *blder.ctrlOptions.EngageWithDefaultCluster { + if err := blder.Watch(unboundedContext, blder.mgr); err != nil { return err } } + for _, src := range blder.rawSources { if err := blder.ctrl.Watch(src); err != nil { return err @@ -350,12 +386,11 @@ func (blder *Builder) getControllerName(gvk schema.GroupVersionKind, hasGVK bool func (blder *Builder) doController(r reconcile.Reconciler) error { globalOpts := blder.mgr.GetControllerOptions() - ctrlOptions := blder.ctrlOptions - if ctrlOptions.Reconciler != nil && r != nil { + if blder.ctrlOptions.Reconciler != nil && r != nil { return errors.New("reconciler was set via WithOptions() and via Build() or Complete()") } - if ctrlOptions.Reconciler == nil { - ctrlOptions.Reconciler = r + if blder.ctrlOptions.Reconciler == nil { + blder.ctrlOptions.Reconciler = r } // Retrieve the GVK from the object we're reconciling @@ -371,17 +406,17 @@ func (blder *Builder) doController(r reconcile.Reconciler) error { } // Setup concurrency. - if ctrlOptions.MaxConcurrentReconciles == 0 && hasGVK { + if blder.ctrlOptions.MaxConcurrentReconciles == 0 && hasGVK { groupKind := gvk.GroupKind().String() if concurrency, ok := globalOpts.GroupKindConcurrency[groupKind]; ok && concurrency > 0 { - ctrlOptions.MaxConcurrentReconciles = concurrency + blder.ctrlOptions.MaxConcurrentReconciles = concurrency } } // Setup cache sync timeout. - if ctrlOptions.CacheSyncTimeout == 0 && globalOpts.CacheSyncTimeout > 0 { - ctrlOptions.CacheSyncTimeout = globalOpts.CacheSyncTimeout + if blder.ctrlOptions.CacheSyncTimeout == 0 && globalOpts.CacheSyncTimeout > 0 { + blder.ctrlOptions.CacheSyncTimeout = globalOpts.CacheSyncTimeout } controllerName, err := blder.getControllerName(gvk, hasGVK) @@ -390,7 +425,7 @@ func (blder *Builder) doController(r reconcile.Reconciler) error { } // Setup the logger. - if ctrlOptions.LogConstructor == nil { + if blder.ctrlOptions.LogConstructor == nil { log := blder.mgr.GetLogger().WithValues( "controller", controllerName, ) @@ -401,7 +436,7 @@ func (blder *Builder) doController(r reconcile.Reconciler) error { ) } - ctrlOptions.LogConstructor = func(req *reconcile.Request) logr.Logger { + blder.ctrlOptions.LogConstructor = func(req *reconcile.Request) logr.Logger { log := log if req != nil { if hasGVK { @@ -415,7 +450,57 @@ func (blder *Builder) doController(r reconcile.Reconciler) error { } } + // Default which clusters to engage with. + if blder.ctrlOptions.EngageWithDefaultCluster == nil { + blder.ctrlOptions.EngageWithDefaultCluster = globalOpts.EngageWithDefaultCluster + } + if blder.ctrlOptions.EngageWithProviderClusters == nil { + blder.ctrlOptions.EngageWithProviderClusters = globalOpts.EngageWithProviderClusters + } + if blder.ctrlOptions.EngageWithDefaultCluster == nil { + return errors.New("EngageWithDefaultCluster must not be nil") // should not happen due to defaulting + } + if blder.ctrlOptions.EngageWithProviderClusters == nil { + return errors.New("EngageWithProviderClusters must not be nil") // should not happen due to defaulting + } + if !*blder.ctrlOptions.EngageWithDefaultCluster && !*blder.ctrlOptions.EngageWithProviderClusters { + return errors.New("EngageWithDefaultCluster and EngageWithProviderClusters are both false, controller will never get triggered") + } + // Build the controller and return. - blder.ctrl, err = newController(controllerName, blder.mgr, ctrlOptions) + blder.ctrl, err = newController(controllerName, blder.mgr, blder.ctrlOptions) return err } + +// ctxBoundedSyncingSource implements source.SyncingSource and wraps the ctx +// passed to the methods into the life-cycle of another context, i.e. stop +// whenever one of the contexts is done. +type ctxBoundedSyncingSource struct { + ctx context.Context + src source.SyncingSource +} + +var unboundedContext context.Context = nil //nolint:revive // keep nil explicit for clarity. + +var _ source.SyncingSource = &ctxBoundedSyncingSource{} + +func (s *ctxBoundedSyncingSource) Start(ctx context.Context, q workqueue.RateLimitingInterface) error { + return s.src.Start(joinContexts(ctx, s.ctx), q) +} + +func (s *ctxBoundedSyncingSource) WaitForSync(ctx context.Context) error { + return s.src.WaitForSync(joinContexts(ctx, s.ctx)) +} + +func joinContexts(ctx, bound context.Context) context.Context { + if bound == unboundedContext { + return ctx + } + + ctx, cancel := context.WithCancel(ctx) + go func() { + defer cancel() + <-bound.Done() + }() + return ctx +} diff --git a/pkg/builder/controller_test.go b/pkg/builder/controller_test.go index 4ff576edad..2664f6c10d 100644 --- a/pkg/builder/controller_test.go +++ b/pkg/builder/controller_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" @@ -77,7 +78,7 @@ func (l *testLogger) WithName(name string) logr.LogSink { var _ = Describe("application", func() { BeforeEach(func() { - newController = controller.New + newController = controller.NewUnmanaged }) noop := reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { @@ -556,6 +557,146 @@ var _ = Describe("application", func() { }).Should(BeTrue()) }) }) + + Context("with cluster provider", func() { + It("should support watching across clusters", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + adapter := &fakeClusterProvider{ + clusterNameList: []string{ + "cluster1", + "cluster2", + }, + watch: make(chan cluster.WatchEvent), + } + mgr, err := manager.New(cfg, manager.Options{ExperimentalClusterProvider: adapter}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a custom namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-multi-cluster-", + }, + } + Expect(mgr.GetClient().Create(ctx, ns)).To(Succeed()) + + ch1 := make(chan reconcile.Request, 1) + ch2 := make(chan reconcile.Request, 1) + Expect( + ControllerManagedBy(mgr). + For(&appsv1.Deployment{}). + Owns(&appsv1.ReplicaSet{}). + Complete(reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + if req.Namespace != ns.Name { + return reconcile.Result{}, nil + } + + defer GinkgoRecover() + switch req.ClusterName { + case "cluster1": + ch1 <- req + case "cluster2": + ch2 <- req + default: + // Do nothing. + } + return reconcile.Result{}, nil + })), + ).To(Succeed()) + + By("Starting the manager") + go func() { + defer GinkgoRecover() + Expect(mgr.Start(ctx)).NotTo(HaveOccurred()) + }() + + cluster1, err := mgr.GetCluster(ctx, "cluster1") + Expect(err).NotTo(HaveOccurred()) + + By("Creating a deployment") + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deploy-multi-cluster", + Namespace: ns.Name, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"foo": "bar"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"foo": "bar"}}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + } + Expect(cluster1.GetClient().Create(ctx, dep)).To(Succeed()) + + By("Waiting for the Deployment Reconcile on both clusters") + Eventually(ch1).Should(Receive(Equal(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: dep.Name, + Namespace: dep.Namespace, + }, + ClusterName: "cluster1", + }))) + Eventually(ch2).Should(Receive(Equal(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: dep.Name, + Namespace: dep.Namespace, + }, + ClusterName: "cluster2", + }))) + + By("Creating a ReplicaSet") + // Expect a Reconcile when an Owned object is managedObjects. + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dep.Namespace, + Name: "rs-multi-cluster", + Labels: dep.Spec.Selector.MatchLabels, + OwnerReferences: []metav1.OwnerReference{ + { + Name: dep.Name, + Kind: "Deployment", + APIVersion: "apps/v1", + Controller: ptr.To(true), + UID: dep.UID, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: dep.Spec.Selector, + Template: dep.Spec.Template, + }, + } + Expect(err).NotTo(HaveOccurred()) + Expect(cluster1.GetClient().Create(ctx, rs)).To(Succeed()) + + By("Waiting for the Deployment Reconcile on both clusters") + Eventually(ch1).Should(Receive(Equal(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: dep.Name, + Namespace: dep.Namespace, + }, + ClusterName: "cluster1", + }))) + Eventually(ch2).Should(Receive(Equal(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: dep.Name, + Namespace: dep.Namespace, + }, + ClusterName: "cluster2", + }))) + }) + }) }) // newNonTypedOnlyCache returns a new cache that wraps the normal cache, @@ -695,3 +836,33 @@ type fakeType struct { func (*fakeType) GetObjectKind() schema.ObjectKind { return nil } func (*fakeType) DeepCopyObject() runtime.Object { return nil } + +type fakeClusterProvider struct { + clusterNameList []string + listErr error + + watch chan cluster.WatchEvent +} + +func (f *fakeClusterProvider) Get(ctx context.Context, clusterName string, opts ...cluster.Option) (cluster.Cluster, error) { + return cluster.New(testenv.Config, opts...) +} + +func (f *fakeClusterProvider) List(ctx context.Context) ([]string, error) { + return f.clusterNameList, f.listErr +} + +func (f *fakeClusterProvider) Watch(ctx context.Context) (cluster.Watcher, error) { + return &fakeLogicalWatcher{ch: f.watch}, nil +} + +type fakeLogicalWatcher struct { + ch chan cluster.WatchEvent +} + +func (f *fakeLogicalWatcher) Stop() { +} + +func (f *fakeLogicalWatcher) ResultChan() <-chan cluster.WatchEvent { + return f.ch +} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index f8e7405174..a22bdf60b4 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -205,6 +205,9 @@ type Options struct { // A typical usecase for this is to use TransformStripManagedFields // to reduce the caches memory usage. DefaultTransform toolscache.TransformFunc + // AdditionalDefaultIndexes are indexes that are added to every informer + // beyond the namespace-name and namespace ones. + AdditionalDefaultIndexes client.Indexers // DefaultWatchErrorHandler will be used to the WatchErrorHandler which is called // whenever ListAndWatch drops the connection with an error. @@ -226,8 +229,10 @@ type Options struct { // If unset, this will fall through to the Default* settings. ByObject map[client.Object]ByObject - // newInformer allows overriding of NewSharedIndexInformer for testing. - newInformer *func(toolscache.ListerWatcher, runtime.Object, time.Duration, toolscache.Indexers) toolscache.SharedIndexInformer + // NewInformer allows to override NewSharedIndexInformer. + // NOTE: LOW LEVEL PRIMITIVE! + // Only use a custom informer if you know what you are doing. + NewInformer func(toolscache.ListerWatcher, runtime.Object, time.Duration, toolscache.Indexers) toolscache.SharedIndexInformer } // ByObject offers more fine-grained control over the cache's ListWatch by object. @@ -395,10 +400,11 @@ func newCache(restConfig *rest.Config, opts Options) newCacheFunc { Label: config.LabelSelector, Field: config.FieldSelector, }, - Transform: config.Transform, - WatchErrorHandler: opts.DefaultWatchErrorHandler, - UnsafeDisableDeepCopy: ptr.Deref(config.UnsafeDisableDeepCopy, false), - NewInformer: opts.newInformer, + Transform: config.Transform, + WatchErrorHandler: opts.DefaultWatchErrorHandler, + UnsafeDisableDeepCopy: ptr.Deref(config.UnsafeDisableDeepCopy, false), + NewInformer: opts.NewInformer, + AdditionalDefaultIndexes: opts.AdditionalDefaultIndexes, }), readerFailOnMissingInformer: opts.ReaderFailOnMissingInformer, } diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index d6c1c4aae4..5ffcd5e24d 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -544,14 +544,9 @@ func NonBlockingGetTest(createCacheFunc func(config *rest.Config, opts cache.Opt Expect(err).NotTo(HaveOccurred()) By("creating the informer cache") - v := reflect.ValueOf(&opts).Elem() - newInformerField := v.FieldByName("newInformer") - newFakeInformer := func(_ kcache.ListerWatcher, _ runtime.Object, _ time.Duration, _ kcache.Indexers) kcache.SharedIndexInformer { + opts.NewInformer = func(_ kcache.ListerWatcher, _ runtime.Object, _ time.Duration, _ kcache.Indexers) kcache.SharedIndexInformer { return &controllertest.FakeInformer{Synced: false} } - reflect.NewAt(newInformerField.Type(), newInformerField.Addr().UnsafePointer()). - Elem(). - Set(reflect.ValueOf(&newFakeInformer)) informerCache, err = createCacheFunc(cfg, opts) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting for it to sync") diff --git a/pkg/cache/informer_cache.go b/pkg/cache/informer_cache.go index 091667b7fa..f31eea91f2 100644 --- a/pkg/cache/informer_cache.go +++ b/pkg/cache/informer_cache.go @@ -21,7 +21,6 @@ import ( "fmt" "strings" - apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -221,40 +220,5 @@ func (ic *informerCache) IndexField(ctx context.Context, obj client.Object, fiel } func indexByField(informer Informer, field string, extractValue client.IndexerFunc) error { - indexFunc := func(objRaw interface{}) ([]string, error) { - // TODO(directxman12): check if this is the correct type? - obj, isObj := objRaw.(client.Object) - if !isObj { - return nil, fmt.Errorf("object of type %T is not an Object", objRaw) - } - meta, err := apimeta.Accessor(obj) - if err != nil { - return nil, err - } - ns := meta.GetNamespace() - - rawVals := extractValue(obj) - var vals []string - if ns == "" { - // if we're not doubling the keys for the namespaced case, just create a new slice with same length - vals = make([]string, len(rawVals)) - } else { - // if we need to add non-namespaced versions too, double the length - vals = make([]string, len(rawVals)*2) - } - for i, rawVal := range rawVals { - // save a namespaced variant, so that we can ask - // "what are all the object matching a given index *in a given namespace*" - vals[i] = internal.KeyToNamespacedKey(ns, rawVal) - if ns != "" { - // if we have a namespace, also inject a special index key for listing - // regardless of the object namespace - vals[i+len(rawVals)] = internal.KeyToNamespacedKey("", rawVal) - } - } - - return vals, nil - } - - return informer.AddIndexers(cache.Indexers{internal.FieldIndexName(field): indexFunc}) + return informer.AddIndexers(cache.Indexers{internal.FieldIndexName(field): internal.IndexFunc(extractValue)}) } diff --git a/pkg/cache/internal/informers.go b/pkg/cache/internal/informers.go index cd8c6774ca..4e4bde3edc 100644 --- a/pkg/cache/internal/informers.go +++ b/pkg/cache/internal/informers.go @@ -36,29 +36,31 @@ import ( "k8s.io/client-go/metadata" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/internal/syncs" ) // InformersOpts configures an InformerMap. type InformersOpts struct { - HTTPClient *http.Client - Scheme *runtime.Scheme - Mapper meta.RESTMapper - ResyncPeriod time.Duration - Namespace string - NewInformer *func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer - Selector Selector - Transform cache.TransformFunc - UnsafeDisableDeepCopy bool - WatchErrorHandler cache.WatchErrorHandler + HTTPClient *http.Client + Scheme *runtime.Scheme + Mapper meta.RESTMapper + ResyncPeriod time.Duration + Namespace string + NewInformer func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer + AdditionalDefaultIndexes client.Indexers + Selector Selector + Transform cache.TransformFunc + UnsafeDisableDeepCopy bool + WatchErrorHandler cache.WatchErrorHandler } // NewInformers creates a new InformersMap that can create informers under the hood. func NewInformers(config *rest.Config, options *InformersOpts) *Informers { newInformer := cache.NewSharedIndexInformer if options.NewInformer != nil { - newInformer = *options.NewInformer + newInformer = options.NewInformer } return &Informers{ config: config, @@ -70,16 +72,17 @@ func NewInformers(config *rest.Config, options *InformersOpts) *Informers { Unstructured: make(map[schema.GroupVersionKind]*Cache), Metadata: make(map[schema.GroupVersionKind]*Cache), }, - codecs: serializer.NewCodecFactory(options.Scheme), - paramCodec: runtime.NewParameterCodec(options.Scheme), - resync: options.ResyncPeriod, - startWait: make(chan struct{}), - namespace: options.Namespace, - selector: options.Selector, - transform: options.Transform, - unsafeDisableDeepCopy: options.UnsafeDisableDeepCopy, - newInformer: newInformer, - watchErrorHandler: options.WatchErrorHandler, + codecs: serializer.NewCodecFactory(options.Scheme), + paramCodec: runtime.NewParameterCodec(options.Scheme), + resync: options.ResyncPeriod, + startWait: make(chan struct{}), + namespace: options.Namespace, + selector: options.Selector, + transform: options.Transform, + additionalDefaultIndexes: options.AdditionalDefaultIndexes, + unsafeDisableDeepCopy: options.UnsafeDisableDeepCopy, + newInformer: newInformer, + watchErrorHandler: options.WatchErrorHandler, } } @@ -171,9 +174,10 @@ type Informers struct { // default or empty string means all namespaces namespace string - selector Selector - transform cache.TransformFunc - unsafeDisableDeepCopy bool + selector Selector + transform cache.TransformFunc + additionalDefaultIndexes client.Indexers + unsafeDisableDeepCopy bool // NewInformer allows overriding of the shared index informer constructor for testing. newInformer func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer @@ -355,6 +359,11 @@ func (ip *Informers) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.O if err != nil { return nil, false, err } + indexers := make(cache.Indexers, len(ip.additionalDefaultIndexes)+1) + for k, fn := range ip.additionalDefaultIndexes { + indexers[FieldIndexName(k)] = IndexFunc(fn) + } + indexers[cache.NamespaceIndex] = cache.MetaNamespaceIndexFunc sharedIndexInformer := ip.newInformer(&cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { ip.selector.ApplyToList(&opts) @@ -365,9 +374,7 @@ func (ip *Informers) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.O opts.Watch = true // Watch needs to be set to true separately return listWatcher.WatchFunc(opts) }, - }, obj, calculateResyncPeriod(ip.resync), cache.Indexers{ - cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, - }) + }, obj, calculateResyncPeriod(ip.resync), indexers) // Set WatchErrorHandler on SharedIndexInformer if set if ip.watchErrorHandler != nil { @@ -594,3 +601,43 @@ func restrictNamespaceBySelector(namespaceOpt string, s Selector) string { } return "" } + +// IndexFunc constructs a low-level cache.IndexFunc from a client.IndexerFunc. +// Returned keys in the former are namespaced and non-namespaced variants of the +// latter. +func IndexFunc(extractValue client.IndexerFunc) cache.IndexFunc { + return func(objRaw interface{}) ([]string, error) { + // TODO(directxman12): check if this is the correct type? + obj, isObj := objRaw.(client.Object) + if !isObj { + return nil, fmt.Errorf("object of type %T is not an Object", objRaw) + } + meta, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + ns := meta.GetNamespace() + + rawVals := extractValue(obj) + var vals []string + if ns == "" { + // if we're not doubling the keys for the namespaced case, just create a new slice with same length + vals = make([]string, len(rawVals)) + } else { + // if we need to add non-namespaced versions too, double the length + vals = make([]string, len(rawVals)*2) + } + for i, rawVal := range rawVals { + // save a namespaced variant, so that we can ask + // "what are all the object matching a given index *in a given namespace*" + vals[i] = KeyToNamespacedKey(ns, rawVal) + if ns != "" { + // if we have a namespace, also inject a special index key for listing + // regardless of the object namespace + vals[i+len(rawVals)] = KeyToNamespacedKey("", rawVal) + } + } + + return vals, nil + } +} diff --git a/pkg/client/interfaces.go b/pkg/client/interfaces.go index 3cd745e4c0..97f2315679 100644 --- a/pkg/client/interfaces.go +++ b/pkg/client/interfaces.go @@ -190,6 +190,9 @@ type WithWatch interface { // namespaced and non-spaced variants, so keys do not need to include namespace. type IndexerFunc func(Object) []string +// Indexers is a map of field name to IndexerFunc. +type Indexers map[string]IndexerFunc + // FieldIndexer knows how to index over a particular "field" such that it // can later be used by a field selector. type FieldIndexer interface { diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 248893ea31..c5ee563c27 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -28,7 +28,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -36,8 +35,38 @@ import ( intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" ) +// AwareRunnable is an interface that can be implemented by runnable types +// that are cluster-aware. +type AwareRunnable interface { + // Engage gets called when the runnable should start operations for the given Cluster. + // The given context is tied to the Cluster's lifecycle and will be cancelled when the + // Cluster is removed or an error occurs. + // + // Implementers should return an error if they cannot start operations for the given Cluster, + // and should ensure this operation is re-entrant and non-blocking. + // + // \_________________|)____.---'--`---.____ + // || \----.________.----/ + // || / / `--' + // __||____/ /_ + // |___ \ + // `--------' + Engage(context.Context, Cluster) error + + // Disengage gets called when the runnable should stop operations for the given Cluster. + Disengage(context.Context, Cluster) error +} + +// ByNameGetterFunc is a function that returns a cluster for a given identifying cluster name. +type ByNameGetterFunc func(ctx context.Context, clusterName string) (Cluster, error) + // Cluster provides various methods to interact with a cluster. type Cluster interface { + // Name returns the name of the cluster. It identifies the cluster in the + // manager if that is attached to a cluster provider. The value is usually + // empty for the default cluster of a manager. + Name() string + // GetHTTPClient returns an HTTP client that can be used to talk to the apiserver GetHTTPClient() *http.Client @@ -76,6 +105,11 @@ type Cluster interface { // Options are the possible options that can be configured for a Cluster. type Options struct { + // name is the name of the cluster. It identifies the cluster in the manager + // if that is attached to a cluster provider. The value is usually empty for + // the default cluster of a manager. + Name string + // Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources // Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better // idea to pass your own scheme in. See the documentation in pkg/scheme for more information. @@ -158,6 +192,8 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { return nil, errors.New("must specify Config") } + // the config returned by GetConfig() is not to be modified. Hence, we have + // copy it before modifying it. originalConfig := config config = rest.CopyConfig(config) @@ -248,6 +284,7 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { } return &cluster{ + name: options.Name, config: originalConfig, httpClient: options.HTTPClient, scheme: options.Scheme, @@ -314,3 +351,13 @@ func setOptionsDefaults(options Options, config *rest.Config) (Options, error) { return options, nil } + +// WithName sets the name of the cluster. The name can only be set once. +func WithName(name string) Option { + return func(o *Options) { + if o.Name != "" { + panic("cluster name cannot be set more than once") + } + o.Name = name + } +} diff --git a/pkg/cluster/internal.go b/pkg/cluster/internal.go index 2742764231..2d15b0e41b 100644 --- a/pkg/cluster/internal.go +++ b/pkg/cluster/internal.go @@ -32,6 +32,11 @@ import ( ) type cluster struct { + // name is the name of the cluster. It identifies the cluster in the manager + // if that is attached to a cluster provider. The value is usually empty for + // the default cluster of a manager. + name string + // config is the rest.config used to talk to the apiserver. Required. config *rest.Config @@ -59,6 +64,10 @@ type cluster struct { logger logr.Logger } +func (c *cluster) Name() string { + return c.name +} + func (c *cluster) GetConfig() *rest.Config { return c.config } diff --git a/pkg/cluster/provider.go b/pkg/cluster/provider.go new file mode 100644 index 0000000000..f4e769344e --- /dev/null +++ b/pkg/cluster/provider.go @@ -0,0 +1,62 @@ +package cluster + +import ( + "context" + + "k8s.io/apimachinery/pkg/watch" +) + +// Provider defines methods to retrieve, list, and watch fleet of clusters. +// The provider is responsible for discovering and managing the lifecycle of each +// cluster. +// +// Example: A Cluster API provider would be responsible for discovering and managing +// clusters that are backed by Cluster API resources, which can live +// in multiple namespaces in a single management cluster. +type Provider interface { + // Get returns a cluster for the given identifying cluster name. The + // options are passed to the cluster constructor in case the cluster has + // not been created yet. Get returns an existing cluster if it has been + // created before. + Get(ctx context.Context, clusterName string, opts ...Option) (Cluster, error) + + // List returns a list of known identifying clusters names. + // This method is used to discover the initial set of known cluster names + // and to refresh the list of cluster names periodically. + List(ctx context.Context) ([]string, error) + + // Watch returns a Watcher that watches for changes to a list of known clusters + // and react to potential changes. + Watch(ctx context.Context) (Watcher, error) +} + +// Watcher watches for changes to clusters and provides events to a channel +// for the Manager to react to. +type Watcher interface { + // Stop stops watching. Will close the channel returned by ResultChan(). Releases + // any resources used by the watch. + Stop() + + // ResultChan returns a chan which will receive all the events. If an error occurs + // or Stop() is called, the implementation will close this channel and + // release any resources used by the watch. + ResultChan() <-chan WatchEvent +} + +// WatchEvent is an event that is sent when a cluster is added, modified, or deleted. +type WatchEvent struct { + // Type is the type of event that occurred. + // + // - ADDED or MODIFIED + // The cluster was added or updated: a new RESTConfig is available, or needs to be refreshed. + // - DELETED + // The cluster was deleted: the cluster is removed. + // - ERROR + // An error occurred while watching the cluster: the cluster is removed. + // - BOOKMARK + // A periodic event is sent that contains no new data: ignored. + Type watch.EventType + + // ClusterName is the identifying name of the cluster related to the event. + ClusterName string +} diff --git a/pkg/config/controller.go b/pkg/config/controller.go index b37dffaeea..cb5536a563 100644 --- a/pkg/config/controller.go +++ b/pkg/config/controller.go @@ -46,4 +46,16 @@ type Controller struct { // NeedLeaderElection indicates whether the controller needs to use leader election. // Defaults to true, which means the controller will use leader election. NeedLeaderElection *bool + + // EngageWithDefaultCluster indicates whether the controller should engage + // with the default cluster. This default to false if a cluster provider + // is configured, and to true otherwise. + // + // This is an experimental feature and is subject to change. + EngageWithDefaultCluster *bool + + // EngageWithProvidedClusters indicates whether the controller should engage + // with the provided clusters of the manager. This defaults to true if a + // cluster provider is set, and to false otherwise. + EngageWithProviderClusters *bool } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 5c9e48beae..942c6bd73b 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -72,6 +72,17 @@ type Options struct { // LogConstructor is used to construct a logger used for this controller and passed // to each reconciliation via the context field. LogConstructor func(request *reconcile.Request) logr.Logger + + // EngageWithDefaultCluster indicates whether the controller should engage + // with the default cluster of a manager. This defaults to false through the + // global controller options of the manager if a cluster provider is set, + // and to true otherwise. Here it can be overridden. + EngageWithDefaultCluster *bool + // EngageWithProvidedClusters indicates whether the controller should engage + // with the provided clusters of a manager. This defaults to true through the + // global controller options of the manager if a cluster provider is set, + // and to false otherwise. Here it can be overridden. + EngageWithProviderClusters *bool } // Controller implements a Kubernetes API. A Controller manages a work queue fed reconcile.Requests diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 0454cb4b90..03a22b3398 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -77,7 +77,6 @@ var _ = Describe("controller.Controller", func() { It("should not leak goroutines when stopped", func() { currentGRs := goleak.IgnoreCurrent() - ctx, cancel := context.WithCancel(context.Background()) watchChan := make(chan event.GenericEvent, 1) watch := source.Channel(watchChan, &handler.EnqueueRequestForObject{}) watchChan <- event.GenericEvent{Object: &corev1.Pod{}} @@ -104,20 +103,21 @@ var _ = Describe("controller.Controller", func() { Expect(c.Watch(watch)).To(Succeed()) Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) go func() { defer GinkgoRecover() Expect(m.Start(ctx)).To(Succeed()) close(controllerFinished) }() - <-reconcileStarted + Eventually(reconcileStarted).Should(BeClosed()) cancel() - <-controllerFinished + Eventually(controllerFinished).Should(BeClosed()) // force-close keep-alive connections. These'll time anyway (after // like 30s or so) but force it to speed up the tests. clientTransport.CloseIdleConnections() - Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) + Eventually(func() error { return goleak.Find(currentGRs) }, 10*time.Second).Should(Succeed()) }) It("should not create goroutines if never started", func() { diff --git a/pkg/controller/example_test.go b/pkg/controller/example_test.go index aea5943450..d11f3a0c36 100644 --- a/pkg/controller/example_test.go +++ b/pkg/controller/example_test.go @@ -101,6 +101,11 @@ func ExampleController_unstructured() { os.Exit(1) } + if err := mgr.Add(c); err != nil { + log.Error(err, "unable to add controller to manager") + os.Exit(1) + } + u := &unstructured.Unstructured{} u.SetGroupVersionKind(schema.GroupVersionKind{ Kind: "Pod", diff --git a/pkg/controller/multicluster.go b/pkg/controller/multicluster.go new file mode 100644 index 0000000000..fcabeae104 --- /dev/null +++ b/pkg/controller/multicluster.go @@ -0,0 +1,106 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "sync" + + "sigs.k8s.io/controller-runtime/pkg/cluster" +) + +// MultiClusterController is a Controller that is aware of the Cluster it is +// running in. It engage and disengage clusters dynamically, starting the +// watches and stopping them. +type MultiClusterController interface { + cluster.AwareRunnable + Controller +} + +// ClusterWatcher starts watches for a given Cluster. The ctx should be +// used to cancel the watch when the Cluster is disengaged. +type ClusterWatcher interface { + Watch(ctx context.Context, cl cluster.Cluster) error +} + +// NewMultiClusterController creates a new MultiClusterController for the given +// controller with the given ClusterWatcher. +func NewMultiClusterController(c Controller, watcher ClusterWatcher) MultiClusterController { + return &multiClusterController{ + Controller: c, + watcher: watcher, + clusters: map[string]struct{}{}, + } +} + +type multiClusterController struct { + Controller + watcher ClusterWatcher + + lock sync.Mutex + clusters map[string]struct{} +} + +// Engage gets called when the runnable should start operations for the given Cluster. +func (c *multiClusterController) Engage(clusterCtx context.Context, cl cluster.Cluster) error { + c.lock.Lock() + defer c.lock.Unlock() + + if _, ok := c.clusters[cl.Name()]; ok { + return nil + } + + // pass through in case the controller itself is cluster aware + if ctrl, ok := c.Controller.(cluster.AwareRunnable); ok { + if err := ctrl.Engage(clusterCtx, cl); err != nil { + return err + } + } + + // start watches on the cluster + if err := c.watcher.Watch(clusterCtx, cl); err != nil { + if ctrl, ok := c.Controller.(cluster.AwareRunnable); ok { + if err := ctrl.Disengage(clusterCtx, cl); err != nil { + return err + } + } + return err + } + c.clusters[cl.Name()] = struct{}{} + + return nil +} + +// Disengage gets called when the runnable should stop operations for the given Cluster. +func (c *multiClusterController) Disengage(ctx context.Context, cl cluster.Cluster) error { + c.lock.Lock() + defer c.lock.Unlock() + + if _, ok := c.clusters[cl.Name()]; !ok { + return nil + } + delete(c.clusters, cl.Name()) + + // pass through in case the controller itself is cluster aware + if ctrl, ok := c.Controller.(cluster.AwareRunnable); ok { + if err := ctrl.Disengage(ctx, cl); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/handler/cluster.go b/pkg/handler/cluster.go new file mode 100644 index 0000000000..6319eafbb1 --- /dev/null +++ b/pkg/handler/cluster.go @@ -0,0 +1,82 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "context" + "time" + + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// ForCluster wraps an EventHandler and adds the cluster name to the reconcile.Requests. +func ForCluster(clusterName string, h EventHandler) EventHandler { + return &clusterAwareHandler{ + clusterName: clusterName, + handler: h, + } +} + +type clusterAwareHandler struct { + handler EventHandler + clusterName string +} + +var _ EventHandler = &clusterAwareHandler{} + +func (c *clusterAwareHandler) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { + c.handler.Create(ctx, evt, &clusterWorkqueue{RateLimitingInterface: q, clusterName: c.clusterName}) +} + +func (c *clusterAwareHandler) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + c.handler.Update(ctx, evt, &clusterWorkqueue{RateLimitingInterface: q, clusterName: c.clusterName}) +} + +func (c *clusterAwareHandler) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + c.handler.Delete(ctx, evt, &clusterWorkqueue{RateLimitingInterface: q, clusterName: c.clusterName}) +} + +func (c *clusterAwareHandler) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { + c.handler.Generic(ctx, evt, &clusterWorkqueue{RateLimitingInterface: q, clusterName: c.clusterName}) +} + +// clusterWorkqueue is a wrapper around a RateLimitingInterface that adds the +// cluster name to the reconcile.Requests +type clusterWorkqueue struct { + workqueue.RateLimitingInterface + clusterName string +} + +func (q *clusterWorkqueue) AddAfter(item interface{}, duration time.Duration) { + req := item.(reconcile.Request) + req.ClusterName = q.clusterName + q.RateLimitingInterface.AddAfter(req, duration) +} + +func (q *clusterWorkqueue) Add(item interface{}) { + req := item.(reconcile.Request) + req.ClusterName = q.clusterName + q.RateLimitingInterface.Add(req) +} + +func (q *clusterWorkqueue) AddRateLimited(item interface{}) { + req := item.(reconcile.Request) + req.ClusterName = q.clusterName + q.RateLimitingInterface.AddRateLimited(req) +} diff --git a/pkg/internal/controller/controller.go b/pkg/internal/controller/controller.go index 9c709404b5..119384e72e 100644 --- a/pkg/internal/controller/controller.go +++ b/pkg/internal/controller/controller.go @@ -304,6 +304,9 @@ func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) { log = log.WithValues("reconcileID", reconcileID) ctx = logf.IntoContext(ctx, log) ctx = addReconcileID(ctx, reconcileID) + if req.ClusterName != "" { + log = log.WithValues("cluster", req.ClusterName) + } // RunInformersAndControllers the syncHandler, passing it the Namespace/Name string of the // resource to be synced. diff --git a/pkg/internal/testing/controlplane/apiserver.go b/pkg/internal/testing/controlplane/apiserver.go index c9a1a232ea..efb884e589 100644 --- a/pkg/internal/testing/controlplane/apiserver.go +++ b/pkg/internal/testing/controlplane/apiserver.go @@ -421,6 +421,9 @@ func (s *APIServer) Stop() error { return err } } + if s.Authn == nil { + return nil + } return s.Authn.Stop() } diff --git a/pkg/internal/testing/process/process_test.go b/pkg/internal/testing/process/process_test.go index 1d01e95b09..64da87dbdc 100644 --- a/pkg/internal/testing/process/process_test.go +++ b/pkg/internal/testing/process/process_test.go @@ -50,13 +50,14 @@ var _ = Describe("Start method", func() { HealthCheck: HealthCheck{ URL: getServerURL(server), }, + StopTimeout: 2 * time.Second, } - processState.Path = "bash" processState.Args = simpleBashScript }) AfterEach(func() { server.Close() + Expect(processState.Stop()).To(Succeed()) }) Context("when process takes too long to start", func() { diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go index 862d3bc8ca..a28e6e7fc0 100644 --- a/pkg/manager/internal.go +++ b/pkg/manager/internal.go @@ -31,6 +31,9 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" @@ -59,6 +62,14 @@ const ( ) var _ Runnable = &controllerManager{} +var _ Manager = &controllerManager{} +var _ cluster.ByNameGetterFunc = (&controllerManager{}).GetCluster + +type engagedCluster struct { + cluster.Cluster + ctx context.Context + cancel context.CancelFunc +} type controllerManager struct { sync.Mutex @@ -68,8 +79,15 @@ type controllerManager struct { errChan chan error runnables *runnables - // cluster holds a variety of methods to interact with a cluster. Required. - cluster cluster.Cluster + // defaultCluster holds a variety of methods to interact with a defaultCluster. Required. + defaultCluster cluster.Cluster + defaultClusterOptions cluster.Option + + clusterProvider cluster.Provider + + clusterLock sync.RWMutex // protects clusters + clusters map[string]*engagedCluster + clusterAwareRunnables []cluster.AwareRunnable // recorderProvider is used to generate event recorders that will be injected into Controllers // (and EventHandlers, Sources and Predicates). @@ -176,6 +194,9 @@ func (cm *controllerManager) Add(r Runnable) error { } func (cm *controllerManager) add(r Runnable) error { + if aware, ok := r.(cluster.AwareRunnable); ok { + cm.clusterAwareRunnables = append(cm.clusterAwareRunnables, aware) + } return cm.runnables.Add(r) } @@ -213,40 +234,160 @@ func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker) return nil } +func (cm *controllerManager) Name() string { + return cm.defaultCluster.Name() +} + +func (cm *controllerManager) GetCluster(ctx context.Context, clusterName string) (cluster.Cluster, error) { + return cm.getCluster(ctx, clusterName) +} + func (cm *controllerManager) GetHTTPClient() *http.Client { - return cm.cluster.GetHTTPClient() + return cm.defaultCluster.GetHTTPClient() } func (cm *controllerManager) GetConfig() *rest.Config { - return cm.cluster.GetConfig() + return cm.defaultCluster.GetConfig() } func (cm *controllerManager) GetClient() client.Client { - return cm.cluster.GetClient() + return cm.defaultCluster.GetClient() } func (cm *controllerManager) GetScheme() *runtime.Scheme { - return cm.cluster.GetScheme() + return cm.defaultCluster.GetScheme() } func (cm *controllerManager) GetFieldIndexer() client.FieldIndexer { - return cm.cluster.GetFieldIndexer() + return cm.defaultCluster.GetFieldIndexer() } func (cm *controllerManager) GetCache() cache.Cache { - return cm.cluster.GetCache() + return cm.defaultCluster.GetCache() } func (cm *controllerManager) GetEventRecorderFor(name string) record.EventRecorder { - return cm.cluster.GetEventRecorderFor(name) + return cm.defaultCluster.GetEventRecorderFor(name) } func (cm *controllerManager) GetRESTMapper() meta.RESTMapper { - return cm.cluster.GetRESTMapper() + return cm.defaultCluster.GetRESTMapper() } func (cm *controllerManager) GetAPIReader() client.Reader { - return cm.cluster.GetAPIReader() + return cm.defaultCluster.GetAPIReader() +} + +func (cm *controllerManager) engageClusterAwareRunnables() { + cm.Lock() + defer cm.Unlock() + + // If we don't have a cluster provider, we cannot sync the runnables. + if cm.clusterProvider == nil { + return + } + + // If we successfully retrieved the cluster, check + // that we schedule all the cluster aware runnables. + for name, cluster := range cm.clusters { + for _, aware := range cm.clusterAwareRunnables { + if err := aware.Engage(cluster.ctx, cluster); err != nil { + cm.logger.Error(err, "failed to engage cluster with runnable, won't retry", "clusterName", name, "runnable", aware) + continue + } + } + } +} + +func (cm *controllerManager) getCluster(ctx context.Context, clusterName string) (c *engagedCluster, err error) { + // Check if the manager was configured with a cluster provider, + // otherwise we cannot retrieve the cluster. + if cm.clusterProvider == nil { + return nil, fmt.Errorf("manager was not configured with a cluster provider, cannot retrieve %q", clusterName) + } + + // Check if the cluster already exists. + cm.clusterLock.RLock() + c, ok := cm.clusters[clusterName] + cm.clusterLock.RUnlock() + if ok { + return c, nil + } + + // Lock the whole function to avoid creating multiple clusters for the same name. + cm.clusterLock.Lock() + defer cm.clusterLock.Unlock() + + // Check again in case another goroutine already created the cluster. + c, ok = cm.clusters[clusterName] + if ok { + return c, nil + } + + // Create a new cluster. + var cl cluster.Cluster + { + // TODO(vincepri): Make this timeout configurable. + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + var watchErr error + if err := wait.PollUntilContextCancel(ctx, 1*time.Second, true, func(ctx context.Context) (done bool, err error) { + cl, watchErr = cm.clusterProvider.Get(ctx, clusterName, cm.defaultClusterOptions, cluster.WithName(clusterName)) + if watchErr != nil { + return false, nil //nolint:nilerr // We want to keep trying. + } + return true, nil + }); err != nil { + return nil, fmt.Errorf("failed create cluster %q: %w", clusterName, kerrors.NewAggregate([]error{err, watchErr})) + } + } + + // We add the Cluster to the manager as a Runnable, which is going to be categorized + // as a Cache-backed Runnable. + // + // Once added, if the manager has already started, it waits for the Cache to sync before returning + // otherwise it enqueues the Runnable to be started when the manager starts. + if err := cm.Add(cl); err != nil { + return nil, fmt.Errorf("cannot add cluster %q to manager: %w", clusterName, err) + } + + // Create a new context for the Cluster, so that it can be stopped independently. + ctx, cancel := context.WithCancel(context.Background()) + + c = &engagedCluster{ + Cluster: cl, + ctx: ctx, + cancel: cancel, + } + cm.clusters[clusterName] = c + return c, nil +} + +func (cm *controllerManager) removeNamedCluster(clusterName string) error { + // Check if the manager was configured with a cluster provider, + // otherwise we cannot retrieve the cluster. + if cm.clusterProvider == nil { + return fmt.Errorf("manager was not configured with a cluster provider, cannot retrieve %q", clusterName) + } + + cm.clusterLock.Lock() + defer cm.clusterLock.Unlock() + c, ok := cm.clusters[clusterName] + if !ok { + return nil + } + + // Disengage all the runnables. + for _, aware := range cm.clusterAwareRunnables { + if err := aware.Disengage(c.ctx, c); err != nil { + return fmt.Errorf("failed to disengage cluster aware runnable: %w", err) + } + } + + // Cancel the context and remove the cluster from the map. + c.cancel() + delete(cm.clusters, clusterName) + return nil } func (cm *controllerManager) GetWebhookServer() webhook.Server { @@ -313,7 +454,7 @@ func (cm *controllerManager) addPprofServer() error { // An error has occurred during in one of the internal operations, // such as leader election, cache start, webhooks, and so on. // Or, the context is cancelled. -func (cm *controllerManager) Start(ctx context.Context) (err error) { +func (cm *controllerManager) Start(ctx context.Context) (err error) { //nolint:gocyclo cm.Lock() if cm.started { cm.Unlock() @@ -353,7 +494,7 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { }() // Add the cluster runnable. - if err := cm.add(cm.cluster); err != nil { + if err := cm.add(cm.defaultCluster); err != nil { return fmt.Errorf("failed to add cluster to runnables: %w", err) } @@ -430,6 +571,81 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { }() } + // If the manager has been configured with a cluster provider, start it. + if cm.clusterProvider != nil { + if err := cm.add(RunnableFunc(func(ctx context.Context) error { + resync := func() error { + clusterList, err := cm.clusterProvider.List(ctx) + if err != nil { + return err + } + for _, name := range clusterList { + if _, err := cm.getCluster(ctx, name); err != nil { + return err + } + } + clusterListSet := sets.New(clusterList...) + for name := range cm.clusters { + if clusterListSet.Has(name) { + continue + } + if err := cm.removeNamedCluster(name); err != nil { + return err + } + } + cm.engageClusterAwareRunnables() + return nil + } + + // Always do an initial full resync. + if err := resync(); err != nil { + return err + } + + // Create a watcher and start watching for changes. + watcher, err := cm.clusterProvider.Watch(ctx) + if err != nil { + return err + } + defer func() { + go func() { + // Drain the watcher result channel to prevent a goroutine leak. + for range watcher.ResultChan() { + } + }() + watcher.Stop() + }() + + for { + select { + case <-time.After(10 * time.Minute): + if err := resync(); err != nil { + return err + } + case event := <-watcher.ResultChan(): + switch event.Type { + case watch.Added, watch.Modified: + if _, err := cm.getCluster(ctx, event.ClusterName); err != nil { + return err + } + cm.engageClusterAwareRunnables() + case watch.Deleted, watch.Error: + if err := cm.removeNamedCluster(event.ClusterName); err != nil { + return err + } + case watch.Bookmark: + continue + } + case <-ctx.Done(): + return nil + } + } + })); err != nil { + return fmt.Errorf("failed to add cluster provider to runnables: %w", err) + } + } + + // Manager is ready. ready = true cm.Unlock() select { diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 7b1bc605b1..e39ef4dfc7 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -33,8 +33,6 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" "k8s.io/utils/ptr" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" @@ -43,12 +41,12 @@ import ( intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/leaderelection" "sigs.k8s.io/controller-runtime/pkg/log" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/recorder" "sigs.k8s.io/controller-runtime/pkg/webhook" ) -// Manager initializes shared dependencies such as Caches and Clients, and provides them to Runnables. -// A Manager is required to create Controllers. +// Manager defines the interface that components must satisfy in order to be managed by a controller. type Manager interface { // Cluster holds a variety of methods to interact with a cluster. cluster.Cluster @@ -78,6 +76,9 @@ type Manager interface { // lock was lost. Start(ctx context.Context) error + // GetCluster retrieves a Cluster from a given identifying cluster name. + GetCluster(ctx context.Context, clusterName string) (cluster.Cluster, error) + // GetWebhookServer returns a webhook.Server GetWebhookServer() webhook.Server @@ -272,6 +273,12 @@ type Options struct { newMetricsServer func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) newHealthProbeListener func(addr string) (net.Listener, error) newPprofListener func(addr string) (net.Listener, error) + + // ExperimentalClusterProvider is an EXPERIMENTAL feature that allows the manager to + // operate against many Kubernetes clusters at once. + // It can be used by invoking WithExperimentalClusterProvider(adapter) on Options. + // Individual clusters can be accessed by calling GetCluster on the Manager. + ExperimentalClusterProvider cluster.Provider } // BaseContextFunc is a function used to provide a base Context to Runnables @@ -316,7 +323,7 @@ func New(config *rest.Config, options Options) (Manager, error) { // Set default values for options fields options = setOptionsDefaults(options) - cluster, err := cluster.New(config, func(clusterOptions *cluster.Options) { + clusterOptions := func(clusterOptions *cluster.Options) { clusterOptions.Scheme = options.Scheme clusterOptions.MapperProvider = options.MapperProvider clusterOptions.Logger = options.Logger @@ -325,7 +332,9 @@ func New(config *rest.Config, options Options) (Manager, error) { clusterOptions.Cache = options.Cache clusterOptions.Client = options.Client clusterOptions.EventBroadcaster = options.EventBroadcaster //nolint:staticcheck - }) + } + + cluster, err := cluster.New(config, clusterOptions) if err != nil { return nil, err } @@ -407,11 +416,13 @@ func New(config *rest.Config, options Options) (Manager, error) { } errChan := make(chan error, 1) - runnables := newRunnables(options.BaseContext, errChan) return &controllerManager{ stopProcedureEngaged: ptr.To(int64(0)), - cluster: cluster, - runnables: runnables, + defaultCluster: cluster, + defaultClusterOptions: clusterOptions, + clusterProvider: options.ExperimentalClusterProvider, + clusters: make(map[string]*engagedCluster), + runnables: newRunnables(options.BaseContext, errChan), errChan: errChan, recorderProvider: recorderProvider, resourceLock: resourceLock, @@ -542,5 +553,10 @@ func setOptionsDefaults(options Options) Options { options.WebhookServer = webhook.NewServer(webhook.Options{}) } + if options.Controller.EngageWithDefaultCluster == nil { + options.Controller.EngageWithDefaultCluster = ptr.To[bool](options.ExperimentalClusterProvider == nil) + options.Controller.EngageWithProviderClusters = ptr.To[bool](options.ExperimentalClusterProvider != nil) + } + return options } diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index 88dcee60c0..687f100c52 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -38,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -1748,7 +1749,7 @@ var _ = Describe("manger.Manager", func() { Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - Expect(m.GetConfig()).To(Equal(mgr.cluster.GetConfig())) + Expect(m.GetConfig()).To(Equal(mgr.defaultCluster.GetConfig())) }) It("should provide a function to get the Client", func() { @@ -1756,7 +1757,7 @@ var _ = Describe("manger.Manager", func() { Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - Expect(m.GetClient()).To(Equal(mgr.cluster.GetClient())) + Expect(m.GetClient()).To(Equal(mgr.defaultCluster.GetClient())) }) It("should provide a function to get the Scheme", func() { @@ -1764,7 +1765,7 @@ var _ = Describe("manger.Manager", func() { Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - Expect(m.GetScheme()).To(Equal(mgr.cluster.GetScheme())) + Expect(m.GetScheme()).To(Equal(mgr.defaultCluster.GetScheme())) }) It("should provide a function to get the FieldIndexer", func() { @@ -1772,7 +1773,7 @@ var _ = Describe("manger.Manager", func() { Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - Expect(m.GetFieldIndexer()).To(Equal(mgr.cluster.GetFieldIndexer())) + Expect(m.GetFieldIndexer()).To(Equal(mgr.defaultCluster.GetFieldIndexer())) }) It("should provide a function to get the EventRecorder", func() { @@ -1785,6 +1786,91 @@ var _ = Describe("manger.Manager", func() { Expect(err).NotTo(HaveOccurred()) Expect(m.GetAPIReader()).NotTo(BeNil()) }) + + Context("with cluster provider", func() { + It("should be able to create a manager with a cluster provider", func() { + adapter := &fakeClusterAdapter{ + clusterNameList: []string{ + "test-cluster", + }, + } + + m, err := New(cfg, Options{ExperimentalClusterProvider: adapter}) + Expect(err).NotTo(HaveOccurred()) + + aware := &fakeClusterAwareRunnable{} + Expect(m.Add(aware)).To(Succeed()) + + By("starting the manager") + ctx, cancel := context.WithCancel(context.Background()) + doneCh := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(doneCh) + Expect(m.Start(ctx)).To(Succeed()) + }() + <-m.Elected() + + Eventually(func() []string { + aware.Lock() + defer aware.Unlock() + return aware.engaged + }).Should(HaveLen(1)) + Expect(aware.engaged[0]).To(BeEquivalentTo("test-cluster")) + By("making sure there's no extra go routines still running after we stop") + cancel() + <-doneCh + }) + + It("should cancel only the removed cluster runnables", func() { + adapter := &fakeClusterAdapter{ + watch: make(chan cluster.WatchEvent), + clusterNameList: []string{ + "test-cluster", + "removed-cluster", + }, + } + + m, err := New(cfg, Options{ExperimentalClusterProvider: adapter}) + Expect(err).NotTo(HaveOccurred()) + + aware := &fakeClusterAwareRunnable{} + Expect(m.Add(aware)).To(Succeed()) + + By("starting the manager") + ctx, cancel := context.WithCancel(context.Background()) + doneCh := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(doneCh) + Expect(m.Start(ctx)).To(Succeed()) + }() + <-m.Elected() + + Eventually(func() []string { + aware.Lock() + defer aware.Unlock() + return aware.engaged + }).Should(HaveLen(2)) + Expect(aware.engaged).To(ConsistOf("test-cluster", "removed-cluster")) + + By("Deleting a cluster") + adapter.watch <- cluster.WatchEvent{ + Type: watch.Deleted, + ClusterName: "removed-cluster", + } + Eventually(func() []string { + aware.Lock() + defer aware.Unlock() + return aware.disengaged + }).Should(HaveLen(1)) + Expect(aware.disengaged).To(ConsistOf("removed-cluster")) + + By("making sure there's no extra go routines still running after we stop") + cancel() + <-doneCh + }) + }) }) type runnableError struct { @@ -1868,3 +1954,57 @@ func (n *needElection) Start(_ context.Context) error { func (n *needElection) NeedLeaderElection() bool { return true } + +type fakeClusterAdapter struct { + clusterNameList []string + listErr error + + watch chan cluster.WatchEvent +} + +func (f *fakeClusterAdapter) Get(ctx context.Context, clusterName string, opts ...cluster.Option) (cluster.Cluster, error) { + return cluster.New(testenv.Config, opts...) +} + +func (f *fakeClusterAdapter) List(ctx context.Context) ([]string, error) { + return f.clusterNameList, f.listErr +} + +func (f *fakeClusterAdapter) Watch(ctx context.Context) (cluster.Watcher, error) { + return &fakeLogicalClusterProvider{ch: f.watch}, nil +} + +type fakeLogicalClusterProvider struct { + ch chan cluster.WatchEvent +} + +func (f *fakeLogicalClusterProvider) Stop() { +} + +func (f *fakeLogicalClusterProvider) ResultChan() <-chan cluster.WatchEvent { + return f.ch +} + +type fakeClusterAwareRunnable struct { + sync.Mutex + engaged []string + disengaged []string +} + +func (f *fakeClusterAwareRunnable) Start(ctx context.Context) error { + return nil +} + +func (f *fakeClusterAwareRunnable) Engage(ctx context.Context, cluster cluster.Cluster) error { + f.Lock() + defer f.Unlock() + f.engaged = append(f.engaged, cluster.Name()) + return nil +} + +func (f *fakeClusterAwareRunnable) Disengage(ctx context.Context, cluster cluster.Cluster) error { + f.Lock() + defer f.Unlock() + f.disengaged = append(f.disengaged, cluster.Name()) + return nil +} diff --git a/pkg/manager/runnable_group.go b/pkg/manager/runnable_group.go index db5cda7c88..c531359b03 100644 --- a/pkg/manager/runnable_group.go +++ b/pkg/manager/runnable_group.go @@ -53,6 +53,15 @@ func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables { // The runnables added before Start are started when Start is called. // The runnables added after Start are started directly. func (r *runnables) Add(fn Runnable) error { + // For wrapped logical runnables, we need to unwrap them to get the underlying runnable. + // And type switch on the unwrapped type. This is needed because the runnable + // might have a different type, but we want to override the Start method to control + // cancellation on a per cluster basis. + // unwrapped := fn + // if wrapped, ok := fn.(*logicalWrappedRunnable); ok { + // unwrapped = wrapped.Unwrap() + // } + switch runnable := fn.(type) { case *Server: if runnable.NeedLeaderElection() { diff --git a/pkg/reconcile/reconcile.go b/pkg/reconcile/reconcile.go index f1cce87c85..63896172e2 100644 --- a/pkg/reconcile/reconcile.go +++ b/pkg/reconcile/reconcile.go @@ -48,10 +48,23 @@ func (r *Result) IsZero() bool { // information to uniquely identify the object - its Name and Namespace. It does NOT contain information about // any specific Event or the object contents itself. type Request struct { + // ClusterName identifies the cluster that the object is in. + // The property is only populated when controllers are setup in a fleet manager. + // +optional + ClusterName string + // NamespacedName is the name and namespace of the object to reconcile. types.NamespacedName } +// String returns the general purpose string representation. +func (r Request) String() string { + if r.ClusterName == "" { + return r.NamespacedName.String() + } + return "cluster://" + r.ClusterName + string(types.Separator) + r.NamespacedName.String() +} + /* Reconciler implements a Kubernetes API for a specific Resource by Creating, Updating or Deleting Kubernetes objects, or by making changes to systems external to the cluster (e.g. cloudproviders, github, etc).