From b6cb42087674acc702495f347de0c3e07cf0d787 Mon Sep 17 00:00:00 2001 From: Plamen Kokanov <35485709+plkokanov@users.noreply.github.com> Date: Mon, 5 Sep 2022 11:04:26 +0300 Subject: [PATCH] Adds `Progressing` status to `BackupBucketsReady` seed condition (#6587) * Adds configuration for the BackupBucketsCheck controller * Adds Progressing status to BackupBucketsReady condition * Adds integration test * Addresses review comments * Adds ability to create test client with field selector support * Deflake integration test * Addresses review comments --- .../configmap-componentconfig.yaml | 9 + charts/gardener/controlplane/values.yaml | 6 + docs/concepts/controller-manager.md | 10 +- ...entconfig-gardener-controller-manager.yaml | 6 + pkg/controllermanager/apis/config/types.go | 15 + .../apis/config/v1alpha1/defaults.go | 15 + .../apis/config/v1alpha1/defaults_test.go | 12 + .../apis/config/v1alpha1/types.go | 19 ++ .../v1alpha1/zz_generated.conversion.go | 36 +++ .../config/v1alpha1/zz_generated.deepcopy.go | 36 +++ .../config/v1alpha1/zz_generated.defaults.go | 3 + .../apis/config/zz_generated.deepcopy.go | 36 +++ .../controller/bastion/add_test.go | 69 +---- pkg/controllermanager/controller/factory.go | 2 +- pkg/controllermanager/controller/seed/seed.go | 116 +++++-- ...go => seed_backup_bucket_check_control.go} | 62 ++-- .../seed_backup_bucket_check_control_test.go | 293 ++++++++++++++++++ .../seed/seed_backup_bucket_reconcile_test.go | 274 ---------------- .../seed/seed_extension_check_control.go | 56 +--- .../seed/seed_lifecycle_reconcile.go | 32 +- .../controller/seed/seed_reconcile.go | 6 +- pkg/utils/test/client.go | 94 ++++++ .../backupbucketscheck_suite_test.go | 175 +++++++++++ .../backupbucketscheck_test.go | 187 +++++++++++ 24 files changed, 1101 insertions(+), 468 deletions(-) rename pkg/controllermanager/controller/seed/{seed_backup_bucket_reconcile.go => seed_backup_bucket_check_control.go} (63%) create mode 100644 pkg/controllermanager/controller/seed/seed_backup_bucket_check_control_test.go delete mode 100644 pkg/controllermanager/controller/seed/seed_backup_bucket_reconcile_test.go create mode 100644 pkg/utils/test/client.go create mode 100644 test/integration/controllermanager/seed/backupbucketscheck/backupbucketscheck_suite_test.go create mode 100644 test/integration/controllermanager/seed/backupbucketscheck/backupbucketscheck_test.go diff --git a/charts/gardener/controlplane/charts/runtime/templates/controller-manager/configmap-componentconfig.yaml b/charts/gardener/controlplane/charts/runtime/templates/controller-manager/configmap-componentconfig.yaml index 4d545a14fef..68b70d64482 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/controller-manager/configmap-componentconfig.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/controller-manager/configmap-componentconfig.yaml @@ -96,6 +96,15 @@ data: {{ toYaml .Values.global.controller.config.controllers.seedExtensionsCheck.conditionThresholds | indent 8 }} {{- end }} {{- end }} + {{- if .Values.global.controller.config.controllers.seedBackupBucketsCheck }} + seedBackupBucketsCheck: + concurrentSyncs: {{ required ".Values.global.controller.config.controllers.seedBackupBucketsCheck.concurrentSyncs is required" .Values.global.controller.config.controllers.seedBackupBucketsCheck.concurrentSyncs }} + syncPeriod: {{ required ".Values.global.controller.config.controllers.seedBackupBucketsCheck.syncPeriod is required" .Values.global.controller.config.controllers.seedBackupBucketsCheck.syncPeriod }} + {{- if .Values.global.controller.config.controllers.seedBackupBucketsCheck.conditionThresholds }} + conditionThresholds: +{{ toYaml .Values.global.controller.config.controllers.seedBackupBucketsCheck.conditionThresholds | indent 8 }} + {{- end }} + {{- end }} {{- if .Values.global.controller.config.controllers.event }} event: {{- if .Values.global.controller.config.controllers.event.concurrentSyncs }} diff --git a/charts/gardener/controlplane/values.yaml b/charts/gardener/controlplane/values.yaml index 17b06011bf1..5eba0f87408 100644 --- a/charts/gardener/controlplane/values.yaml +++ b/charts/gardener/controlplane/values.yaml @@ -395,6 +395,12 @@ global: conditionThresholds: - type: ExtensionsCheck duration: 1m + seedBackupBucketsCheck: + concurrentSyncs: 5 + syncPeriod: 30s + conditionThresholds: + - type: BackupBucketsReady + duration: 1m shootMaintenance: concurrentSyncs: 5 enableShootControlPlaneRestarter: true diff --git a/docs/concepts/controller-manager.md b/docs/concepts/controller-manager.md index 35f00dd97d0..65e8a7e7f70 100644 --- a/docs/concepts/controller-manager.md +++ b/docs/concepts/controller-manager.md @@ -89,7 +89,7 @@ controllers: ``` The Project controller takes the shown `config` and creates a `ResourceQuota` with the name `gardener` in the project namespace. -If a `ResourceQuota` resource with the name `gardener` already exists, the controller will only update fields in `spec.hard` which are **unavailable** at that time. +If a `ResourceQuota` resource with the name `gardener` already exists, the controller will only update fields in `spec.hard` which are **unavailable** at that time. Labels and annotations on the `ResourceQuota` `config` get merged with the respective fields on existing `ResourceQuota`s. An optional `projectSelector` narrows down the amount of projects that are equipped with the given `config`. If multiple configs match for a project, then only the first match in the list is applied to the project namespace. @@ -186,12 +186,12 @@ The "main" reconciler takes care about this replication: |:-------:|:---------:|:-----:| | Secret | garden | gardener.cloud/role | -#### "Backup Bucket" Reconciler +#### "Backup Buckets Check" Reconciler Every time a `BackupBucket` object is created or updated, the referenced `Seed` object is enqueued for reconciliation. It's the reconciler's task to check the `status` subresource of all existing `BackupBuckets` that belong to this seed. If at least one `BackupBucket` has `.status.lastError`, the seed condition `BackupBucketsReady` will turn `false` and -consequently the seed is considered as `NotReady`. Once the `BackupBucket` is healthy again, the seed will be re-queued +consequently the seed is considered as `NotReady`. If the `SeedBackupBucketsCheckControllerConfiguration`, which is part of `gardener-controller-manager`s `ControllerManagerControllerConfiguration`, contains a `conditionThreshold` for the `BackupBucketsReady`, the condition will instead first be set to `progressing` and eventually to `false` once the `conditionThreshold` expires, see [the example config file](../../example/20-componentconfig-gardener-controller-manager.yaml) for details. Once the `BackupBucket` is healthy again, the seed will be re-queued and the condition will turn `true`. #### "Lifecycle" Reconciler @@ -205,7 +205,7 @@ In case a `Lease` is not renewed for the configured amount in `config.controller 1. The reconciler assumes that the Gardenlet stopped operating and updates the `GardenletReady` condition to `Unknown`. 2. Additionally, conditions and constraints of all `Shoot` resources scheduled on the affected seed are set to `Unknown` as well because a striking Gardenlet won't be able to maintain these conditions any more. -3. If the gardenlet's client certificate has expired (identified based on the `.status.clientCertificateExpirationTimestamp` field in the `Seed` resource) and if it is managed by a `ManagedSeed` then this will be triggered for a reconciliation. This will trigger the bootstrapping process again and allows gardenlets to obtain a fresh client certificate. +3. If the gardenlet's client certificate has expired (identified based on the `.status.clientCertificateExpirationTimestamp` field in the `Seed` resource) and if it is managed by a `ManagedSeed` then this will be triggered for a reconciliation. This will trigger the bootstrapping process again and allows gardenlets to obtain a fresh client certificate. ### ControllerRegistration Controller @@ -241,5 +241,5 @@ On startup the gardenlet uses a `kubeconfig` with a [bootstrap token](https://ku The controller in `gardener-controller-manager` checks whether the `CertificateSigningRequest` has the expected organisation, common name and usages which the gardenlet would request. -It only auto-approves the CSR if the client making the request is allowed to "create" the +It only auto-approves the CSR if the client making the request is allowed to "create" the `certificatesigningrequests/seedclient` subresource. Clients with the `system:bootstrappers` group are bound to the `gardener.cloud:system:seed-bootstrapper` `ClusterRole`, hence, they have such privileges. As the bootstrap kubeconfig for the gardenlet contains a bootstrap token which is authenticated as being part of the [`systems:bootstrappers` group](../../charts/gardener/controlplane/charts/application/templates/clusterrolebinding-seed-bootstrapper.yaml), its created CSR gets auto-approved. diff --git a/example/20-componentconfig-gardener-controller-manager.yaml b/example/20-componentconfig-gardener-controller-manager.yaml index 306a6f14738..1f2b5a4b515 100644 --- a/example/20-componentconfig-gardener-controller-manager.yaml +++ b/example/20-componentconfig-gardener-controller-manager.yaml @@ -21,6 +21,12 @@ controllers: conditionThresholds: - type: ExtensionsReady duration: 1m + seedBackupBucketsCheck: + concurrentSyncs: 5 + syncPeriod: 30s + conditionThresholds: + - type: BackupBucketsReady + duration: 1m shootMaintenance: concurrentSyncs: 5 # enableShootControlPlaneRestarter: true diff --git a/pkg/controllermanager/apis/config/types.go b/pkg/controllermanager/apis/config/types.go index 56919883fc7..d81980d64b2 100644 --- a/pkg/controllermanager/apis/config/types.go +++ b/pkg/controllermanager/apis/config/types.go @@ -71,6 +71,8 @@ type ControllerManagerControllerConfiguration struct { Seed *SeedControllerConfiguration // SeedExtensionsCheck defines the configuration of the SeedExtensionsCheck controller. SeedExtensionsCheck *SeedExtensionsCheckControllerConfiguration + // SeedBackupBucketsCheck defines the configuration of the SeedBackupBucketsCheck controller. + SeedBackupBucketsCheck *SeedBackupBucketsCheckControllerConfiguration // ShootMaintenance defines the configuration of the ShootMaintenance controller. ShootMaintenance ShootMaintenanceControllerConfiguration // ShootQuota defines the configuration of the ShootQuota controller. @@ -217,6 +219,19 @@ type SeedExtensionsCheckControllerConfiguration struct { ConditionThresholds []ConditionThreshold } +// SeedBackupBucketsCheckControllerConfiguration defines the configuration of the +// SeedBackupBucketsCheck controller. +type SeedBackupBucketsCheckControllerConfiguration struct { + // ConcurrentSyncs is the number of workers used for the controller to work on + // events. + ConcurrentSyncs *int + // SyncPeriod is the duration how often the existing resources are reconciled (how + // often the health check of BackupBuckets is performed). + SyncPeriod *metav1.Duration + // ConditionThresholds defines the condition threshold per condition type. + ConditionThresholds []ConditionThreshold +} + // ShootMaintenanceControllerConfiguration defines the configuration of the // ShootMaintenance controller. type ShootMaintenanceControllerConfiguration struct { diff --git a/pkg/controllermanager/apis/config/v1alpha1/defaults.go b/pkg/controllermanager/apis/config/v1alpha1/defaults.go index 1c48a44f11f..8ef80e3816b 100644 --- a/pkg/controllermanager/apis/config/v1alpha1/defaults.go +++ b/pkg/controllermanager/apis/config/v1alpha1/defaults.go @@ -141,6 +141,10 @@ func SetDefaults_ControllerManagerConfiguration(obj *ControllerManagerConfigurat obj.Controllers.SeedExtensionsCheck = &SeedExtensionsCheckControllerConfiguration{} } + if obj.Controllers.SeedBackupBucketsCheck == nil { + obj.Controllers.SeedBackupBucketsCheck = &SeedBackupBucketsCheckControllerConfiguration{} + } + if obj.Controllers.ShootMaintenance.ConcurrentSyncs == nil { v := DefaultControllerConcurrentSyncs obj.Controllers.ShootMaintenance.ConcurrentSyncs = &v @@ -301,3 +305,14 @@ func SetDefaults_SeedExtensionsCheckControllerConfiguration(obj *SeedExtensionsC obj.SyncPeriod = &v } } + +// SetDefaults_SeedBackupBucketsCheckControllerConfiguration sets defaults for the given SeedBackupBucketsCheckControllerConfiguration. +func SetDefaults_SeedBackupBucketsCheckControllerConfiguration(obj *SeedBackupBucketsCheckControllerConfiguration) { + if obj.ConcurrentSyncs == nil { + v := DefaultControllerConcurrentSyncs + obj.ConcurrentSyncs = &v + } + if obj.SyncPeriod == nil { + obj.SyncPeriod = &metav1.Duration{Duration: 30 * time.Second} + } +} diff --git a/pkg/controllermanager/apis/config/v1alpha1/defaults_test.go b/pkg/controllermanager/apis/config/v1alpha1/defaults_test.go index 8003efbd1c3..7d4e8991524 100644 --- a/pkg/controllermanager/apis/config/v1alpha1/defaults_test.go +++ b/pkg/controllermanager/apis/config/v1alpha1/defaults_test.go @@ -86,6 +86,8 @@ var _ = Describe("Defaults", func() { Expect(obj.Controllers.SeedExtensionsCheck).NotTo(BeNil()) + Expect(obj.Controllers.SeedBackupBucketsCheck).NotTo(BeNil()) + Expect(obj.Controllers.ShootMaintenance.ConcurrentSyncs).NotTo(BeNil()) Expect(obj.Controllers.ShootMaintenance.ConcurrentSyncs).To(PointTo(Equal(5))) @@ -226,6 +228,16 @@ var _ = Describe("Defaults", func() { Expect(obj.SyncPeriod).To(PointTo(Equal(metav1.Duration{Duration: 30 * time.Second}))) }) }) + + Describe("#SetDefaults_SeedBackupBucketsCheckControllerConfiguration", func() { + It("should correctly default the SeedBackupBucketsCheck Controller configuration", func() { + obj := &SeedBackupBucketsCheckControllerConfiguration{} + + SetDefaults_SeedBackupBucketsCheckControllerConfiguration(obj) + Expect(obj.ConcurrentSyncs).To(PointTo(Equal(5))) + Expect(obj.SyncPeriod).To(PointTo(Equal(metav1.Duration{Duration: 30 * time.Second}))) + }) + }) }) var _ = Describe("Constants", func() { diff --git a/pkg/controllermanager/apis/config/v1alpha1/types.go b/pkg/controllermanager/apis/config/v1alpha1/types.go index 24e950c627a..a3965a576c2 100644 --- a/pkg/controllermanager/apis/config/v1alpha1/types.go +++ b/pkg/controllermanager/apis/config/v1alpha1/types.go @@ -85,6 +85,9 @@ type ControllerManagerControllerConfiguration struct { // SeedExtensionsCheck defines the configuration of the SeedExtensionsCheck controller. // +optional SeedExtensionsCheck *SeedExtensionsCheckControllerConfiguration `json:"seedExtensionsCheck,omitempty"` + // SeedBackupBucketsCheck defines the configuration of the SeedBackupBucketsCheck controller. + // +optional + SeedBackupBucketsCheck *SeedBackupBucketsCheckControllerConfiguration `json:"seedBackupBucketsCheck,omitempty"` // ShootMaintenance defines the configuration of the ShootMaintenance controller. ShootMaintenance ShootMaintenanceControllerConfiguration `json:"shootMaintenance"` // ShootQuota defines the configuration of the ShootQuota controller. @@ -259,6 +262,22 @@ type SeedExtensionsCheckControllerConfiguration struct { ConditionThresholds []ConditionThreshold `json:"conditionThresholds,omitempty"` } +// SeedBackupBucketsCheckControllerConfiguration defines the configuration of the SeedBackupBucketsCheck +// controller. +type SeedBackupBucketsCheckControllerConfiguration struct { + // ConcurrentSyncs is the number of workers used for the controller to work on + // events. + // +optional + ConcurrentSyncs *int `json:"concurrentSyncs,omitempty"` + // SyncPeriod is the duration how often the existing resources are reconciled (how + // often the health check of BackupBuckets is performed). + // +optional + SyncPeriod *metav1.Duration `json:"syncPeriod,omitempty"` + // ConditionThresholds defines the condition threshold per condition type. + // +optional + ConditionThresholds []ConditionThreshold `json:"conditionThresholds,omitempty"` +} + // ShootMaintenanceControllerConfiguration defines the configuration of the // ShootMaintenance controller. type ShootMaintenanceControllerConfiguration struct { diff --git a/pkg/controllermanager/apis/config/v1alpha1/zz_generated.conversion.go b/pkg/controllermanager/apis/config/v1alpha1/zz_generated.conversion.go index b34e240869d..ea0df0a2f64 100644 --- a/pkg/controllermanager/apis/config/v1alpha1/zz_generated.conversion.go +++ b/pkg/controllermanager/apis/config/v1alpha1/zz_generated.conversion.go @@ -174,6 +174,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*SeedBackupBucketsCheckControllerConfiguration)(nil), (*config.SeedBackupBucketsCheckControllerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_SeedBackupBucketsCheckControllerConfiguration_To_config_SeedBackupBucketsCheckControllerConfiguration(a.(*SeedBackupBucketsCheckControllerConfiguration), b.(*config.SeedBackupBucketsCheckControllerConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.SeedBackupBucketsCheckControllerConfiguration)(nil), (*SeedBackupBucketsCheckControllerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_SeedBackupBucketsCheckControllerConfiguration_To_v1alpha1_SeedBackupBucketsCheckControllerConfiguration(a.(*config.SeedBackupBucketsCheckControllerConfiguration), b.(*SeedBackupBucketsCheckControllerConfiguration), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*SeedControllerConfiguration)(nil), (*config.SeedControllerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_SeedControllerConfiguration_To_config_SeedControllerConfiguration(a.(*SeedControllerConfiguration), b.(*config.SeedControllerConfiguration), scope) }); err != nil { @@ -474,6 +484,7 @@ func autoConvert_v1alpha1_ControllerManagerControllerConfiguration_To_config_Con out.SecretBinding = (*config.SecretBindingControllerConfiguration)(unsafe.Pointer(in.SecretBinding)) out.Seed = (*config.SeedControllerConfiguration)(unsafe.Pointer(in.Seed)) out.SeedExtensionsCheck = (*config.SeedExtensionsCheckControllerConfiguration)(unsafe.Pointer(in.SeedExtensionsCheck)) + out.SeedBackupBucketsCheck = (*config.SeedBackupBucketsCheckControllerConfiguration)(unsafe.Pointer(in.SeedBackupBucketsCheck)) if err := Convert_v1alpha1_ShootMaintenanceControllerConfiguration_To_config_ShootMaintenanceControllerConfiguration(&in.ShootMaintenance, &out.ShootMaintenance, s); err != nil { return err } @@ -516,6 +527,7 @@ func autoConvert_config_ControllerManagerControllerConfiguration_To_v1alpha1_Con out.SecretBinding = (*SecretBindingControllerConfiguration)(unsafe.Pointer(in.SecretBinding)) out.Seed = (*SeedControllerConfiguration)(unsafe.Pointer(in.Seed)) out.SeedExtensionsCheck = (*SeedExtensionsCheckControllerConfiguration)(unsafe.Pointer(in.SeedExtensionsCheck)) + out.SeedBackupBucketsCheck = (*SeedBackupBucketsCheckControllerConfiguration)(unsafe.Pointer(in.SeedBackupBucketsCheck)) if err := Convert_config_ShootMaintenanceControllerConfiguration_To_v1alpha1_ShootMaintenanceControllerConfiguration(&in.ShootMaintenance, &out.ShootMaintenance, s); err != nil { return err } @@ -735,6 +747,30 @@ func Convert_config_SecretBindingControllerConfiguration_To_v1alpha1_SecretBindi return autoConvert_config_SecretBindingControllerConfiguration_To_v1alpha1_SecretBindingControllerConfiguration(in, out, s) } +func autoConvert_v1alpha1_SeedBackupBucketsCheckControllerConfiguration_To_config_SeedBackupBucketsCheckControllerConfiguration(in *SeedBackupBucketsCheckControllerConfiguration, out *config.SeedBackupBucketsCheckControllerConfiguration, s conversion.Scope) error { + out.ConcurrentSyncs = (*int)(unsafe.Pointer(in.ConcurrentSyncs)) + out.SyncPeriod = (*v1.Duration)(unsafe.Pointer(in.SyncPeriod)) + out.ConditionThresholds = *(*[]config.ConditionThreshold)(unsafe.Pointer(&in.ConditionThresholds)) + return nil +} + +// Convert_v1alpha1_SeedBackupBucketsCheckControllerConfiguration_To_config_SeedBackupBucketsCheckControllerConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_SeedBackupBucketsCheckControllerConfiguration_To_config_SeedBackupBucketsCheckControllerConfiguration(in *SeedBackupBucketsCheckControllerConfiguration, out *config.SeedBackupBucketsCheckControllerConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_SeedBackupBucketsCheckControllerConfiguration_To_config_SeedBackupBucketsCheckControllerConfiguration(in, out, s) +} + +func autoConvert_config_SeedBackupBucketsCheckControllerConfiguration_To_v1alpha1_SeedBackupBucketsCheckControllerConfiguration(in *config.SeedBackupBucketsCheckControllerConfiguration, out *SeedBackupBucketsCheckControllerConfiguration, s conversion.Scope) error { + out.ConcurrentSyncs = (*int)(unsafe.Pointer(in.ConcurrentSyncs)) + out.SyncPeriod = (*v1.Duration)(unsafe.Pointer(in.SyncPeriod)) + out.ConditionThresholds = *(*[]ConditionThreshold)(unsafe.Pointer(&in.ConditionThresholds)) + return nil +} + +// Convert_config_SeedBackupBucketsCheckControllerConfiguration_To_v1alpha1_SeedBackupBucketsCheckControllerConfiguration is an autogenerated conversion function. +func Convert_config_SeedBackupBucketsCheckControllerConfiguration_To_v1alpha1_SeedBackupBucketsCheckControllerConfiguration(in *config.SeedBackupBucketsCheckControllerConfiguration, out *SeedBackupBucketsCheckControllerConfiguration, s conversion.Scope) error { + return autoConvert_config_SeedBackupBucketsCheckControllerConfiguration_To_v1alpha1_SeedBackupBucketsCheckControllerConfiguration(in, out, s) +} + func autoConvert_v1alpha1_SeedControllerConfiguration_To_config_SeedControllerConfiguration(in *SeedControllerConfiguration, out *config.SeedControllerConfiguration, s conversion.Scope) error { out.ConcurrentSyncs = (*int)(unsafe.Pointer(in.ConcurrentSyncs)) out.MonitorPeriod = (*v1.Duration)(unsafe.Pointer(in.MonitorPeriod)) diff --git a/pkg/controllermanager/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/controllermanager/apis/config/v1alpha1/zz_generated.deepcopy.go index 69fc5330b67..0dba51ad18d 100644 --- a/pkg/controllermanager/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/controllermanager/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -215,6 +215,11 @@ func (in *ControllerManagerControllerConfiguration) DeepCopyInto(out *Controller *out = new(SeedExtensionsCheckControllerConfiguration) (*in).DeepCopyInto(*out) } + if in.SeedBackupBucketsCheck != nil { + in, out := &in.SeedBackupBucketsCheck, &out.SeedBackupBucketsCheck + *out = new(SeedBackupBucketsCheckControllerConfiguration) + (*in).DeepCopyInto(*out) + } in.ShootMaintenance.DeepCopyInto(&out.ShootMaintenance) in.ShootQuota.DeepCopyInto(&out.ShootQuota) in.ShootHibernation.DeepCopyInto(&out.ShootHibernation) @@ -463,6 +468,37 @@ func (in *SecretBindingControllerConfiguration) DeepCopy() *SecretBindingControl return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SeedBackupBucketsCheckControllerConfiguration) DeepCopyInto(out *SeedBackupBucketsCheckControllerConfiguration) { + *out = *in + if in.ConcurrentSyncs != nil { + in, out := &in.ConcurrentSyncs, &out.ConcurrentSyncs + *out = new(int) + **out = **in + } + if in.SyncPeriod != nil { + in, out := &in.SyncPeriod, &out.SyncPeriod + *out = new(v1.Duration) + **out = **in + } + if in.ConditionThresholds != nil { + in, out := &in.ConditionThresholds, &out.ConditionThresholds + *out = make([]ConditionThreshold, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SeedBackupBucketsCheckControllerConfiguration. +func (in *SeedBackupBucketsCheckControllerConfiguration) DeepCopy() *SeedBackupBucketsCheckControllerConfiguration { + if in == nil { + return nil + } + out := new(SeedBackupBucketsCheckControllerConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SeedControllerConfiguration) DeepCopyInto(out *SeedControllerConfiguration) { *out = *in diff --git a/pkg/controllermanager/apis/config/v1alpha1/zz_generated.defaults.go b/pkg/controllermanager/apis/config/v1alpha1/zz_generated.defaults.go index e2234065c64..639a20130cc 100644 --- a/pkg/controllermanager/apis/config/v1alpha1/zz_generated.defaults.go +++ b/pkg/controllermanager/apis/config/v1alpha1/zz_generated.defaults.go @@ -44,6 +44,9 @@ func SetObjectDefaults_ControllerManagerConfiguration(in *ControllerManagerConfi if in.Controllers.SeedExtensionsCheck != nil { SetDefaults_SeedExtensionsCheckControllerConfiguration(in.Controllers.SeedExtensionsCheck) } + if in.Controllers.SeedBackupBucketsCheck != nil { + SetDefaults_SeedBackupBucketsCheckControllerConfiguration(in.Controllers.SeedBackupBucketsCheck) + } SetDefaults_ShootHibernationControllerConfiguration(&in.Controllers.ShootHibernation) if in.Controllers.ShootRetry != nil { SetDefaults_ShootRetryControllerConfiguration(in.Controllers.ShootRetry) diff --git a/pkg/controllermanager/apis/config/zz_generated.deepcopy.go b/pkg/controllermanager/apis/config/zz_generated.deepcopy.go index dd4ca1d518f..03db46efcbf 100644 --- a/pkg/controllermanager/apis/config/zz_generated.deepcopy.go +++ b/pkg/controllermanager/apis/config/zz_generated.deepcopy.go @@ -215,6 +215,11 @@ func (in *ControllerManagerControllerConfiguration) DeepCopyInto(out *Controller *out = new(SeedExtensionsCheckControllerConfiguration) (*in).DeepCopyInto(*out) } + if in.SeedBackupBucketsCheck != nil { + in, out := &in.SeedBackupBucketsCheck, &out.SeedBackupBucketsCheck + *out = new(SeedBackupBucketsCheckControllerConfiguration) + (*in).DeepCopyInto(*out) + } in.ShootMaintenance.DeepCopyInto(&out.ShootMaintenance) in.ShootQuota.DeepCopyInto(&out.ShootQuota) in.ShootHibernation.DeepCopyInto(&out.ShootHibernation) @@ -465,6 +470,37 @@ func (in *SecretBindingControllerConfiguration) DeepCopy() *SecretBindingControl return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SeedBackupBucketsCheckControllerConfiguration) DeepCopyInto(out *SeedBackupBucketsCheckControllerConfiguration) { + *out = *in + if in.ConcurrentSyncs != nil { + in, out := &in.ConcurrentSyncs, &out.ConcurrentSyncs + *out = new(int) + **out = **in + } + if in.SyncPeriod != nil { + in, out := &in.SyncPeriod, &out.SyncPeriod + *out = new(v1.Duration) + **out = **in + } + if in.ConditionThresholds != nil { + in, out := &in.ConditionThresholds, &out.ConditionThresholds + *out = make([]ConditionThreshold, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SeedBackupBucketsCheckControllerConfiguration. +func (in *SeedBackupBucketsCheckControllerConfiguration) DeepCopy() *SeedBackupBucketsCheckControllerConfiguration { + if in == nil { + return nil + } + out := new(SeedBackupBucketsCheckControllerConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SeedControllerConfiguration) DeepCopyInto(out *SeedControllerConfiguration) { *out = *in diff --git a/pkg/controllermanager/controller/bastion/add_test.go b/pkg/controllermanager/controller/bastion/add_test.go index 2a0ecb92d50..16f4fdf711e 100644 --- a/pkg/controllermanager/controller/bastion/add_test.go +++ b/pkg/controllermanager/controller/bastion/add_test.go @@ -16,18 +16,13 @@ package bastion_test import ( "context" - "fmt" - - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - "github.com/gardener/gardener/pkg/apis/operations" operationsv1alpha1 "github.com/gardener/gardener/pkg/apis/operations/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" . "github.com/gardener/gardener/pkg/controllermanager/controller/bastion" bastionstrategy "github.com/gardener/gardener/pkg/registry/operations/bastion" + "github.com/gardener/gardener/pkg/utils/test" "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" @@ -146,7 +141,10 @@ var _ = Describe("Add", func() { BeforeEach(func() { log = logr.Discard() - fakeClient = clientWithFieldSelectorSupport{fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).Build()} + fakeClient = test.NewClientWithFieldSelectorSupport( + fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).Build(), + bastionstrategy.ToSelectableFields, + ) }) It("should do nothing if the object is no shoot", func() { @@ -209,60 +207,3 @@ var _ = Describe("Add", func() { }) }) }) - -// TODO: remove this again once the controller-runtime fake client supports field selectors -type clientWithFieldSelectorSupport struct { - client.Client -} - -func (c clientWithFieldSelectorSupport) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error { - if err := c.Client.List(ctx, obj, opts...); err != nil { - return err - } - - listOpts := client.ListOptions{} - listOpts.ApplyOptions(opts) - - if listOpts.FieldSelector != nil { - objs, err := meta.ExtractList(obj) - if err != nil { - return err - } - filteredObjs, err := filterWithFieldSelector(objs, listOpts.FieldSelector) - if err != nil { - return err - } - err = meta.SetList(obj, filteredObjs) - if err != nil { - return err - } - } - - return nil -} - -func filterWithFieldSelector(objs []runtime.Object, sel fields.Selector) ([]runtime.Object, error) { - outItems := make([]runtime.Object, 0, len(objs)) - for _, obj := range objs { - // convert to internal - bastion := &operations.Bastion{} - if err := kubernetes.GardenScheme.Convert(obj, bastion, nil); err != nil { - return nil, err - } - - fieldSet := bastionstrategy.ToSelectableFields(bastion) - - // complain about non-selectable fields if any - for _, req := range sel.Requirements() { - if !fieldSet.Has(req.Field) { - return nil, fmt.Errorf("field selector not supported for field %q", req.Field) - } - } - - if !sel.Matches(fieldSet) { - continue - } - outItems = append(outItems, obj.DeepCopyObject()) - } - return outItems, nil -} diff --git a/pkg/controllermanager/controller/factory.go b/pkg/controllermanager/controller/factory.go index c42e55027b8..ea63293bb94 100644 --- a/pkg/controllermanager/controller/factory.go +++ b/pkg/controllermanager/controller/factory.go @@ -95,7 +95,7 @@ func (f *LegacyControllerFactory) Start(ctx context.Context) error { go csrController.Run(ctx, 1) go projectController.Run(ctx, *f.Config.Controllers.Project.ConcurrentSyncs) go secretBindingController.Run(ctx, *f.Config.Controllers.SecretBinding.ConcurrentSyncs) - go seedController.Run(ctx, *f.Config.Controllers.Seed.ConcurrentSyncs, *f.Config.Controllers.SeedExtensionsCheck.ConcurrentSyncs) + go seedController.Run(ctx, *f.Config.Controllers.Seed.ConcurrentSyncs, *f.Config.Controllers.SeedBackupBucketsCheck.ConcurrentSyncs, *f.Config.Controllers.SeedExtensionsCheck.ConcurrentSyncs) go shootController.Run(ctx, *f.Config.Controllers.ShootMaintenance.ConcurrentSyncs, *f.Config.Controllers.ShootQuota.ConcurrentSyncs, *f.Config.Controllers.ShootHibernation.ConcurrentSyncs, *f.Config.Controllers.ShootReference.ConcurrentSyncs, *f.Config.Controllers.ShootRetry.ConcurrentSyncs, *f.Config.Controllers.ShootConditions.ConcurrentSyncs, *f.Config.Controllers.ShootStatusLabel.ConcurrentSyncs) go managedSeedSetController.Run(ctx, *f.Config.Controllers.ManagedSeedSet.ConcurrentSyncs) diff --git a/pkg/controllermanager/controller/seed/seed.go b/pkg/controllermanager/controller/seed/seed.go index 9978e01955d..376282a8ae3 100644 --- a/pkg/controllermanager/controller/seed/seed.go +++ b/pkg/controllermanager/controller/seed/seed.go @@ -25,6 +25,7 @@ import ( "k8s.io/utils/clock" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" "github.com/gardener/gardener/pkg/controllermanager/apis/config" "github.com/gardener/gardener/pkg/controllerutils" @@ -45,15 +46,15 @@ type Controller struct { config *config.ControllerManagerConfiguration log logr.Logger - secretsReconciler reconcile.Reconciler - seedBackupReconciler reconcile.Reconciler - lifeCycleReconciler reconcile.Reconciler - extensionsCheckReconciler reconcile.Reconciler + secretsReconciler reconcile.Reconciler + seedBackupBucketsCheckReconciler reconcile.Reconciler + lifeCycleReconciler reconcile.Reconciler + extensionsCheckReconciler reconcile.Reconciler - secretsQueue workqueue.RateLimitingInterface - seedBackupBucketQueue workqueue.RateLimitingInterface - seedLifecycleQueue workqueue.RateLimitingInterface - seedExtensionsCheckQueue workqueue.RateLimitingInterface + secretsQueue workqueue.RateLimitingInterface + seedBackupBucketsCheckQueue workqueue.RateLimitingInterface + seedLifecycleQueue workqueue.RateLimitingInterface + seedExtensionsCheckQueue workqueue.RateLimitingInterface hasSyncedFuncs []cache.InformerSynced workerCh chan int @@ -99,16 +100,16 @@ func NewSeedController( config: config, log: log, - secretsReconciler: NewSecretsReconciler(gardenClient), - lifeCycleReconciler: NewLifecycleReconciler(gardenClient, config), - seedBackupReconciler: NewBackupBucketReconciler(gardenClient), - extensionsCheckReconciler: NewExtensionsCheckReconciler(gardenClient, *config.Controllers.SeedExtensionsCheck, clock.RealClock{}), + secretsReconciler: NewSecretsReconciler(gardenClient), + lifeCycleReconciler: NewLifecycleReconciler(gardenClient, config), + seedBackupBucketsCheckReconciler: NewBackupBucketsCheckReconciler(gardenClient, *config.Controllers.SeedBackupBucketsCheck, clock.RealClock{}), + extensionsCheckReconciler: NewExtensionsCheckReconciler(gardenClient, *config.Controllers.SeedExtensionsCheck, clock.RealClock{}), - secretsQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Seed Secrets"), - seedBackupBucketQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Backup Bucket"), - seedLifecycleQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Seed Lifecycle"), - seedExtensionsCheckQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Extensions Check"), - workerCh: make(chan int), + secretsQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Seed Secrets"), + seedBackupBucketsCheckQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Backup Buckets Check"), + seedLifecycleQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Seed Lifecycle"), + seedExtensionsCheckQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Extensions Check"), + workerCh: make(chan int), } backupBucketInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -149,7 +150,7 @@ func NewSeedController( } // Run runs the Controller until the given stop channel can be read from. -func (c *Controller) Run(ctx context.Context, seedWorkers, seedExtensionsCheckWorkers int) { +func (c *Controller) Run(ctx context.Context, seedWorkers, seedBackupBucketsCheckWorkers, seedExtensionsCheckWorkers int) { if !cache.WaitForCacheSync(ctx.Done(), c.hasSyncedFuncs...) { c.log.Error(wait.ErrWaitTimeout, "Timed out waiting for caches to sync") return @@ -168,9 +169,10 @@ func (c *Controller) Run(ctx context.Context, seedWorkers, seedExtensionsCheckWo for i := 0; i < seedWorkers; i++ { controllerutils.CreateWorker(ctx, c.secretsQueue, "Seed Secrets", c.secretsReconciler, &waitGroup, c.workerCh, controllerutils.WithLogger(c.log.WithName(seedSecretsReconcilerName))) controllerutils.CreateWorker(ctx, c.seedLifecycleQueue, "Seed Lifecycle", c.lifeCycleReconciler, &waitGroup, c.workerCh, controllerutils.WithLogger(c.log.WithName(seedLifecycleReconcilerName))) - controllerutils.CreateWorker(ctx, c.seedBackupBucketQueue, "Seed Backup Bucket", c.seedBackupReconciler, &waitGroup, c.workerCh, controllerutils.WithLogger(c.log.WithName(seedBackupBucketReconcilerName))) } - + for i := 0; i < seedBackupBucketsCheckWorkers; i++ { + controllerutils.CreateWorker(ctx, c.seedBackupBucketsCheckQueue, "Seed Backup Bucket Check", c.seedBackupBucketsCheckReconciler, &waitGroup, c.workerCh, controllerutils.WithLogger(c.log.WithName(seedBackupBucketsCheckReconcilerName))) + } for i := 0; i < seedExtensionsCheckWorkers; i++ { controllerutils.CreateWorker(ctx, c.seedExtensionsCheckQueue, "Seed Extension Check", c.extensionsCheckReconciler, &waitGroup, c.workerCh, controllerutils.WithLogger(c.log.WithName(extensionCheckReconcilerName))) } @@ -178,12 +180,12 @@ func (c *Controller) Run(ctx context.Context, seedWorkers, seedExtensionsCheckWo // Shutdown handling <-ctx.Done() c.secretsQueue.ShutDown() - c.seedBackupBucketQueue.ShutDown() + c.seedBackupBucketsCheckQueue.ShutDown() c.seedLifecycleQueue.ShutDown() c.seedExtensionsCheckQueue.ShutDown() for { - queueLength := c.secretsQueue.Len() + c.seedBackupBucketQueue.Len() + c.seedLifecycleQueue.Len() + c.seedExtensionsCheckQueue.Len() + queueLength := c.secretsQueue.Len() + c.seedBackupBucketsCheckQueue.Len() + c.seedLifecycleQueue.Len() + c.seedExtensionsCheckQueue.Len() if queueLength == 0 && c.numberOfRunningWorkers == 0 { c.log.V(1).Info("No running Seed worker and no items left in the queues. Terminating Seed controller") break @@ -195,10 +197,72 @@ func (c *Controller) Run(ctx context.Context, seedWorkers, seedExtensionsCheckWo waitGroup.Wait() } -func reconcileAfter(d time.Duration) (reconcile.Result, error) { - return reconcile.Result{RequeueAfter: d}, nil +func setToProgressingOrUnknown( + clock clock.Clock, + conditionThreshold time.Duration, + condition gardencorev1beta1.Condition, + reason, message string, + codes ...gardencorev1beta1.ErrorCode, +) gardencorev1beta1.Condition { + return setToProgressingIfWithinThreshold(clock, conditionThreshold, condition, gardencorev1beta1.ConditionUnknown, reason, message, codes...) +} + +func setToProgressingOrFalse( + clock clock.Clock, + conditionThreshold time.Duration, + condition gardencorev1beta1.Condition, + reason, message string, + codes ...gardencorev1beta1.ErrorCode, +) gardencorev1beta1.Condition { + return setToProgressingIfWithinThreshold(clock, conditionThreshold, condition, gardencorev1beta1.ConditionFalse, reason, message, codes...) +} + +func setToProgressingIfWithinThreshold( + clock clock.Clock, + conditionThreshold time.Duration, + condition gardencorev1beta1.Condition, + eventualConditionStatus gardencorev1beta1.ConditionStatus, + reason, message string, + codes ...gardencorev1beta1.ErrorCode, +) gardencorev1beta1.Condition { + switch condition.Status { + case gardencorev1beta1.ConditionTrue: + if conditionThreshold == 0 { + return gardencorev1beta1helper.UpdatedCondition(condition, eventualConditionStatus, reason, message, codes...) + } + return gardencorev1beta1helper.UpdatedCondition(condition, gardencorev1beta1.ConditionProgressing, reason, message, codes...) + + case gardencorev1beta1.ConditionProgressing: + if conditionThreshold == 0 { + return gardencorev1beta1helper.UpdatedCondition(condition, eventualConditionStatus, reason, message, codes...) + } + + if delta := clock.Now().UTC().Sub(condition.LastTransitionTime.Time.UTC()); delta <= conditionThreshold { + return gardencorev1beta1helper.UpdatedCondition(condition, gardencorev1beta1.ConditionProgressing, reason, message, codes...) + } + return gardencorev1beta1helper.UpdatedCondition(condition, eventualConditionStatus, reason, message, codes...) + } + + return gardencorev1beta1helper.UpdatedCondition(condition, eventualConditionStatus, reason, message, codes...) +} + +func getThresholdForCondition(conditions []config.ConditionThreshold, conditionType gardencorev1beta1.ConditionType) time.Duration { + for _, threshold := range conditions { + if threshold.Type == string(conditionType) { + return threshold.Duration.Duration + } + } + return 0 } -func reconcileResult(err error) (reconcile.Result, error) { - return reconcile.Result{}, err +func patchSeedCondition(ctx context.Context, c client.StatusClient, seed *gardencorev1beta1.Seed, condition gardencorev1beta1.Condition) error { + patch := client.StrategicMergeFrom(seed.DeepCopy()) + + conditions := gardencorev1beta1helper.MergeConditions(seed.Status.Conditions, condition) + if !gardencorev1beta1helper.ConditionsNeedUpdate(seed.Status.Conditions, conditions) { + return nil + } + + seed.Status.Conditions = conditions + return c.Status().Patch(ctx, seed, patch) } diff --git a/pkg/controllermanager/controller/seed/seed_backup_bucket_reconcile.go b/pkg/controllermanager/controller/seed/seed_backup_bucket_check_control.go similarity index 63% rename from pkg/controllermanager/controller/seed/seed_backup_bucket_reconcile.go rename to pkg/controllermanager/controller/seed/seed_backup_bucket_check_control.go index 2a64711e633..9c477b7b369 100644 --- a/pkg/controllermanager/controller/seed/seed_backup_bucket_reconcile.go +++ b/pkg/controllermanager/controller/seed/seed_backup_bucket_check_control.go @@ -18,8 +18,8 @@ import ( "context" "fmt" - apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -27,9 +27,10 @@ import ( "github.com/gardener/gardener/pkg/apis/core" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/controllermanager/apis/config" ) -const seedBackupBucketReconcilerName = "backupbucket" +const seedBackupBucketsCheckReconcilerName = "backupbuckets-check" func (c *Controller) backupBucketEnqueue(bb *gardencorev1beta1.BackupBucket) { seedName := bb.Spec.SeedName @@ -37,7 +38,7 @@ func (c *Controller) backupBucketEnqueue(bb *gardencorev1beta1.BackupBucket) { return } - c.seedBackupBucketQueue.Add(*seedName) + c.seedBackupBucketsCheckQueue.Add(*seedName) } func (c *Controller) backupBucketAdd(obj interface{}) { @@ -57,20 +58,25 @@ func (c *Controller) backupBucketUpdate(oldObj, newObj interface{}) { return } - if !apiequality.Semantic.DeepEqual(oldBackupBucket.Status, newBackupBucket.Status) || !apiequality.Semantic.DeepEqual(oldBackupBucket.Spec, newBackupBucket.Spec) { + if lastErrorChanged(oldBackupBucket.Status.LastError, newBackupBucket.Status.LastError) { c.backupBucketEnqueue(newBackupBucket) } } -// NewBackupBucketReconciler returns a new default control to checks backup buckets of related seeds. -func NewBackupBucketReconciler(gardenClient client.Client) *backupBucketReconciler { - return &backupBucketReconciler{ +// NewBackupBucketsCheckReconciler creates a new reconciler that maintains the BackupBucketsReady condition of Seeds +// according to the observed status of BackupBuckets. +func NewBackupBucketsCheckReconciler(gardenClient client.Client, config config.SeedBackupBucketsCheckControllerConfiguration, clock clock.Clock) *backupBucketsCheckReconciler { + return &backupBucketsCheckReconciler{ gardenClient: gardenClient, + config: config, + clock: clock, } } -type backupBucketReconciler struct { +type backupBucketsCheckReconciler struct { gardenClient client.Client + config config.SeedBackupBucketsCheckControllerConfiguration + clock clock.Clock } type backupBucketInfo struct { @@ -78,11 +84,11 @@ type backupBucketInfo struct { errorMsg string } -func (b *backupBucketInfo) String() string { +func (b backupBucketInfo) String() string { return fmt.Sprintf("Name: %s, Error: %s", b.name, b.errorMsg) } -func (b *backupBucketReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { +func (b *backupBucketsCheckReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) seed := &gardencorev1beta1.Seed{} @@ -96,7 +102,7 @@ func (b *backupBucketReconciler) Reconcile(ctx context.Context, req reconcile.Re backupBucketList := &gardencorev1beta1.BackupBucketList{} if err := b.gardenClient.List(ctx, backupBucketList, client.MatchingFields{core.BackupBucketSeedName: seed.Name}); err != nil { - return reconcileResult(err) + return reconcile.Result{}, err } conditionBackupBucketsReady := gardencorev1beta1helper.GetOrInitCondition(seed.Status.Conditions, gardencorev1beta1.SeedBackupBucketsReady) @@ -115,40 +121,34 @@ func (b *backupBucketReconciler) Reconcile(ctx context.Context, req reconcile.Re } } + conditionThreshold := getThresholdForCondition(b.config.ConditionThresholds, gardencorev1beta1.SeedBackupBucketsReady) switch { case len(erroneousBackupBuckets) > 0: errorMsg := "The following BackupBuckets have issues:" for _, bb := range erroneousBackupBuckets { - errorMsg += fmt.Sprintf("\n* %s", bb.String()) + errorMsg += fmt.Sprintf("\n* %s", bb) } - - if updateErr := patchSeedCondition(ctx, b.gardenClient, seed, gardencorev1beta1helper.UpdatedCondition(conditionBackupBucketsReady, - gardencorev1beta1.ConditionFalse, "BackupBucketsError", errorMsg)); updateErr != nil { - return reconcileResult(updateErr) + conditionBackupBucketsReady = setToProgressingOrFalse(b.clock, conditionThreshold, conditionBackupBucketsReady, "BackupBucketsError", errorMsg) + if updateErr := patchSeedCondition(ctx, b.gardenClient, seed, conditionBackupBucketsReady); updateErr != nil { + return reconcile.Result{}, updateErr } case bbCount > 0: if updateErr := patchSeedCondition(ctx, b.gardenClient, seed, gardencorev1beta1helper.UpdatedCondition(conditionBackupBucketsReady, gardencorev1beta1.ConditionTrue, "BackupBucketsAvailable", "Backup Buckets are available.")); updateErr != nil { - return reconcileResult(updateErr) + return reconcile.Result{}, updateErr } case bbCount == 0: - if updateErr := patchSeedCondition(ctx, b.gardenClient, seed, gardencorev1beta1helper.UpdatedCondition(conditionBackupBucketsReady, - gardencorev1beta1.ConditionUnknown, "BackupBucketsGone", "Backup Buckets are gone.")); updateErr != nil { - return reconcileResult(updateErr) + conditionBackupBucketsReady = setToProgressingOrUnknown(b.clock, conditionThreshold, conditionBackupBucketsReady, "BackupBucketsGone", "Backup Buckets are gone.") + if updateErr := patchSeedCondition(ctx, b.gardenClient, seed, conditionBackupBucketsReady); updateErr != nil { + return reconcile.Result{}, updateErr } } - return reconcileResult(nil) + return reconcile.Result{RequeueAfter: b.config.SyncPeriod.Duration}, nil } -func patchSeedCondition(ctx context.Context, c client.StatusClient, seed *gardencorev1beta1.Seed, condition gardencorev1beta1.Condition) error { - patch := client.StrategicMergeFrom(seed.DeepCopy()) - - conditions := gardencorev1beta1helper.MergeConditions(seed.Status.Conditions, condition) - if !gardencorev1beta1helper.ConditionsNeedUpdate(seed.Status.Conditions, conditions) { - return nil - } - - seed.Status.Conditions = conditions - return c.Status().Patch(ctx, seed, patch) +func lastErrorChanged(oldLastError, newLastError *gardencorev1beta1.LastError) bool { + return oldLastError == nil && newLastError != nil || + oldLastError != nil && newLastError == nil || + oldLastError != nil && newLastError != nil && oldLastError.Description != newLastError.Description } diff --git a/pkg/controllermanager/controller/seed/seed_backup_bucket_check_control_test.go b/pkg/controllermanager/controller/seed/seed_backup_bucket_check_control_test.go new file mode 100644 index 00000000000..cbc71998f1a --- /dev/null +++ b/pkg/controllermanager/controller/seed/seed_backup_bucket_check_control_test.go @@ -0,0 +1,293 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package seed_test + +import ( + "context" + "time" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/controllermanager/apis/config" + . "github.com/gardener/gardener/pkg/controllermanager/controller/seed" + backupbucketstrategy "github.com/gardener/gardener/pkg/registry/core/backupbucket" + "github.com/gardener/gardener/pkg/utils/test" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + "github.com/onsi/gomega/types" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + testclock "k8s.io/utils/clock/testing" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var _ = Describe("BackupBucketsCheckReconciler", func() { + const syncPeriod = 1 * time.Second + + var ( + ctx context.Context + c client.Client + + conf config.SeedBackupBucketsCheckControllerConfiguration + fakeClock *testclock.FakeClock + + expectedRequeueAfter time.Duration + ) + + Describe("#Reconcile", func() { + var ( + seed *gardencorev1beta1.Seed + backupBuckets []gardencorev1beta1.BackupBucket + + reconciler reconcile.Reconciler + request reconcile.Request + + matchExpectedCondition types.GomegaMatcher + ) + + BeforeEach(func() { + seed = &gardencorev1beta1.Seed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "seed", + }, + } + + request = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(seed)} + + fakeClock = testclock.NewFakeClock(time.Now().Round(time.Second)) + + c = test.NewClientWithFieldSelectorSupport( + fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).WithObjects(seed).Build(), + backupbucketstrategy.ToSelectableFields, + ) + + conf = config.SeedBackupBucketsCheckControllerConfiguration{ + SyncPeriod: &metav1.Duration{Duration: syncPeriod}, + } + + expectedRequeueAfter = syncPeriod + }) + + JustBeforeEach(func() { + reconciler = NewBackupBucketsCheckReconciler(c, conf, fakeClock) + + for _, backupBucket := range backupBuckets { + Expect(c.Create(ctx, &backupBucket)).To(Succeed()) + } + }) + + AfterEach(func() { + result, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(seed)}) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{RequeueAfter: expectedRequeueAfter})) + + if err := c.Get(ctx, request.NamespacedName, seed); !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + Expect(seed.Status.Conditions).To(ConsistOf(matchExpectedCondition)) + } + }) + + It("should do nothing if Seed is gone", func() { + Expect(c.Delete(ctx, seed)).To(Succeed()) + expectedRequeueAfter = 0 + }) + + Context("when Seed has healthy backup buckets", func() { + BeforeEach(func() { + backupBuckets = []gardencorev1beta1.BackupBucket{ + createBackupBucket("1", seed.Name, nil), + createBackupBucket("2", "fooSeed", nil), + createBackupBucket("3", "barSeed", nil), + createBackupBucket("4", seed.Name, nil), + } + }) + + It("should set condition to `True` when none was given", func() { + matchExpectedCondition = MatchFields(IgnoreExtras, Fields{ + "Message": Equal("Backup Buckets are available."), + "Reason": Equal("BackupBucketsAvailable"), + "Status": Equal(gardencorev1beta1.ConditionTrue), + "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), + }) + }) + + It("should set condition to `True` when it was false", func() { + seed.Status.Conditions = []gardencorev1beta1.Condition{ + { + Message: "foo", + Reason: "bar", + Status: gardencorev1beta1.ConditionFalse, + Type: gardencorev1beta1.SeedBackupBucketsReady, + }, + } + Expect(c.Update(ctx, seed)).To(Succeed()) + + matchExpectedCondition = MatchFields(IgnoreExtras, Fields{ + "Message": Equal("Backup Buckets are available."), + "Reason": Equal("BackupBucketsAvailable"), + "Status": Equal(gardencorev1beta1.ConditionTrue), + "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), + }) + }) + }) + + Context("when there is a problem with the Seed's backup buckets", func() { + var tests = func(expectedConditionStatus gardencorev1beta1.ConditionStatus, reason string, matchMessage types.GomegaMatcher) { + It("should set correct condition status", func() { + seed.Status.Conditions = []gardencorev1beta1.Condition{ + { + Message: "Backup Buckets are available.", + Reason: "BackupBucketsAvailable", + Status: gardencorev1beta1.ConditionTrue, + Type: gardencorev1beta1.SeedBackupBucketsReady, + }, + } + Expect(c.Update(ctx, seed)).To(Succeed()) + + matchExpectedCondition = MatchFields(IgnoreExtras, Fields{ + "Message": matchMessage, + "Reason": Equal(reason), + "Status": Equal(expectedConditionStatus), + "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), + }) + }) + + Context("when condition threshold is set", func() { + BeforeEach(func() { + conf = config.SeedBackupBucketsCheckControllerConfiguration{ + SyncPeriod: &metav1.Duration{Duration: syncPeriod}, + ConditionThresholds: []config.ConditionThreshold{{ + Type: string(gardencorev1beta1.SeedBackupBucketsReady), + Duration: metav1.Duration{Duration: time.Minute}, + }}, + } + }) + + It("should set condition to `Progressing`", func() { + seed.Status.Conditions = []gardencorev1beta1.Condition{ + { + Message: "Backup Buckets are available.", + Reason: "BackupBucketsAvailable", + Status: gardencorev1beta1.ConditionTrue, + Type: gardencorev1beta1.SeedBackupBucketsReady, + LastTransitionTime: metav1.Time{Time: fakeClock.Now().Add(-30 * time.Second)}, + LastUpdateTime: metav1.Time{Time: fakeClock.Now().Add(-30 * time.Second)}, + }, + } + Expect(c.Update(ctx, seed)).To(Succeed()) + + matchExpectedCondition = MatchFields(IgnoreExtras, Fields{ + "Message": matchMessage, + "Reason": Equal(reason), + "Status": Equal(gardencorev1beta1.ConditionProgressing), + "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), + }) + }) + + It("should set correct condition status when condition threshold expires", func() { + seed.Status.Conditions = []gardencorev1beta1.Condition{ + { + Message: "foo", + Reason: "BackupBucketsError", + Status: gardencorev1beta1.ConditionProgressing, + Type: gardencorev1beta1.SeedBackupBucketsReady, + LastTransitionTime: metav1.Time{Time: fakeClock.Now().Add(-2 * time.Minute)}, + LastUpdateTime: metav1.Time{Time: fakeClock.Now().Add(-2 * time.Minute)}, + }, + } + Expect(c.Update(ctx, seed)).To(Succeed()) + + matchExpectedCondition = MatchFields(IgnoreExtras, Fields{ + "Message": matchMessage, + "Reason": Equal(reason), + "Status": Equal(expectedConditionStatus), + "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), + }) + }) + }) + } + + Context("when Seed has unhealthy backup buckets", func() { + BeforeEach(func() { + backupBuckets = []gardencorev1beta1.BackupBucket{ + createBackupBucket("1", seed.Name, &gardencorev1beta1.LastError{Description: "foo error"}), + createBackupBucket("2", "fooSeed", nil), + createBackupBucket("3", seed.Name, &gardencorev1beta1.LastError{Description: "bar error"}), + createBackupBucket("4", "barSeed", nil), + } + }) + + tests(gardencorev1beta1.ConditionFalse, "BackupBucketsError", SatisfyAll(ContainSubstring("Name: 1, Error: foo error"), ContainSubstring("Name: 3, Error: bar error"))) + }) + + Context("when a Seed's backup buckets are gone", func() { + BeforeEach(func() { + backupBuckets = []gardencorev1beta1.BackupBucket{ + createBackupBucket("1", "fooSeed", &gardencorev1beta1.LastError{Description: "foo error"}), + createBackupBucket("2", "barSeed", nil), + } + }) + + tests(gardencorev1beta1.ConditionUnknown, "BackupBucketsGone", Equal("Backup Buckets are gone.")) + }) + + Context("when a Seed's unhealthy backup bucket switches", func() { + BeforeEach(func() { + backupBuckets = []gardencorev1beta1.BackupBucket{ + createBackupBucket("1", seed.Name, &gardencorev1beta1.LastError{Description: "foo error"}), + createBackupBucket("2", seed.Name, nil), + } + }) + + It("should set condition to `False` and remove successful bucket from message", func() { + seed.Status.Conditions = []gardencorev1beta1.Condition{ + { + Message: "The following BackupBuckets have issues:\n* Name: 2, Error: some error", + Reason: "BackupBucketsError", + Status: gardencorev1beta1.ConditionFalse, + Type: gardencorev1beta1.SeedBackupBucketsReady, + }, + } + Expect(c.Update(ctx, seed)).To(Succeed()) + + matchExpectedCondition = MatchFields(IgnoreExtras, Fields{ + "Message": Equal("The following BackupBuckets have issues:\n* Name: 1, Error: foo error"), + "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), + "Status": Equal(gardencorev1beta1.ConditionFalse), + }) + }) + }) + }) + }) +}) + +func createBackupBucket(name, seedName string, lastErr *gardencorev1beta1.LastError) gardencorev1beta1.BackupBucket { + return gardencorev1beta1.BackupBucket{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: gardencorev1beta1.BackupBucketSpec{ + SeedName: pointer.String(seedName), + }, + Status: gardencorev1beta1.BackupBucketStatus{ + LastError: lastErr, + }, + } +} diff --git a/pkg/controllermanager/controller/seed/seed_backup_bucket_reconcile_test.go b/pkg/controllermanager/controller/seed/seed_backup_bucket_reconcile_test.go deleted file mode 100644 index b3e83e2ef33..00000000000 --- a/pkg/controllermanager/controller/seed/seed_backup_bucket_reconcile_test.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package seed_test - -import ( - "context" - "encoding/json" - - gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - . "github.com/gardener/gardener/pkg/controllermanager/controller/seed" - mockclient "github.com/gardener/gardener/pkg/mock/controller-runtime/client" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" - - "github.com/golang/mock/gomock" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -var _ = Describe("BackupBucketReconciler", func() { - var ( - ctx = context.TODO() - ctrl *gomock.Controller - ) - - BeforeEach(func() { - ctrl = gomock.NewController(GinkgoT()) - }) - - AfterEach(func() { - ctrl.Finish() - }) - - Describe("#Reconcile", func() { - var ( - c *mockclient.MockClient - sw *mockclient.MockStatusWriter - - seed, seedPatch *gardencorev1beta1.Seed - bbs []gardencorev1beta1.BackupBucket - - control reconcile.Reconciler - ) - - BeforeEach(func() { - c = mockclient.NewMockClient(ctrl) - sw = mockclient.NewMockStatusWriter(ctrl) - c.EXPECT().Status().Return(sw).AnyTimes() - seed = &gardencorev1beta1.Seed{ - ObjectMeta: metav1.ObjectMeta{ - Name: "seed", - }, - } - - seedPatch = &gardencorev1beta1.Seed{} - }) - - JustBeforeEach(func() { - sw.EXPECT().Patch(gomock.Any(), gomock.AssignableToTypeOf(seed), gomock.Any()).DoAndReturn( - func(_ context.Context, obj client.Object, patch client.Patch, _ ...client.PatchOption) error { - patchData, err := patch.Data(obj) - Expect(err).NotTo(HaveOccurred()) - Expect(json.Unmarshal(patchData, seedPatch)).To(Succeed()) - return nil - }) - - control = NewBackupBucketReconciler(c) - - c.EXPECT().Get(ctx, kutil.Key(seed.Name), gomock.AssignableToTypeOf(&gardencorev1beta1.Seed{})).DoAndReturn(func(_ context.Context, _ client.ObjectKey, obj *gardencorev1beta1.Seed) error { - *obj = *seed - return nil - }) - }) - - Context("when Seed has healthy backup buckets", func() { - BeforeEach(func() { - bbs = []gardencorev1beta1.BackupBucket{ - createBackupBucket("1", seed.Name, nil), - createBackupBucket("2", "fooSeed", nil), - createBackupBucket("3", "barSeed", nil), - createBackupBucket("4", seed.Name, nil), - } - - c.EXPECT().List(ctx, gomock.AssignableToTypeOf(&gardencorev1beta1.BackupBucketList{}), client.MatchingFields{"spec.seedName": seed.Name}).DoAndReturn(func(ctx context.Context, list *gardencorev1beta1.BackupBucketList, opts ...client.ListOption) error { - (&gardencorev1beta1.BackupBucketList{Items: backupBucketsForSeed(bbs, seed.Name)}).DeepCopyInto(list) - return nil - }) - }) - - It("should set condition to `True` when none was given", func() { - result, err := control.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(seed)}) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal(reconcile.Result{})) - Expect(seedPatch.Status.Conditions).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "Message": Equal("Backup Buckets are available."), - "Reason": Equal("BackupBucketsAvailable"), - "Status": Equal(gardencorev1beta1.ConditionTrue), - "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), - }), - )) - }) - - It("should set condition to `True` when one was false", func() { - seed.Status.Conditions = []gardencorev1beta1.Condition{ - { - Message: "foo", - Reason: "bar", - Status: gardencorev1beta1.ConditionTrue, - Type: gardencorev1beta1.SeedExtensionsReady, - }, - { - Message: "foo", - Reason: "bar", - Status: gardencorev1beta1.ConditionFalse, - Type: gardencorev1beta1.SeedBackupBucketsReady, - }, - } - - result, err := control.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(seed)}) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal(reconcile.Result{})) - Expect(seedPatch.Status.Conditions).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "Message": Equal("Backup Buckets are available."), - "Reason": Equal("BackupBucketsAvailable"), - "Status": Equal(gardencorev1beta1.ConditionTrue), - "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), - }), - )) - }) - }) - - Context("when Seed has unhealthy backup buckets", func() { - BeforeEach(func() { - bbs = []gardencorev1beta1.BackupBucket{ - createBackupBucket("1", seed.Name, &gardencorev1beta1.LastError{Description: "foo error"}), - createBackupBucket("2", "fooSeed", nil), - createBackupBucket("3", seed.Name, &gardencorev1beta1.LastError{Description: "bar error"}), - createBackupBucket("4", "barSeed", nil), - } - - c.EXPECT().List(ctx, gomock.AssignableToTypeOf(&gardencorev1beta1.BackupBucketList{}), client.MatchingFields{"spec.seedName": seed.Name}).DoAndReturn(func(ctx context.Context, list *gardencorev1beta1.BackupBucketList, opts ...client.ListOption) error { - (&gardencorev1beta1.BackupBucketList{Items: backupBucketsForSeed(bbs, seed.Name)}).DeepCopyInto(list) - return nil - }) - }) - - It("should set condition to `False`", func() { - result, err := control.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(seed)}) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal(reconcile.Result{})) - Expect(seedPatch.Status.Conditions).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "Message": SatisfyAll(ContainSubstring("Name: 1, Error: foo error"), ContainSubstring("Name: 3, Error: bar error")), - "Reason": Equal("BackupBucketsError"), - "Status": Equal(gardencorev1beta1.ConditionFalse), - "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), - }), - )) - }) - }) - - Context("when a Seed's unhealthy backup bucket switches", func() { - BeforeEach(func() { - bbs = []gardencorev1beta1.BackupBucket{ - createBackupBucket("1", seed.Name, &gardencorev1beta1.LastError{Description: "foo error"}), - createBackupBucket("2", seed.Name, nil), - } - c.EXPECT().List(ctx, gomock.AssignableToTypeOf(&gardencorev1beta1.BackupBucketList{}), client.MatchingFields{"spec.seedName": seed.Name}).DoAndReturn(func(ctx context.Context, list *gardencorev1beta1.BackupBucketList, opts ...client.ListOption) error { - (&gardencorev1beta1.BackupBucketList{Items: backupBucketsForSeed(bbs, seed.Name)}).DeepCopyInto(list) - return nil - }) - }) - - It("should set condition to `False` and remove successful bucket from message", func() { - seed.Status.Conditions = []gardencorev1beta1.Condition{ - { - Message: "The following BackupBuckets have issues:\n* Name: 2, Error: some error", - Reason: "BackupBucketsError", - Status: gardencorev1beta1.ConditionFalse, - Type: gardencorev1beta1.SeedBackupBucketsReady, - }, - } - result, err := control.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(seed)}) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal(reconcile.Result{})) - Expect(seedPatch.Status.Conditions).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "Message": Equal("The following BackupBuckets have issues:\n* Name: 1, Error: foo error"), - "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), - }), - )) - }) - }) - - Context("when a Seed's backup buckets are gone", func() { - BeforeEach(func() { - bbs = []gardencorev1beta1.BackupBucket{ - createBackupBucket("1", "fooSeed", &gardencorev1beta1.LastError{Description: "foo error"}), - createBackupBucket("2", "barSeed", nil), - } - c.EXPECT().List(ctx, gomock.AssignableToTypeOf(&gardencorev1beta1.BackupBucketList{}), client.MatchingFields{"spec.seedName": seed.Name}).DoAndReturn(func(ctx context.Context, list *gardencorev1beta1.BackupBucketList, opts ...client.ListOption) error { - (&gardencorev1beta1.BackupBucketList{Items: backupBucketsForSeed(bbs, seed.Name)}).DeepCopyInto(list) - return nil - }) - }) - - It("should set condition to `False` and remove successful bucket from message", func() { - seed.Status.Conditions = []gardencorev1beta1.Condition{ - { - Message: "Backup Buckets are available.", - Reason: "BackupBucketsAvailable", - Status: gardencorev1beta1.ConditionTrue, - Type: gardencorev1beta1.SeedBackupBucketsReady, - }, - } - result, err := control.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(seed)}) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal(reconcile.Result{})) - Expect(seedPatch.Status.Conditions).To(ConsistOf( - MatchFields(IgnoreExtras, Fields{ - "Message": Equal("Backup Buckets are gone."), - "Reason": Equal("BackupBucketsGone"), - "Status": Equal(gardencorev1beta1.ConditionUnknown), - "Type": Equal(gardencorev1beta1.SeedBackupBucketsReady), - }), - )) - }) - }) - }) -}) - -func createBackupBucket(name, seedName string, lastErr *gardencorev1beta1.LastError) gardencorev1beta1.BackupBucket { - return gardencorev1beta1.BackupBucket{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: gardencorev1beta1.BackupBucketSpec{ - SeedName: pointer.String(seedName), - }, - Status: gardencorev1beta1.BackupBucketStatus{ - LastError: lastErr, - }, - } -} - -func backupBucketsForSeed(items []gardencorev1beta1.BackupBucket, seedName string) []gardencorev1beta1.BackupBucket { - var out []gardencorev1beta1.BackupBucket - - for _, item := range items { - if pointer.StringPtrDerefOr(item.Spec.SeedName, "") == seedName { - out = append(out, item) - } - } - - return out -} diff --git a/pkg/controllermanager/controller/seed/seed_extension_check_control.go b/pkg/controllermanager/controller/seed/seed_extension_check_control.go index 18f16eadac8..2abff85fa65 100644 --- a/pkg/controllermanager/controller/seed/seed_extension_check_control.go +++ b/pkg/controllermanager/controller/seed/seed_extension_check_control.go @@ -17,7 +17,6 @@ package seed import ( "context" "fmt" - "time" "github.com/gardener/gardener/pkg/apis/core" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -163,65 +162,22 @@ func (r *extensionCheckReconciler) reconcile(ctx context.Context, seed *gardenco } condition := helper.GetOrInitCondition(seed.Status.Conditions, gardencorev1beta1.SeedExtensionsReady) - extensionsReadyThreshold := r.getExtensionsReadyThreshold() + extensionsReadyThreshold := getThresholdForCondition(r.config.ConditionThresholds, gardencorev1beta1.SeedExtensionsReady) switch { case len(notValid) != 0: - condition = r.failedCondition(extensionsReadyThreshold, condition, "NotAllExtensionsValid", fmt.Sprintf("Some extensions are not valid: %+v", notValid)) + condition = setToProgressingOrFalse(r.clock, extensionsReadyThreshold, condition, "NotAllExtensionsValid", fmt.Sprintf("Some extensions are not valid: %+v", notValid)) case len(notInstalled) != 0: - condition = r.failedCondition(extensionsReadyThreshold, condition, "NotAllExtensionsInstalled", fmt.Sprintf("Some extensions are not installed: %+v", notInstalled)) + condition = setToProgressingOrFalse(r.clock, extensionsReadyThreshold, condition, "NotAllExtensionsInstalled", fmt.Sprintf("Some extensions are not installed: %+v", notInstalled)) case len(notHealthy) != 0: - condition = r.failedCondition(extensionsReadyThreshold, condition, "NotAllExtensionsHealthy", fmt.Sprintf("Some extensions are not healthy: %+v", notHealthy)) + condition = setToProgressingOrFalse(r.clock, extensionsReadyThreshold, condition, "NotAllExtensionsHealthy", fmt.Sprintf("Some extensions are not healthy: %+v", notHealthy)) case len(progressing) != 0: - condition = r.failedCondition(extensionsReadyThreshold, condition, "SomeExtensionsProgressing", fmt.Sprintf("Some extensions are progressing: %+v", progressing)) + condition = setToProgressingOrFalse(r.clock, extensionsReadyThreshold, condition, "SomeExtensionsProgressing", fmt.Sprintf("Some extensions are progressing: %+v", progressing)) default: condition = helper.UpdatedCondition(condition, gardencorev1beta1.ConditionTrue, "AllExtensionsReady", "All extensions installed into the seed cluster are ready and healthy.") } - // patch ExtensionsReady condition - patch := client.StrategicMergeFrom(seed.DeepCopy()) - newConditions := helper.MergeConditions(seed.Status.Conditions, condition) - if !helper.ConditionsNeedUpdate(seed.Status.Conditions, newConditions) { - return nil - } - seed.Status.Conditions = newConditions - return r.gardenClient.Status().Patch(ctx, seed, patch) -} - -func (r *extensionCheckReconciler) getExtensionsReadyThreshold() time.Duration { - for _, threshold := range r.config.ConditionThresholds { - if threshold.Type == string(gardencorev1beta1.SeedExtensionsReady) { - return threshold.Duration.Duration - } - } - return 0 -} - -func (r *extensionCheckReconciler) failedCondition( - conditionThreshold time.Duration, - condition gardencorev1beta1.Condition, - reason, message string, - codes ...gardencorev1beta1.ErrorCode, -) gardencorev1beta1.Condition { - switch condition.Status { - case gardencorev1beta1.ConditionTrue: - if conditionThreshold == 0 { - return gardencorev1beta1helper.UpdatedCondition(condition, gardencorev1beta1.ConditionFalse, reason, message, codes...) - } - return gardencorev1beta1helper.UpdatedCondition(condition, gardencorev1beta1.ConditionProgressing, reason, message, codes...) - - case gardencorev1beta1.ConditionProgressing: - if conditionThreshold == 0 { - return gardencorev1beta1helper.UpdatedCondition(condition, gardencorev1beta1.ConditionFalse, reason, message, codes...) - } - - if delta := r.clock.Now().UTC().Sub(condition.LastTransitionTime.Time.UTC()); delta <= conditionThreshold { - return gardencorev1beta1helper.UpdatedCondition(condition, gardencorev1beta1.ConditionProgressing, reason, message, codes...) - } - return gardencorev1beta1helper.UpdatedCondition(condition, gardencorev1beta1.ConditionFalse, reason, message, codes...) - } - - return gardencorev1beta1helper.UpdatedCondition(condition, gardencorev1beta1.ConditionFalse, reason, message, codes...) + return patchSeedCondition(ctx, r.gardenClient, seed, condition) } func shouldEnqueueControllerInstallation(oldConditions, newConditions []gardencorev1beta1.Condition) bool { diff --git a/pkg/controllermanager/controller/seed/seed_lifecycle_reconcile.go b/pkg/controllermanager/controller/seed/seed_lifecycle_reconcile.go index c32a3b5e5c0..a39e88928d5 100644 --- a/pkg/controllermanager/controller/seed/seed_lifecycle_reconcile.go +++ b/pkg/controllermanager/controller/seed/seed_lifecycle_reconcile.go @@ -37,7 +37,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -const seedLifecycleReconcilerName = "lifecycle" +const ( + seedLifecycleReconcilerName = "lifecycle" + syncPeriod = 10 * time.Second + syncPeriodAfterExpiredMonitoringPeriod = 1 * time.Minute +) func (c *Controller) seedLifecycleAdd(obj interface{}) { key, err := cache.MetaNamespaceKeyFunc(obj) @@ -80,27 +84,27 @@ func (c *livecycleReconciler) Reconcile(ctx context.Context, req reconcile.Reque // New seeds don't have conditions - gardenlet never reported anything yet. Wait for grace period. if len(seed.Status.Conditions) == 0 { - return reconcileAfter(10 * time.Second) + return reconcile.Result{RequeueAfter: syncPeriod}, nil } observedSeedLease := &coordinationv1.Lease{} if err := c.gardenClient.Get(ctx, kutil.Key(gardencorev1beta1.GardenerSeedLeaseNamespace, seed.Name), observedSeedLease); client.IgnoreNotFound(err) != nil { - return reconcileResult(err) + return reconcile.Result{}, err } if observedSeedLease.Spec.RenewTime != nil { if observedSeedLease.Spec.RenewTime.UTC().After(time.Now().UTC().Add(-c.seedMonitorPeriod)) { - return reconcileAfter(10 * time.Second) + return reconcile.Result{RequeueAfter: syncPeriod}, nil } // Get the latest Lease object in cases which the LeaseLister cache is outdated, to ensure that the lease is really expired latestLeaseObject := &coordinationv1.Lease{} if err := c.gardenClient.Get(ctx, kutil.Key(gardencorev1beta1.GardenerSeedLeaseNamespace, seed.Name), latestLeaseObject); err != nil { - return reconcileResult(err) + return reconcile.Result{}, err } if latestLeaseObject.Spec.RenewTime.UTC().After(time.Now().UTC().Add(-c.seedMonitorPeriod)) { - return reconcileAfter(10 * time.Second) + return reconcile.Result{RequeueAfter: syncPeriod}, nil } } @@ -108,7 +112,7 @@ func (c *livecycleReconciler) Reconcile(ctx context.Context, req reconcile.Reque bldr, err := gardencorev1beta1helper.NewConditionBuilder(gardencorev1beta1.SeedGardenletReady) if err != nil { - return reconcileResult(err) + return reconcile.Result{}, err } conditionGardenletReady := gardencorev1beta1helper.GetCondition(seed.Status.Conditions, gardencorev1beta1.SeedGardenletReady) @@ -122,7 +126,7 @@ func (c *livecycleReconciler) Reconcile(ctx context.Context, req reconcile.Reque if newCondition, update := bldr.WithNowFunc(metav1.Now).Build(); update { seed.Status.Conditions = gardencorev1beta1helper.MergeConditions(seed.Status.Conditions, newCondition) if err := c.gardenClient.Status().Update(ctx, seed); err != nil { - return reconcileResult(err) + return reconcile.Result{}, err } } @@ -131,7 +135,7 @@ func (c *livecycleReconciler) Reconcile(ctx context.Context, req reconcile.Reque if seed.Status.ClientCertificateExpirationTimestamp != nil && seed.Status.ClientCertificateExpirationTimestamp.UTC().Before(time.Now().UTC()) { managedSeed, err := kutil.GetManagedSeedByName(ctx, c.gardenClient, seed.Name) if err != nil { - return reconcileResult(err) + return reconcile.Result{}, err } if managedSeed != nil { @@ -140,7 +144,7 @@ func (c *livecycleReconciler) Reconcile(ctx context.Context, req reconcile.Reque patch := client.MergeFrom(managedSeed.DeepCopy()) metav1.SetMetaDataAnnotation(&managedSeed.ObjectMeta, v1beta1constants.GardenerOperation, v1beta1constants.GardenerOperationReconcile) if err := c.gardenClient.Patch(ctx, managedSeed, patch); err != nil { - return reconcileResult(err) + return reconcile.Result{}, err } } } @@ -150,14 +154,14 @@ func (c *livecycleReconciler) Reconcile(ctx context.Context, req reconcile.Reque // anymore, hence, it most likely didn't check the shoot status. This means that the current shoot status might not reflect the truth // anymore. We are indicating this by marking it as `Unknown`. if conditionGardenletReady != nil && !conditionGardenletReady.LastTransitionTime.UTC().Before(time.Now().UTC().Add(-c.shootMonitorPeriod)) { - return reconcileAfter(10 * time.Second) + return reconcile.Result{RequeueAfter: syncPeriod}, nil } log.Info("Gardenlet has not sent heartbeat for at least the configured shoot monitor period, setting shoot conditions and constraints to 'Unknown' for all shoots on this seed", "shootMonitorPeriod", c.shootMonitorPeriod) shootList := &gardencorev1beta1.ShootList{} if err := c.gardenClient.List(ctx, shootList, client.MatchingFields{core.ShootSeedName: seed.Name}); err != nil { - return reconcileResult(err) + return reconcile.Result{}, err } var fns []flow.TaskFn @@ -170,10 +174,10 @@ func (c *livecycleReconciler) Reconcile(ctx context.Context, req reconcile.Reque } if err := flow.Parallel(fns...)(ctx); err != nil { - return reconcileResult(err) + return reconcile.Result{}, err } - return reconcileAfter(1 * time.Minute) + return reconcile.Result{RequeueAfter: syncPeriodAfterExpiredMonitoringPeriod}, nil } func setShootStatusToUnknown(ctx context.Context, c client.StatusClient, shoot *gardencorev1beta1.Shoot) error { diff --git a/pkg/controllermanager/controller/seed/seed_reconcile.go b/pkg/controllermanager/controller/seed/seed_reconcile.go index 9f48d35ccdb..5c92781d323 100644 --- a/pkg/controllermanager/controller/seed/seed_reconcile.go +++ b/pkg/controllermanager/controller/seed/seed_reconcile.go @@ -110,14 +110,14 @@ func (r *reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco syncedSecrets, err := r.syncGardenSecrets(ctx, r.gardenClient, namespace) if err != nil { - return reconcileResult(fmt.Errorf("failed to sync garden secrets: %v", err)) + return reconcile.Result{}, fmt.Errorf("failed to sync garden secrets: %v", err) } if err := r.cleanupStaleSecrets(ctx, r.gardenClient, syncedSecrets, namespace.Name); err != nil { - return reconcileResult(fmt.Errorf("failed to clean up secrets in seed namespace: %v", err)) + return reconcile.Result{}, fmt.Errorf("failed to clean up secrets in seed namespace: %v", err) } - return reconcileResult(nil) + return reconcile.Result{}, nil } var ( diff --git a/pkg/utils/test/client.go b/pkg/utils/test/client.go new file mode 100644 index 00000000000..057a21b3f49 --- /dev/null +++ b/pkg/utils/test/client.go @@ -0,0 +1,94 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "fmt" + + "github.com/gardener/gardener/pkg/client/kubernetes" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// NewClientWithFieldSelectorSupport takes a fake client and a function that returns selectable fields for the type T +// and adds support for field selectors to the client. +// TODO(plkokanov): remove this once the controller-runtime fake client supports field selectors. +func NewClientWithFieldSelectorSupport[T any](c client.Client, toSelectableFieldsFunc func(t *T) fields.Set) client.Client { + return &clientWithFieldSelectorSupport[T]{ + Client: c, + toSelectableFieldsFunc: toSelectableFieldsFunc, + } +} + +type clientWithFieldSelectorSupport[T any] struct { + client.Client + toSelectableFieldsFunc func(t *T) fields.Set +} + +func (c clientWithFieldSelectorSupport[T]) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error { + if err := c.Client.List(ctx, obj, opts...); err != nil { + return err + } + + listOpts := client.ListOptions{} + listOpts.ApplyOptions(opts) + + if listOpts.FieldSelector != nil { + objs, err := meta.ExtractList(obj) + if err != nil { + return err + } + filteredObjs, err := c.filterWithFieldSelector(objs, listOpts.FieldSelector) + if err != nil { + return err + } + err = meta.SetList(obj, filteredObjs) + if err != nil { + return err + } + } + + return nil +} + +func (c clientWithFieldSelectorSupport[T]) filterWithFieldSelector(objs []runtime.Object, sel fields.Selector) ([]runtime.Object, error) { + outItems := make([]runtime.Object, 0, len(objs)) + for _, obj := range objs { + // convert to internal + internalObj := new(T) + if err := kubernetes.GardenScheme.Convert(obj, internalObj, nil); err != nil { + return nil, err + } + + fieldSet := c.toSelectableFieldsFunc(internalObj) + + // complain about non-selectable fields if any + for _, req := range sel.Requirements() { + if !fieldSet.Has(req.Field) { + return nil, fmt.Errorf("field selector not supported for field %q", req.Field) + } + } + + if !sel.Matches(fieldSet) { + continue + } + outItems = append(outItems, obj.DeepCopyObject()) + } + return outItems, nil +} diff --git a/test/integration/controllermanager/seed/backupbucketscheck/backupbucketscheck_suite_test.go b/test/integration/controllermanager/seed/backupbucketscheck/backupbucketscheck_suite_test.go new file mode 100644 index 00000000000..9e0884d5708 --- /dev/null +++ b/test/integration/controllermanager/seed/backupbucketscheck/backupbucketscheck_suite_test.go @@ -0,0 +1,175 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package backupbucketscheck_test + +import ( + "context" + "testing" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/controllermanager/apis/config" + "github.com/gardener/gardener/pkg/controllermanager/controller/seed" + "github.com/gardener/gardener/pkg/controllerutils/mapper" + gardenerenvtest "github.com/gardener/gardener/pkg/envtest" + "github.com/gardener/gardener/pkg/logger" + "github.com/gardener/gardener/pkg/utils" + "github.com/gardener/gardener/pkg/utils/test" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/rest" + testclock "k8s.io/utils/clock/testing" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +func TestSeedBackupBucketsCheck(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Seed BackupBucketsCheck Controller Integration Test Suite") +} + +const testID = "backupbucketscheck-controller-test" + +var ( + testRunID = testID + "-" + utils.ComputeSHA256Hex([]byte(uuid.NewUUID()))[:8] + + ctx = context.Background() + log logr.Logger + + restConfig *rest.Config + testEnv *gardenerenvtest.GardenerTestEnvironment + testClient client.Client + + fakeClock *testclock.FakeClock +) + +var _ = BeforeSuite(func() { + logf.SetLogger(logger.MustNewZapLogger(logger.DebugLevel, logger.FormatJSON, zap.WriteTo(GinkgoWriter))) + log = logf.Log.WithName(testID) + + By("starting test environment") + testEnv = &gardenerenvtest.GardenerTestEnvironment{ + GardenerAPIServer: &gardenerenvtest.GardenerAPIServer{ + Args: []string{"--disable-admission-plugins=DeletionConfirmation,ResourceReferenceManager,SeedValidator,ExtensionValidator"}, + }, + } + + var err error + restConfig, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(restConfig).NotTo(BeNil()) + + DeferCleanup(func() { + By("stopping test environment") + Expect(testEnv.Stop()).To(Succeed()) + }) + + By("creating test client") + testClient, err = client.New(restConfig, client.Options{Scheme: kubernetes.GardenScheme}) + Expect(err).NotTo(HaveOccurred()) + + By("setup manager") + mgr, err := manager.New(restConfig, manager.Options{ + Scheme: kubernetes.GardenScheme, + MetricsBindAddress: "0", + NewCache: cache.BuilderWithOptions(cache.Options{ + SelectorsByObject: map[client.Object]cache.ObjectSelector{ + &gardencorev1beta1.BackupBucket{}: { + Label: labels.SelectorFromSet(labels.Set{testID: testRunID}), + }, + &gardencorev1beta1.Seed{}: { + Label: labels.SelectorFromSet(labels.Set{testID: testRunID}), + }, + }, + }), + }) + Expect(err).NotTo(HaveOccurred()) + + fakeClock = &testclock.FakeClock{} + //This is required so that the BackupsBucketReady condition is created with appropriate lastUpdateTimestamp and lastTransitionTimestamp. + DeferCleanup(test.WithVars( + &gardencorev1beta1helper.Now, func() metav1.Time { return metav1.Time{Time: fakeClock.Now()} }, + )) + + By("registering controller") + Expect(addSeedBackupBucketsCheckControllerToManager(mgr)).To(Succeed()) + + By("starting manager") + mgrContext, mgrCancel := context.WithCancel(ctx) + + go func() { + defer GinkgoRecover() + Expect(mgr.Start(mgrContext)).NotTo(HaveOccurred()) + }() + + DeferCleanup(func() { + By("stopping manager") + mgrCancel() + }) +}) + +func addSeedBackupBucketsCheckControllerToManager(mgr manager.Manager) error { + c, err := controller.New( + "seed-backupbuckets-check", + mgr, + controller.Options{ + Reconciler: seed.NewBackupBucketsCheckReconciler( + testClient, + config.SeedBackupBucketsCheckControllerConfiguration{ + SyncPeriod: &metav1.Duration{Duration: syncPeriod}, + ConditionThresholds: []config.ConditionThreshold{{ + Type: string(gardencorev1beta1.SeedBackupBucketsReady), + Duration: metav1.Duration{Duration: conditionThreshold}, + }}, + }, + fakeClock, + ), + }, + ) + if err != nil { + return err + } + + return c.Watch( + &source.Kind{Type: &gardencorev1beta1.BackupBucket{}}, + mapper.EnqueueRequestsFrom( + mapper.MapFunc(mapBackupBucketToSeed), + mapper.UpdateWithOldAndNew, + log, + ), + ) +} + +func mapBackupBucketToSeed(_ context.Context, _ logr.Logger, _ client.Reader, obj client.Object) []reconcile.Request { + backupBucket := obj.(*gardencorev1beta1.BackupBucket) + return []reconcile.Request{{ + NamespacedName: types.NamespacedName{ + Name: *backupBucket.Spec.SeedName, + }, + }} +} diff --git a/test/integration/controllermanager/seed/backupbucketscheck/backupbucketscheck_test.go b/test/integration/controllermanager/seed/backupbucketscheck/backupbucketscheck_test.go new file mode 100644 index 00000000000..4664260d5be --- /dev/null +++ b/test/integration/controllermanager/seed/backupbucketscheck/backupbucketscheck_test.go @@ -0,0 +1,187 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package backupbucketscheck_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + . "github.com/gardener/gardener/pkg/utils/test/matchers" +) + +const ( + conditionThreshold = 1 * time.Second + syncPeriod = 1 * time.Millisecond +) + +var _ = Describe("Seed BackupBucketsCheck controller tests", func() { + var ( + seed *gardencorev1beta1.Seed + bb1 *gardencorev1beta1.BackupBucket + bb2 *gardencorev1beta1.BackupBucket + ) + + BeforeEach(func() { + By("Create Seed") + seed = &gardencorev1beta1.Seed{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: testID + "-", + Labels: map[string]string{testID: testRunID}, + }, + Spec: gardencorev1beta1.SeedSpec{ + Provider: gardencorev1beta1.SeedProvider{ + Region: "region", + Type: "providerType", + }, + Settings: &gardencorev1beta1.SeedSettings{ + ShootDNS: &gardencorev1beta1.SeedSettingShootDNS{Enabled: true}, + Scheduling: &gardencorev1beta1.SeedSettingScheduling{Visible: true}, + }, + Networks: gardencorev1beta1.SeedNetworks{ + Pods: "10.0.0.0/16", + Services: "10.1.0.0/16", + Nodes: pointer.String("10.2.0.0/16"), + ShootDefaults: &gardencorev1beta1.ShootNetworks{ + Pods: pointer.String("100.128.0.0/11"), + Services: pointer.String("100.72.0.0/13"), + }, + }, + DNS: gardencorev1beta1.SeedDNS{ + IngressDomain: pointer.String("someingress.example.com"), + }, + }, + } + Expect(testClient.Create(ctx, seed)).To(Succeed()) + log.Info("Created seed for test", "seed", client.ObjectKeyFromObject(seed)) + + DeferCleanup(func() { + By("Delete Seed") + Expect(client.IgnoreNotFound(testClient.Delete(ctx, seed))).To(Succeed()) + }) + + By("Create BackupBuckets") + bb1 = &gardencorev1beta1.BackupBucket{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "foo-1-", + Labels: map[string]string{ + "provider.extensions.gardener.cloud/providerType": "true", + testID: testRunID, + }, + }, + Spec: gardencorev1beta1.BackupBucketSpec{ + SeedName: &seed.Name, + Provider: gardencorev1beta1.BackupBucketProvider{ + Type: "providerType", + Region: "region", + }, + SecretRef: corev1.SecretReference{ + Name: "secretName", + Namespace: "garden", + }, + }, + } + + bb2 = bb1.DeepCopy() + bb2.SetGenerateName("foo-2-") + + for _, backupBucket := range []*gardencorev1beta1.BackupBucket{bb1, bb2} { + Expect(testClient.Create(ctx, backupBucket)).To(Succeed()) + log.Info("Created backupbucket for test", "backupbucket", client.ObjectKeyFromObject(backupBucket)) + } + + DeferCleanup(func() { + By("Delete BackupBuckets") + for _, backupBucket := range []*gardencorev1beta1.BackupBucket{bb1, bb2} { + Expect(client.IgnoreNotFound(testClient.Delete(ctx, backupBucket))).To(Succeed()) + } + }) + + By("waiting until BackupBucketsReady condition is set to True") + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(seed), seed)).To(Succeed()) + g.Expect(seed.Status.Conditions).To(containCondition(ofType(gardencorev1beta1.SeedBackupBucketsReady), withStatus(gardencorev1beta1.ConditionTrue), withReason("BackupBucketsAvailable"))) + }).Should(Succeed()) + }) + + var tests = func(expectedConditionStatus gardencorev1beta1.ConditionStatus, reason string) { + It("should set BackupBucketsReady to Progressing and eventually to expected status when condition threshold expires", func() { + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(seed), seed)).To(Succeed()) + g.Expect(seed.Status.Conditions).To(containCondition(ofType(gardencorev1beta1.SeedBackupBucketsReady), withStatus(gardencorev1beta1.ConditionProgressing), withReason(reason))) + }).Should(Succeed()) + + fakeClock.Step(conditionThreshold + 1*time.Second) + Eventually(func(g Gomega) { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(seed), seed)).To(Succeed()) + g.Expect(seed.Status.Conditions).To(containCondition(ofType(gardencorev1beta1.SeedBackupBucketsReady), withStatus(expectedConditionStatus), withReason(reason))) + }).Should(Succeed()) + }) + } + + Context("when one BackupBucket becomes erroneous", func() { + BeforeEach(func() { + bb1.Status.LastError = &gardencorev1beta1.LastError{ + Description: "foo", + } + Expect(testClient.Status().Update(ctx, bb1)).To(Succeed()) + }) + + tests(gardencorev1beta1.ConditionFalse, "BackupBucketsError") + }) + + Context("when BackupBuckets for the Seed are gone", func() { + BeforeEach(func() { + for _, backupBucket := range []*gardencorev1beta1.BackupBucket{bb1, bb2} { + Expect(client.IgnoreNotFound(testClient.Delete(ctx, backupBucket))).To(Succeed()) + Eventually(func() error { + return testClient.Get(ctx, client.ObjectKeyFromObject(backupBucket), backupBucket) + }).Should(BeNotFoundError()) + } + }) + + tests(gardencorev1beta1.ConditionUnknown, "BackupBucketsGone") + }) +}) + +func containCondition(matchers ...gomegatypes.GomegaMatcher) gomegatypes.GomegaMatcher { + return ContainElement(And(matchers...)) +} + +func ofType(conditionType gardencorev1beta1.ConditionType) gomegatypes.GomegaMatcher { + return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Type": Equal(conditionType), + }) +} + +func withStatus(status gardencorev1beta1.ConditionStatus) gomegatypes.GomegaMatcher { + return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Status": Equal(status), + }) +} + +func withReason(reason string) gomegatypes.GomegaMatcher { + return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Reason": Equal(reason), + }) +}