From b6a5752ebc63ba48ece2a3f10d26115a01914e66 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 4 Nov 2021 22:53:49 +0100 Subject: [PATCH 1/2] Start web hooks first --- pkg/manager/internal.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go index 878142b0ba..aacbc4f09a 100644 --- a/pkg/manager/internal.go +++ b/pkg/manager/internal.go @@ -476,6 +476,11 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { go cm.serveHealthProbes() } + // Webhooks MUST start before any cache is populated, otherwise there is a race condition + // between conversion webhooks and the cache sync (usually initial list) which causes the webhooks + // to never start because no cache can be populated. + cm.startWebhookRunnables() + go cm.startNonLeaderElectionRunnables() go func() { @@ -573,13 +578,10 @@ func (cm *controllerManager) waitForRunnableToEnd(shutdownCancel context.CancelF return nil } -func (cm *controllerManager) startNonLeaderElectionRunnables() { +func (cm *controllerManager) startWebhookRunnables() { cm.mu.Lock() defer cm.mu.Unlock() - // First start any webhook servers, which includes conversion, validation, and defaulting - // webhooks that are registered. - // // WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition // between conversion webhooks and the cache sync (usually initial list) which causes the webhooks // to never start because no cache can be populated. @@ -588,6 +590,11 @@ func (cm *controllerManager) startNonLeaderElectionRunnables() { cm.startRunnable(c) } } +} + +func (cm *controllerManager) startNonLeaderElectionRunnables() { + cm.mu.Lock() + defer cm.mu.Unlock() // Start and wait for caches. cm.waitForCache(cm.internalCtx) From 708d5397c4f300c897aaa0fbacab4804964e1f25 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Fri, 5 Nov 2021 10:09:59 +0100 Subject: [PATCH 2/2] =?UTF-8?q?start=20healhprobes=20sync=20Co-authored-by?= =?UTF-8?q?:=20Stefan=20B=C3=BCringer=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/manager/internal.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go index aacbc4f09a..59794d9620 100644 --- a/pkg/manager/internal.go +++ b/pkg/manager/internal.go @@ -425,11 +425,13 @@ func (cm *controllerManager) serveHealthProbes() { cm.healthzStarted = true }() - // Shutdown the server when stop is closed - <-cm.internalProceduresStop - if err := server.Shutdown(cm.shutdownCtx); err != nil { - cm.errChan <- err - } + go func() { + // Shutdown the server when stop is closed + <-cm.internalProceduresStop + if err := server.Shutdown(cm.shutdownCtx); err != nil { + cm.errChan <- err + } + }() } func (cm *controllerManager) Start(ctx context.Context) (err error) { @@ -473,7 +475,7 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { // Serve health probes if cm.healthProbeListener != nil { - go cm.serveHealthProbes() + cm.serveHealthProbes() } // Webhooks MUST start before any cache is populated, otherwise there is a race condition