From 798f2ae763c4a0c863b431fea55ba37fd6c5bf05 Mon Sep 17 00:00:00 2001 From: "Scott G. Miller" Date: Tue, 11 May 2021 16:33:25 -0500 Subject: [PATCH 1/3] Add an exponential backoff to TCP listeners to avoid fast loops in error scenarios --- vault/cluster/cluster.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/vault/cluster/cluster.go b/vault/cluster/cluster.go index b4ec28cf6f4cb..182c2f20cb8ac 100644 --- a/vault/cluster/cluster.go +++ b/vault/cluster/cluster.go @@ -279,6 +279,15 @@ func (cl *Listener) Run(ctx context.Context) error { close(closeCh) }() + // baseDelay is the initial delay after an Accept() error before attempting again + const baseDelay = 5 * time.Millisecond + + // maxDelay is the maximum delay after an Accept() error before attempting again. + // In the case that this function is error-looping, it will delay the shutdown check. + // Therefore, changes to maxDelay may have an effect on the latency of shutdown. + const maxDelay = 1 * time.Second + + var loopDelay time.Duration for { if atomic.LoadUint32(cl.shutdown) > 0 { return @@ -298,6 +307,18 @@ func (cl *Listener) Run(ctx context.Context) error { if conn != nil { conn.Close() } + + if loopDelay == 0 { + loopDelay = baseDelay + } else { + loopDelay *= 2 + } + + if loopDelay > maxDelay { + loopDelay = maxDelay + } + + time.Sleep(loopDelay) continue } if conn == nil { From 52c5976f5b5413dba2eaefd433c9857b80d07aff Mon Sep 17 00:00:00 2001 From: "Scott G. Miller" Date: Tue, 11 May 2021 16:35:08 -0500 Subject: [PATCH 2/3] reset loop delay --- vault/cluster/cluster.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vault/cluster/cluster.go b/vault/cluster/cluster.go index 182c2f20cb8ac..016e73c32abd5 100644 --- a/vault/cluster/cluster.go +++ b/vault/cluster/cluster.go @@ -321,6 +321,9 @@ func (cl *Listener) Run(ctx context.Context) error { time.Sleep(loopDelay) continue } + // No error, reset loop delay + loopDelay = 0 + if conn == nil { continue } From adde4528777f0aecd2a960aec7288a1c70008ae2 Mon Sep 17 00:00:00 2001 From: "Scott G. Miller" Date: Tue, 11 May 2021 16:36:27 -0500 Subject: [PATCH 3/3] changelog --- changelog/11588.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/11588.txt diff --git a/changelog/11588.txt b/changelog/11588.txt new file mode 100644 index 0000000000000..1f7f0365abfa5 --- /dev/null +++ b/changelog/11588.txt @@ -0,0 +1,3 @@ +```release-note:improvement +core: Add a small (<1s) exponential backoff to failed TCP listener Accept failures. +``` \ No newline at end of file