Cap maximum grpc wait time when heartbeating to heartbeatTimeout/2

replication.go

@@ -406,8 +406,10 @@ func (r *Raft) heartbeat(s *followerReplication, stopCh chan struct{}) {
 			r.observe(FailedHeartbeatObservation{PeerID: peer.ID, LastContact: s.LastContact()})
 			failures++
 			select {
-			case <-time.After(backoff(failureWait, failures, maxFailureScale)):
+			case <-time.After(cappedExponentialBackoff(failureWait, failures, maxFailureScale,
+				r.config().HeartbeatTimeout)):
 			case <-stopCh:
+				return
 			}
 		} else {
 			if failures > 0 {

util.go

@@ -144,6 +144,20 @@ func backoff(base time.Duration, round, limit uint64) time.Duration {
 	return base
 }
 
+// cappedExponentialBackoff computes the exponential backoff with an adjustable
+// cap on the max timeout.
+func cappedExponentialBackoff(base time.Duration, round, limit uint64, cap time.Duration) time.Duration {
+	power := min(round, limit)
+	for power > 2 {
+		if base > cap {
+			return base
+		}
+		base *= 2
+		power--
+	}
+	return base
+}
+
 // Needed for sorting []uint64, used to determine commitment
 type uint64Slice []uint64