cache: add timeout for groupcache's fetch operation

CHANGELOG.md

@@ -13,7 +13,9 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 ### Fixed
 
 ### Changed
+
 - [#5205](https://github.com/thanos-io/thanos/pull/5205) Rule: Add ruler labels as external labels in stateless ruler mode.
+- [#5206](https://github.com/thanos-io/thanos/pull/5206) Cache: add timeout for groupcache's fetch operation
 
 ### Removed
 

docs/components/store.md

@@ -429,6 +429,7 @@ config:
     - http://10.123.22.100:8080
   groupcache_group: test_group
   dns_interval: 1s
+  timeout: 500ms
 ```
 
 In this case, three Thanos Store nodes are running in the same group meaning that they all point to the same remote object storage.
@@ -441,6 +442,8 @@ In the `peers` section it is possible to use the prefix form to automatically lo
 
 Note that there must be no trailing slash in the `peers` configuration i.e. one of the strings must be identical to `self_url` and others should have the same form. Without this, loading data from peers may fail.
 
+If timeout is set to zero then there is no timeout for fetching and fetching's lifetime is equal to the lifetime to the original request's lifetime. It is recommended to keep it more than zero.
+
 ## Index Header
 
 In order to query series inside blocks from object storage, Store Gateway has to know certain initial info from each block index. In order to achieve so, on startup the Gateway builds an `index-header` for each block and stores it on local disk; such `index-header` is build by downloading specific pieces of original block's index, stored on local disk and then mmaped and used by Store Gateway.

pkg/cache/groupcache.go

@@ -36,6 +36,7 @@ type Groupcache struct {
 	galaxy   *galaxycache.Galaxy
 	universe *galaxycache.Universe
 	logger   log.Logger
+	timeout  time.Duration
 }
 
 // GroupcacheConfig holds the in-memory cache config.
@@ -59,13 +60,17 @@ type GroupcacheConfig struct {
 
 	// How often we should resolve the addresses.
 	DNSInterval time.Duration `yaml:"dns_interval"`
+
+	// Timeout specifies the read/write timeout.
+	Timeout time.Duration
 }
 
 var (
 	DefaultGroupcacheConfig = GroupcacheConfig{
 		MaxSize:       250 * 1024 * 1024,
 		DNSSDResolver: dns.GolangResolverType,
 		DNSInterval:   1 * time.Minute,
+		Timeout:       500 * time.Millisecond,
 	}
 )
 
@@ -255,6 +260,7 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf
 		logger:   logger,
 		galaxy:   galaxy,
 		universe: universe,
+		timeout:  conf.Timeout,
 	}, nil
 }
 
@@ -265,6 +271,12 @@ func (c *Groupcache) Store(ctx context.Context, data map[string][]byte, ttl time
 func (c *Groupcache) Fetch(ctx context.Context, keys []string) map[string][]byte {
 	data := map[string][]byte{}
 
+	if c.timeout != 0 {
+		timeoutCtx, cancel := context.WithTimeout(ctx, c.timeout)
+		ctx = timeoutCtx
+		defer cancel()
+	}
+
 	for _, k := range keys {
 		codec := galaxycache.ByteCodec{}