info: Return store info only when the service is ready (#5255)

* return store info only when the service is ready Signed-off-by: Ben Ye <ben.ye@bytedance.com> * fix test Signed-off-by: Ben Ye <ben.ye@bytedance.com>
thanos-io · Mar 25, 2022 · f0e673a · f0e673a
1 parent a636c19
commit f0e673a
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 28 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -26,6 +26,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#5224](https://github.com/thanos-io/thanos/pull/5224) Receive: Remove sort on label hashing
 - [#5231](https://github.com/thanos-io/thanos/pull/5231) Tools: Bucket verify tool ignores blocks with deletion markers.
 - [#5244](https://github.com/thanos-io/thanos/pull/5244) Query: Promote negative offset and `@` modifier to stable features as per Prometheus [#10121](https://github.com/prometheus/prometheus/pull/10121).
+- [#5255](https://github.com/thanos-io/thanos/pull/5255) InfoAPI: Set store API unavailable when stores are not ready.
 
 ### Removed
 

diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go
@@ -661,11 +661,14 @@ func runQuery(
 			component.Query.String(),
 			info.WithLabelSetFunc(func() []labelpb.ZLabelSet { return proxy.LabelSet() }),
 			info.WithStoreInfoFunc(func() *infopb.StoreInfo {
-				minTime, maxTime := proxy.TimeRange()
-				return &infopb.StoreInfo{
-					MinTime: minTime,
-					MaxTime: maxTime,
+				if httpProbe.IsReady() {
+					mint, maxt := proxy.TimeRange()
+					return &infopb.StoreInfo{
+						MinTime: mint,
+						MaxTime: maxt,
+					}
 				}
+				return nil
 			}),
 			info.WithExemplarsInfoFunc(),
 			info.WithRulesInfoFunc(),

diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go
@@ -259,7 +259,8 @@ func runReceive(
 
 	level.Debug(logger).Log("msg", "setting up grpc server")
 	{
-		if err := setupAndRunGRPCServer(g, logger, reg, tracer, conf, reloadGRPCServer, comp, dbs, webHandler, grpcLogOpts, tagOpts, grpcProbe); err != nil {
+		if err := setupAndRunGRPCServer(g, logger, reg, tracer, conf, reloadGRPCServer, comp, dbs, webHandler, grpcLogOpts,
+			tagOpts, grpcProbe, httpProbe.IsReady); err != nil {
 			return err
 		}
 	}
@@ -294,7 +295,7 @@ func setupAndRunGRPCServer(g *run.Group,
 	grpcLogOpts []grpc_logging.Option,
 	tagOpts []tags.Option,
 	grpcProbe *prober.GRPCProbe,
-
+	isReady func() bool,
 ) error {
 
 	var s *grpcserver.Server
@@ -329,11 +330,14 @@ func setupAndRunGRPCServer(g *run.Group,
 				component.Receive.String(),
 				info.WithLabelSetFunc(func() []labelpb.ZLabelSet { return mts.LabelSet() }),
 				info.WithStoreInfoFunc(func() *infopb.StoreInfo {
-					minTime, maxTime := mts.TimeRange()
-					return &infopb.StoreInfo{
-						MinTime: minTime,
-						MaxTime: maxTime,
+					if isReady() {
+						minTime, maxTime := mts.TimeRange()
+						return &infopb.StoreInfo{
+							MinTime: minTime,
+							MaxTime: maxTime,
+						}
 					}
+					return nil
 				}),
 				info.WithExemplarsInfoFunc(),
 			)

diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go
@@ -604,11 +604,14 @@ func runRule(
 				return tsdbStore.LabelSet()
 			}),
 			info.WithStoreInfoFunc(func() *infopb.StoreInfo {
-				mint, maxt := tsdbStore.TimeRange()
-				return &infopb.StoreInfo{
-					MinTime: mint,
-					MaxTime: maxt,
+				if httpProbe.IsReady() {
+					mint, maxt := tsdbStore.TimeRange()
+					return &infopb.StoreInfo{
+						MinTime: mint,
+						MaxTime: maxt,
+					}
 				}
+				return nil
 			}),
 		)
 		options = append(options, grpcserver.WithServer(store.RegisterStoreServer(tsdbStore)))

diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go
@@ -265,11 +265,14 @@ func runSidecar(
 				return promStore.LabelSet()
 			}),
 			info.WithStoreInfoFunc(func() *infopb.StoreInfo {
-				mint, maxt := promStore.Timestamps()
-				return &infopb.StoreInfo{
-					MinTime: mint,
-					MaxTime: maxt,
+				if httpProbe.IsReady() {
+					mint, maxt := promStore.Timestamps()
+					return &infopb.StoreInfo{
+						MinTime: mint,
+						MaxTime: maxt,
+					}
 				}
+				return nil
 			}),
 			info.WithExemplarsInfoFunc(),
 			info.WithRulesInfoFunc(),

diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go
@@ -390,11 +390,14 @@ func runStore(
 			return bs.LabelSet()
 		}),
 		info.WithStoreInfoFunc(func() *infopb.StoreInfo {
-			mint, maxt := bs.TimeRange()
-			return &infopb.StoreInfo{
-				MinTime: mint,
-				MaxTime: maxt,
+			if httpProbe.IsReady() {
+				mint, maxt := bs.TimeRange()
+				return &infopb.StoreInfo{
+					MinTime: mint,
+					MaxTime: maxt,
+				}
 			}
+			return nil
 		}),
 	)
 

diff --git a/pkg/prober/http.go b/pkg/prober/http.go
@@ -32,7 +32,7 @@ func (p *HTTPProbe) HealthyHandler(logger log.Logger) http.HandlerFunc {
 
 // ReadyHandler returns a HTTP Handler which responds readiness checks.
 func (p *HTTPProbe) ReadyHandler(logger log.Logger) http.HandlerFunc {
-	return p.handler(logger, p.isReady)
+	return p.handler(logger, p.IsReady)
 }
 
 func (p *HTTPProbe) handler(logger log.Logger, c check) http.HandlerFunc {
@@ -47,8 +47,8 @@ func (p *HTTPProbe) handler(logger log.Logger, c check) http.HandlerFunc {
 	}
 }
 
-// isReady returns true if component is ready.
-func (p *HTTPProbe) isReady() bool {
+// IsReady returns true if component is ready.
+func (p *HTTPProbe) IsReady() bool {
 	ready := p.ready.Load()
 	return ready > 0
 }

diff --git a/pkg/prober/http_test.go b/pkg/prober/http_test.go
@@ -27,7 +27,7 @@ func TestHTTPProberHealthInitialState(t *testing.T) {
 func TestHTTPProberReadinessInitialState(t *testing.T) {
 	p := NewHTTP()
 
-	testutil.Assert(t, !p.isReady(), "initially should not be ready")
+	testutil.Assert(t, !p.IsReady(), "initially should not be ready")
 }
 
 func TestHTTPProberHealthyStatusSetting(t *testing.T) {
@@ -49,11 +49,11 @@ func TestHTTPProberReadyStatusSetting(t *testing.T) {
 
 	p.Ready()
 
-	testutil.Assert(t, p.isReady(), "should be ready")
+	testutil.Assert(t, p.IsReady(), "should be ready")
 
 	p.NotReady(testError)
 
-	testutil.Assert(t, !p.isReady(), "should not be ready")
+	testutil.Assert(t, !p.IsReady(), "should not be ready")
 }
 
 func TestHTTPProberMuxRegistering(t *testing.T) {