From ff0d256e6dc21645112cf4fa1f6153bc3094f044 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sat, 12 Mar 2022 18:04:18 -0800 Subject: [PATCH 1/3] ignore blocks with deletion markers Signed-off-by: Ben Ye --- cmd/thanos/tools_bucket.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 6964b4c71b..8693546106 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -333,7 +333,9 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path return err } - fetcher, err := block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), nil) + // We ignore any block that has the deletion marker file. + filters := []block.MetadataFilter{block.NewIgnoreDeletionMarkFilter(logger, bkt, 0, 32)} + fetcher, err := block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), filters) if err != nil { return err } From fe6e5c1b380dc4a39f23dd74e66f512033bf5190 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 13 Mar 2022 01:34:34 -0800 Subject: [PATCH 2/3] update changelog Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b21adc101..44721110d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#5206](https://github.com/thanos-io/thanos/pull/5206) Cache: add timeout for groupcache's fetch operation. - [#5218](https://github.com/thanos-io/thanos/pull/5218) Tools: Run bucket downsample tools continuously. - [#5224](https://github.com/thanos-io/thanos/pull/5224) Receive: Remove sort on label hashing +- [#5231](https://github.com/thanos-io/thanos/pull/5231) Tools: Bucket verify tool ignores blocks with deletion markers. ### Removed From 01c42dbb47d4b6450beb7adad5cc7400f0e81fd7 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 14 Mar 2022 10:28:57 -0700 Subject: [PATCH 3/3] use fetcher concurrency Signed-off-by: Ben Ye --- cmd/thanos/tools_bucket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 8693546106..959d2a146c 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -334,7 +334,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path } // We ignore any block that has the deletion marker file. - filters := []block.MetadataFilter{block.NewIgnoreDeletionMarkFilter(logger, bkt, 0, 32)} + filters := []block.MetadataFilter{block.NewIgnoreDeletionMarkFilter(logger, bkt, 0, block.FetcherConcurrency)} fetcher, err := block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), filters) if err != nil { return err