Merge branch 'main' into all-tenant-stats

thanos-io · Jul 7, 2022 · 6137fec · 6137fec
2 parents 99207ad + 5f31fc5
commit 6137fec
Show file tree

Hide file tree

Showing 37 changed files with 2,504 additions and 407 deletions.
diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk
@@ -101,11 +101,11 @@ $(MDOX): $(BINGO_DIR)/mdox.mod
 	@echo "(re)installing $(GOBIN)/mdox-v0.9.0"
 	@cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.9.0 "github.com/bwplotka/mdox"
 
-MINIO := $(GOBIN)/minio-v0.0.0-20201125204248-91130e884b5d
+MINIO := $(GOBIN)/minio-v0.0.0-20220704210254-b311abed3176
 $(MINIO): $(BINGO_DIR)/minio.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
-	@echo "(re)installing $(GOBIN)/minio-v0.0.0-20201125204248-91130e884b5d"
-	@cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=minio.mod -o=$(GOBIN)/minio-v0.0.0-20201125204248-91130e884b5d "github.com/minio/minio"
+	@echo "(re)installing $(GOBIN)/minio-v0.0.0-20220704210254-b311abed3176"
+	@cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=minio.mod -o=$(GOBIN)/minio-v0.0.0-20220704210254-b311abed3176 "github.com/minio/minio"
 
 PROMDOC := $(GOBIN)/promdoc-v0.8.0
 $(PROMDOC): $(BINGO_DIR)/promdoc.mod

diff --git a/.bingo/golangci-lint.sum b/.bingo/golangci-lint.sum
diff --git a/.bingo/minio.mod b/.bingo/minio.mod
@@ -2,4 +2,4 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
 
 go 1.14
 
-require github.com/minio/minio v0.0.0-20201125204248-91130e884b5d
+require github.com/minio/minio v0.0.0-20220704210254-b311abed3176
diff --git a/.bingo/minio.sum b/.bingo/minio.sum
diff --git a/.bingo/prometheus.1.mod b/.bingo/prometheus.1.mod
@@ -2,6 +2,10 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
 
 go 1.14
 
-replace k8s.io/klog => github.com/simonpasquier/klog-gokit v0.1.0
+replace (
+	github.com/json-iterator/go => github.com/json-iterator/go v1.1.12
+	github.com/modern-go/reflect2 => github.com/modern-go/reflect2 v1.0.2
+	k8s.io/klog => github.com/simonpasquier/klog-gokit v0.1.0
+)
 
 require github.com/prometheus/prometheus v1.8.2-0.20200724121523-657ba532e42f // cmd/prometheus
diff --git a/.bingo/prometheus.2.mod b/.bingo/prometheus.2.mod
@@ -2,8 +2,11 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
 
 go 1.15
 
-replace k8s.io/klog => github.com/simonpasquier/klog-gokit v0.3.0
-
-replace k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v2 v2.0.1
+replace (
+	github.com/json-iterator/go => github.com/json-iterator/go v1.1.12
+	github.com/modern-go/reflect2 => github.com/modern-go/reflect2 v1.0.2
+	k8s.io/klog => github.com/simonpasquier/klog-gokit v0.3.0
+	k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v2 v2.0.1
+)
 
 require github.com/prometheus/prometheus v1.8.2-0.20210331101223-3cafc58827d1 // cmd/prometheus
diff --git a/.bingo/prometheus.mod b/.bingo/prometheus.mod
@@ -10,7 +10,9 @@ replace (
 	github.com/cockroachdb/cockroach => github.com/cockroachdb/cockroach v0.0.0-20170608034007-84bc9597164f
 	github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.2.3-0.20180520015035-48a0ecefe2e4
 	github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.18.0
+	github.com/json-iterator/go => github.com/json-iterator/go v1.1.12
 	github.com/miekg/dns => github.com/miekg/dns v1.0.4
+	github.com/modern-go/reflect2 => github.com/modern-go/reflect2 v1.0.2
 	github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.0-pre1.0.20180607123607-faf4ec335fe0
 	github.com/prometheus/common => github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1
 	github.com/prometheus/tsdb => github.com/prometheus/tsdb v0.0.0-20180921053122-9c8ca47399a7

diff --git a/.bingo/variables.env b/.bingo/variables.env
@@ -36,7 +36,7 @@ JSONNETFMT="${GOBIN}/jsonnetfmt-v0.17.0"
 
 MDOX="${GOBIN}/mdox-v0.9.0"
 
-MINIO="${GOBIN}/minio-v0.0.0-20201125204248-91130e884b5d"
+MINIO="${GOBIN}/minio-v0.0.0-20220704210254-b311abed3176"
 
 PROMDOC="${GOBIN}/promdoc-v0.8.0"
 

diff --git a/.busybox-versions b/.busybox-versions
@@ -1,6 +1,6 @@
 # Auto generated by busybox-updater.sh. DO NOT EDIT
-amd64=1bc572099a3f2c20f3246ecf342fb7b12995dd2a38fcf4d04854d85551ab4320
-arm64=85cb18ba7a83a404d4c23a2993d7f3ca5a2dc24e05094b4ad9b796fc2d54b409
-arm=58f18f7f81cd013b2288e08d3b735b6b41ca31d6b2bfe2fbe77a09d188d5cc81
-ppc64le=012e0a29d057e095e5e15af58d5d52525554729977056827694886e20d5691d3
-s390x=f97502190dbe8ff512c9baf8928c352dcfa1e439a264f17f8de9f0032d719c63
+amd64=6594015d3ec4a8fa22c96d419491c2064462e409dd59b68101297de59f83f55d
+arm64=28be5d98d670de64bf80c22bfdd6a55fb417a7c9e8055e699ba61697dc01ce4f
+arm=9c3b43398c1f190b7001fd930a3ec802f1bb1ec33cc9378751ddd3078a547891
+ppc64le=e7072613e28f62090be3b2f7022c58b352014d1b4ff8568c46a868f9a2afd480
+s390x=7339dbcc59ca36f40e6ac17b8ce4188689157bc08401ac303932b0588fc2da06
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,15 +2,15 @@
 version: 2.1
 
 orbs:
-  go: circleci/go@1.5.0
+  go: circleci/go@1.7.1
 
 executors:
   golang:
     docker:
-      - image: cimg/go:1.17-node
+      - image: cimg/go:1.18-node
   golang-test:
     docker:
-      - image: cimg/go:1.17-node
+      - image: cimg/go:1.18-node
       - image: quay.io/thanos/docker-swift-onlyone-authv2-keystone:v0.1
 
 jobs:
@@ -22,7 +22,7 @@ jobs:
       - checkout
       - go/mod-download-cached
       - setup_remote_docker:
-          version: 20.10.11
+          version: 20.10.12
       - run:
           name: Create Secret if PR is not forked
           # GCS integration tests are run only for author's PR that have write access, because these tests
@@ -74,7 +74,7 @@ jobs:
       - checkout
       - go/mod-download-cached
       - setup_remote_docker:
-          version: 20.10.11
+          version: 20.10.12
       - attach_workspace:
           at: .
       # Register qemu to support multi-arch.
@@ -96,7 +96,7 @@ jobs:
       - checkout
       - go/mod-download-cached
       - setup_remote_docker:
-          version: 20.10.11
+          version: 20.10.12
       - attach_workspace:
           at: .
       - run: make tarballs-release

diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml
@@ -16,9 +16,9 @@ jobs:
       GOBIN: /tmp/.bin
     steps:
       - name: Install Go.
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.17.x
+          go-version: 1.18.x
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v1
@@ -59,16 +59,16 @@ jobs:
       fail-fast: true
       matrix:
         parallelism: [4]
-        index: [0,1,2,3]  
+        index: [0,1,2,3]
     runs-on: ubuntu-latest
     name: Thanos end-to-end tests
     env:
       GOBIN: /tmp/.bin
     steps:
       - name: Install Go.
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.17.x
+          go-version: 1.18.x
 
       - name: Check out code into the Go module directory.
         uses: actions/checkout@v2

diff --git a/.go-version b/.go-version
@@ -1 +1 @@
-1.17
+1.18
diff --git a/.promu.yml b/.promu.yml
@@ -1,5 +1,5 @@
 go:
-  version: 1.17
+  version: 1.18
 repository:
   path: github.com/thanos-io/thanos
 build:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,19 +9,24 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 We use *breaking :warning:* to mark changes that are not backward compatible (relates only to v0.y.z releases.)
 
 ## Unreleased
+- [#5453](https://github.com/thanos-io/thanos/pull/5453) Compact: Skip erroneous empty non `*AggrChunk` chunks during 1h downsampling of 5m resolution blocks.
 
 ### Fixed
 
 ### Added
 
 - [#5440](https://github.com/thanos-io/thanos/pull/5440) HTTP metrics: export number of in-flight HTTP requests.
-- [#5402](https://github.com/thanos-io/thanos/pull/5402) Receive: Implement exposing TSDB stats for all tenants 
+- [#5424](https://github.com/thanos-io/thanos/pull/5424) Receive: Export metrics regarding size of remote write requests.
+- [#5420](https://github.com/thanos-io/thanos/pull/5420) Receive: Automatically remove stale tenants.
+- [#5470](https://github.com/thanos-io/thanos/pull/5470) Receive: Implement exposing TSDB stats for all tenants 
 
 ### Changed
 
+- [#5447](https://github.com/thanos-io/thanos/pull/5447) Promclient: Ignore 405 status codes for Prometheus buildVersion requests
+
 ### Removed
 
-## [v0.27.0-rc0](https://github.com/thanos-io/thanos/tree/release-0.27) - Release in progress
+## [v0.27.0](https://github.com/thanos-io/thanos/tree/release-0.27) - 2022.07.05
 
 ### Fixed
 - [#5339](https://github.com/thanos-io/thanos/pull/5339) Receive: Fix deadlock on interrupt in routerOnly mode.
@@ -36,13 +41,11 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#5391](https://github.com/thanos-io/thanos/pull/5391) Receive: Add relabeling support.
 - [#5408](https://github.com/thanos-io/thanos/pull/5408) Receive: Add support for consistent hashrings.
 - [#5391](https://github.com/thanos-io/thanos/pull/5391) Receive: Implement api/v1/status/tsdb.
-- [#5424](https://github.com/thanos-io/thanos/pull/5424) Receive: export metrics regarding size of remote write requests.
-- [#5420](https://github.com/thanos-io/thanos/pull/5420) Receive: Automatically remove stale tenants.
 
 ### Changed
 
 - [#5410](https://github.com/thanos-io/thanos/pull/5410) Query: Close() after using query. This should reduce bumps in memory allocations.
-- [#5417](https://github.com/thanos-io/thanos/pull/5417) Ruler: Change default evaluation interval to 1 minute.
+- [#5417](https://github.com/thanos-io/thanos/pull/5417) Ruler: *Breaking if you have not set this value (`--eval-interval`) yourself and rely on that value. :warning:*. Change the default evaluation interval from 30s to 1 minute in order to be compliant with Prometheus alerting compliance specification: https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#executing-an-alerting-rule.
 
 ### Removed
 

diff --git a/Dockerfile.multi-stage b/Dockerfile.multi-stage
@@ -1,6 +1,6 @@
 # By default we pin to amd64 sha. Use make docker to automatically adjust for arm64 versions.
 ARG BASE_DOCKER_SHA="14d68ca3d69fceaa6224250c83d81d935c053fb13594c811038c461194599973"
-FROM golang:1.17-alpine3.15 as builder
+FROM golang:1.18-alpine3.15 as builder
 
 WORKDIR $GOPATH/src/github.com/thanos-io/thanos
 # Change in the docker context invalidates the cache so to leverage docker

diff --git a/cmd/thanos/query_frontend.go b/cmd/thanos/query_frontend.go
@@ -4,6 +4,7 @@
 package main
 
 import (
+	"net"
 	"net/http"
 	"time"
 
@@ -25,7 +26,6 @@ import (
 
 	"github.com/thanos-io/thanos/pkg/api"
 	"github.com/thanos-io/thanos/pkg/component"
-	"github.com/thanos-io/thanos/pkg/exthttp"
 	"github.com/thanos-io/thanos/pkg/extkingpin"
 	"github.com/thanos-io/thanos/pkg/extprom"
 	extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http"
@@ -150,7 +150,19 @@ func registerQueryFrontend(app *extkingpin.App) {
 }
 
 func parseTransportConfiguration(downstreamTripperConfContentYaml []byte) (*http.Transport, error) {
-	downstreamTripper := exthttp.NewTransport()
+	downstreamTripper := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
 
 	if len(downstreamTripperConfContentYaml) > 0 {
 		tripperConfig := &queryfrontend.DownstreamTripperConfig{}

diff --git a/docs/components/compact.md b/docs/components/compact.md
@@ -253,7 +253,7 @@ Generally there two scalability directions:
 
 You should horizontally scale Compactor to cope with this using [label sharding](../sharding.md#compactor). This allows to assign multiple streams to each instance of compactor.
 
-1. TSDB blocks from single stream is too big, it takes too much time or resources.
+2. TSDB blocks from single stream is too big, it takes too much time or resources.
 
 This is rare as first you would need to ingest that amount of data into Prometheus and it's usually not recommended to have bigger than 10 millions series in the 2 hours blocks. However, with 2 weeks blocks, potential [Vertical Compaction](#vertical-compactions) enabled and other producers than Prometheus (e.g backfilling) this scalability concern can appear as well. See [Limit size of blocks](https://github.com/thanos-io/thanos/issues/3068) ticket to track progress of solution if you are hitting this.
 

diff --git a/docs/release-process.md b/docs/release-process.md
@@ -21,32 +21,33 @@ Release shepherd responsibilities:
 * Perform releases (from first RC to actual release).
 * Announce all releases on all communication channels.
 
-| Release | Time of first RC     | Shepherd (GitHub handle)    |
-|---------|----------------------|-----------------------------|
-| v0.28.0 | (planned) 2022.07.22 | No one ATM                  |
-| v0.27.0 | 2022.06.10           | `@wiardvanrij`              |
-| v0.26.0 | 2022.04.29           | `@wiardvanrij`              |
-| v0.25.0 | 2022.03.01           | `@bwplotka` and `@matej-g`  |
-| v0.24.0 | 2021.12.22           | `@squat`                    |
-| v0.23.0 | 2021.09.02           | `@bwplotka`                 |
-| v0.22.0 | 2021.07.06           | `@GiedriusS`                |
-| v0.21.0 | 2021.05.28           | `@metalmatze` and `@onprem` |
-| v0.20.0 | 2021.04.23           | `@kakkoyun`                 |
-| v0.19.0 | 2021.03.02           | `@bwplotka`                 |
-| v0.18.0 | 2021.01.06           | `@squat`                    |
-| v0.17.0 | 2020.11.18           | `@metalmatze`               |
-| v0.16.0 | 2020.10.26           | `@bwplotka`                 |
-| v0.15.0 | 2020.08.12           | `@kakkoyun`                 |
-| v0.14.0 | 2020.07.10           | `@kakkoyun`                 |
-| v0.13.0 | 2020.05.13           | `@bwplotka`                 |
-| v0.12.0 | 2020.04.15           | `@squat`                    |
-| v0.11.0 | 2020.02.19           | `@metalmatze`               |
-| v0.10.0 | 2020.01.08           | `@GiedriusS`                |
-| v0.9.0  | 2019.11.26           | `@bwplotka`                 |
-| v0.8.0  | 2019.10.09           | `@bwplotka`                 |
-| v0.7.0  | 2019.08.28           | `@domgreen`                 |
-| v0.6.0  | 2019.07.12           | `@GiedriusS`                |
-| v0.5.0  | 2019.06.31           | `@bwplotka`                 |
+| Release | Time of first RC     | Shepherd (GitHub handle)      |
+|---------|----------------------|-------------------------------|
+| v0.29.0 | (planned) 2022.09.29 | No one ATM                    |
+| v0.28.0 | (planned) 2022.08.18 | No one ATM                    |
+| v0.27.0 | 2022.06.21           | `@wiardvanrij` and `@matej-g` |
+| v0.26.0 | 2022.04.29           | `@wiardvanrij`                |
+| v0.25.0 | 2022.03.01           | `@bwplotka` and `@matej-g`    |
+| v0.24.0 | 2021.12.22           | `@squat`                      |
+| v0.23.0 | 2021.09.02           | `@bwplotka`                   |
+| v0.22.0 | 2021.07.06           | `@GiedriusS`                  |
+| v0.21.0 | 2021.05.28           | `@metalmatze` and `@onprem`   |
+| v0.20.0 | 2021.04.23           | `@kakkoyun`                   |
+| v0.19.0 | 2021.03.02           | `@bwplotka`                   |
+| v0.18.0 | 2021.01.06           | `@squat`                      |
+| v0.17.0 | 2020.11.18           | `@metalmatze`                 |
+| v0.16.0 | 2020.10.26           | `@bwplotka`                   |
+| v0.15.0 | 2020.08.12           | `@kakkoyun`                   |
+| v0.14.0 | 2020.07.10           | `@kakkoyun`                   |
+| v0.13.0 | 2020.05.13           | `@bwplotka`                   |
+| v0.12.0 | 2020.04.15           | `@squat`                      |
+| v0.11.0 | 2020.02.19           | `@metalmatze`                 |
+| v0.10.0 | 2020.01.08           | `@GiedriusS`                  |
+| v0.9.0  | 2019.11.26           | `@bwplotka`                   |
+| v0.8.0  | 2019.10.09           | `@bwplotka`                   |
+| v0.7.0  | 2019.08.28           | `@domgreen`                   |
+| v0.6.0  | 2019.07.12           | `@GiedriusS`                  |
+| v0.5.0  | 2019.06.31           | `@bwplotka`                   |
 
 # For maintainers: Cutting individual release
 

diff --git a/docs/storage.md b/docs/storage.md
@@ -86,6 +86,7 @@ config:
       key_file: ""
       server_name: ""
       insecure_skip_verify: false
+    disable_compression: false
   trace:
     enable: false
   list_objects_version: ""
@@ -356,13 +357,13 @@ config:
     max_idle_conns: 0
     max_idle_conns_per_host: 0
     max_conns_per_host: 0
-    disable_compression: false
     tls_config:
       ca_file: ""
       cert_file: ""
       key_file: ""
       server_name: ""
       insecure_skip_verify: false
+    disable_compression: false
 prefix: ""
 ```
 
@@ -425,11 +426,19 @@ config:
   http_config:
     idle_conn_timeout: 1m30s
     response_header_timeout: 2m
+    insecure_skip_verify: false
     tls_handshake_timeout: 10s
     expect_continue_timeout: 1s
     max_idle_conns: 100
     max_idle_conns_per_host: 100
     max_conns_per_host: 0
+    tls_config:
+      ca_file: ""
+      cert_file: ""
+      key_file: ""
+      server_name: ""
+      insecure_skip_verify: false
+    disable_compression: false
 prefix: ""
 ```