Make Query requests idempotent (#1022)

* Make Query requests idempotent

Address #1020.

Signed-off-by: Tomáš Dohnálek <dohnto@gmail.com>

* Use empty header

Signed-off-by: Tomáš Dohnálek <dohnto@gmail.com>

* Document issue with original documentation

Signed-off-by: Tomáš Dohnálek <dohnto@gmail.com>

api/prometheus/v1/api.go

@@ -1139,6 +1139,14 @@ func (h *apiClientImpl) DoGetFallback(ctx context.Context, u *url.URL, args url.
 		return nil, nil, nil, err
 	}
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	// Following comment originates from https://pkg.go.dev/net/http#Transport
+	// Transport only retries a request upon encountering a network error if the request is
+	// idempotent and either has no body or has its Request.GetBody defined. HTTP requests
+	// are considered idempotent if they have HTTP methods GET, HEAD, OPTIONS, or TRACE; or
+	// if their Header map contains an "Idempotency-Key" or "X-Idempotency-Key" entry. If the
+	// idempotency key value is a zero-length slice, the request is treated as idempotent but
+	// the header is not sent on the wire.
+	req.Header["Idempotency-Key"] = nil
 
 	resp, body, warnings, err := h.Do(ctx, req)
 	if resp != nil && (resp.StatusCode == http.StatusMethodNotAllowed || resp.StatusCode == http.StatusNotImplemented) {