diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 9b662249b36..a6fd98fe4fd 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -149,8 +149,6 @@ type flagConfig struct { featureList []string // These options are extracted from featureList // for ease of use. - enablePromQLAtModifier bool - enablePromQLNegativeOffset bool enableExpandExternalLabels bool enableNewSDManager bool @@ -166,12 +164,6 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error { opts := strings.Split(f, ",") for _, o := range opts { switch o { - case "promql-at-modifier": - c.enablePromQLAtModifier = true - level.Info(logger).Log("msg", "promql-at-modifier enabled") - case "promql-negative-offset": - c.enablePromQLNegativeOffset = true - level.Info(logger).Log("msg", "promql-negative-offset enabled") case "remote-write-receiver": c.web.EnableRemoteWriteReceiver = true level.Warn(logger).Log("msg", "Remote write receiver enabled via feature flag remote-write-receiver. This is DEPRECATED. Use --web.enable-remote-write-receiver.") @@ -195,6 +187,8 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error { level.Info(logger).Log("msg", "Experimental agent mode enabled.") case "": continue + case "promql-at-modifier", "promql-negative-offset": + level.Warn(logger).Log("msg", "This option for --enable-feature is now permanently enabled and therefore a no-op.", "option", o) default: level.Warn(logger).Log("msg", "Unknown option for --enable-feature", "option", o) } @@ -570,8 +564,6 @@ func main() { ActiveQueryTracker: promql.NewActiveQueryTracker(localStoragePath, cfg.queryConcurrency, log.With(logger, "component", "activeQueryTracker")), LookbackDelta: time.Duration(cfg.lookbackDelta), NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get, - EnableAtModifier: cfg.enablePromQLAtModifier, - EnableNegativeOffset: cfg.enablePromQLNegativeOffset, } queryEngine = promql.NewEngine(opts) diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go index a8098bea1df..58443f63aa8 100644 --- a/cmd/promtool/main.go +++ b/cmd/promtool/main.go @@ -57,7 +57,6 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" "github.com/prometheus/prometheus/notifier" - "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/scrape" ) @@ -203,17 +202,14 @@ func main() { p = &promqlPrinter{} } - var queryOpts promql.LazyLoaderOpts for _, f := range *featureList { opts := strings.Split(f, ",") for _, o := range opts { switch o { - case "promql-at-modifier": - queryOpts.EnableAtModifier = true - case "promql-negative-offset": - queryOpts.EnableNegativeOffset = true case "": continue + case "promql-at-modifier", "promql-negative-offset": + fmt.Printf(" WARNING: Option for --enable-feature is a no-op after promotion to a stable feature: %q\n", o) default: fmt.Printf(" WARNING: Unknown option for --enable-feature: %q\n", o) } @@ -258,7 +254,7 @@ func main() { os.Exit(QueryLabels(*queryLabelsServer, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p)) case testRulesCmd.FullCommand(): - os.Exit(RulesUnitTest(queryOpts, *testRulesFiles...)) + os.Exit(RulesUnitTest(*testRulesFiles...)) case tsdbBenchWriteCmd.FullCommand(): os.Exit(checkErr(benchmarkWrite(*benchWriteOutPath, *benchSamplesFile, *benchWriteNumMetrics, *benchWriteNumScrapes))) diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index b8704231d0e..1b3a87fe221 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -39,11 +39,11 @@ import ( // RulesUnitTest does unit testing of rules based on the unit testing files provided. // More info about the file format can be found in the docs. -func RulesUnitTest(queryOpts promql.LazyLoaderOpts, files ...string) int { +func RulesUnitTest(files ...string) int { failed := false for _, f := range files { - if errs := ruleUnitTest(f, queryOpts); errs != nil { + if errs := ruleUnitTest(f); errs != nil { fmt.Fprintln(os.Stderr, " FAILED:") for _, e := range errs { fmt.Fprintln(os.Stderr, e.Error()) @@ -61,7 +61,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, files ...string) int { return successExitCode } -func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts) []error { +func ruleUnitTest(filename string) []error { fmt.Println("Unit Testing: ", filename) b, err := ioutil.ReadFile(filename) @@ -96,7 +96,7 @@ func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts) []error { // Testing. var errs []error for _, t := range unitTestInp.Tests { - ers := t.test(evalInterval, groupOrderMap, queryOpts, unitTestInp.RuleFiles...) + ers := t.test(evalInterval, groupOrderMap, unitTestInp.RuleFiles...) if ers != nil { errs = append(errs, ers...) } @@ -152,9 +152,9 @@ type testGroup struct { } // test performs the unit tests. -func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, ruleFiles ...string) []error { +func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, ruleFiles ...string) []error { // Setup testing suite. - suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString(), queryOpts) + suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString()) if err != nil { return []error{err} } diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go index 1e024405418..14a38ba8492 100644 --- a/cmd/promtool/unittest_test.go +++ b/cmd/promtool/unittest_test.go @@ -15,8 +15,6 @@ package main import ( "testing" - - "github.com/prometheus/prometheus/promql" ) func TestRulesUnitTest(t *testing.T) { @@ -24,10 +22,9 @@ func TestRulesUnitTest(t *testing.T) { files []string } tests := []struct { - name string - args args - queryOpts promql.LazyLoaderOpts - want int + name string + args args + want int }{ { name: "Passing Unit Tests", @@ -79,43 +76,23 @@ func TestRulesUnitTest(t *testing.T) { want: 1, }, { - name: "Disabled feature (@ modifier)", - args: args{ - files: []string{"./testdata/at-modifier-test.yml"}, - }, - want: 1, - }, - { - name: "Enabled feature (@ modifier)", + name: "@ modifier always works", args: args{ files: []string{"./testdata/at-modifier-test.yml"}, }, - queryOpts: promql.LazyLoaderOpts{ - EnableAtModifier: true, - }, want: 0, }, { - name: "Disabled feature (negative offset)", + name: "Negative offset always works", args: args{ files: []string{"./testdata/negative-offset-test.yml"}, }, - want: 1, - }, - { - name: "Enabled feature (negative offset)", - args: args{ - files: []string{"./testdata/negative-offset-test.yml"}, - }, - queryOpts: promql.LazyLoaderOpts{ - EnableNegativeOffset: true, - }, want: 0, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := RulesUnitTest(tt.queryOpts, tt.args.files...); got != tt.want { + if got := RulesUnitTest(tt.args.files...); got != tt.want { t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want) } }) diff --git a/docs/feature_flags.md b/docs/feature_flags.md index 33d8221350c..cb5b6fa2f83 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -11,18 +11,6 @@ Their behaviour can change in future releases which will be communicated via the You can enable them using the `--enable-feature` flag with a comma separated list of features. They may be enabled by default in future versions. -## `@` Modifier in PromQL - -`--enable-feature=promql-at-modifier` - -The `@` modifier lets you specify the evaluation time for instant vector selectors, -range vector selectors, and subqueries. More details can be found [here](querying/basics.md#modifier). - -This feature is considered stable, but since it breaks the invariant that -PromQL does not look ahead of the evaluation time, it still needs to be enabled -via this feature flag. The feature will be permanently enabled (and the feature -flag ignored) upon major release v3. - ## Expand environment variables in external labels `--enable-feature=expand-external-labels` @@ -31,25 +19,6 @@ Replace `${var}` or `$var` in the [`external_labels`](configuration/configuratio values according to the values of the current environment variables. References to undefined variables are replaced by the empty string. -## Negative offset in PromQL - -This negative offset is disabled by default since it breaks the invariant -that PromQL does not look ahead of the evaluation time for samples. - -`--enable-feature=promql-negative-offset` - -In contrast to the positive offset modifier, the negative offset modifier lets -one shift a vector selector into the future. An example in which one may want -to use a negative offset is reviewing past data and making temporal comparisons -with more recent data. - -More details can be found [here](querying/basics.md#offset-modifier). - -This feature is considered stable, but since it breaks the invariant that -PromQL does not look ahead of the evaluation time, it still needs to be enabled -via this feature flag. The feature will be permanently enabled (and the feature -flag ignored) upon major release v3. - ## Remote Write Receiver `--enable-feature=remote-write-receiver` diff --git a/docs/querying/basics.md b/docs/querying/basics.md index f4c176ed7d7..8a74f0df56a 100644 --- a/docs/querying/basics.md +++ b/docs/querying/basics.md @@ -209,9 +209,7 @@ can be specified: rate(http_requests_total[5m] offset -1w) -This feature is enabled by setting `--enable-feature=promql-negative-offset` -flag. See [feature flags](../feature_flags.md) for more details about -this flag. +Note that this allows a query to look ahead of its evaluation time. ### @ modifier @@ -249,10 +247,6 @@ These 2 queries will produce the same result. # offset before @ http_requests_total offset 5m @ 1609746000 -This modifier is disabled by default since it breaks the invariant that PromQL -does not look ahead of the evaluation time for samples. It can be enabled by setting -`--enable-feature=promql-at-modifier` flag. See [feature flags](../feature_flags.md) for more details about this flag. - Additionally, `start()` and `end()` can also be used as values for the `@` modifier as special values. For a range query, they resolve to the start and end of the range query respectively and remain the same for all steps. @@ -262,6 +256,8 @@ For an instant query, `start()` and `end()` both resolve to the evaluation time. http_requests_total @ start() rate(http_requests_total[5m] @ end()) +Note that the `@` modifier allows a query to look ahead of its evaluation time. + ## Subquery Subquery allows you to run an instant query for a given range and resolution. The result of a subquery is a range vector. diff --git a/go.mod b/go.mod index 3a8d250a888..cdc9eae836d 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 - golang.org/x/tools v0.1.8 + golang.org/x/tools v0.1.9-0.20211209172050-90a85b2969be google.golang.org/api v0.64.0 google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb google.golang.org/protobuf v1.27.1 diff --git a/go.sum b/go.sum index cae70a0c075..5d8a7b229aa 100644 --- a/go.sum +++ b/go.sum @@ -1785,8 +1785,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w= -golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.9-0.20211209172050-90a85b2969be h1:JRBiPXZpZ1FsceyPRRme0vX394zXC3xlhqu705k9nzM= +golang.org/x/tools v0.1.9-0.20211209172050-90a85b2969be/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/promql/engine.go b/promql/engine.go index 9b0a328797e..68214d6ac3e 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -221,12 +221,6 @@ type EngineOpts struct { // NoStepSubqueryIntervalFn is the default evaluation interval of // a subquery in milliseconds if no step in range vector was specified `[30m:]`. NoStepSubqueryIntervalFn func(rangeMillis int64) int64 - - // EnableAtModifier if true enables @ modifier. Disabled otherwise. - EnableAtModifier bool - - // EnableNegativeOffset if true enables negative (-) offset values. Disabled otherwise. - EnableNegativeOffset bool } // Engine handles the lifetime of queries from beginning to end. @@ -241,8 +235,6 @@ type Engine struct { queryLoggerLock sync.RWMutex lookbackDelta time.Duration noStepSubqueryIntervalFn func(rangeMillis int64) int64 - enableAtModifier bool - enableNegativeOffset bool } // NewEngine returns a new engine. @@ -323,8 +315,6 @@ func NewEngine(opts EngineOpts) *Engine { activeQueryTracker: opts.ActiveQueryTracker, lookbackDelta: opts.LookbackDelta, noStepSubqueryIntervalFn: opts.NoStepSubqueryIntervalFn, - enableAtModifier: opts.EnableAtModifier, - enableNegativeOffset: opts.EnableNegativeOffset, } } @@ -386,10 +376,6 @@ func (ng *Engine) NewRangeQuery(q storage.Queryable, qs string, start, end time. } func (ng *Engine) newQuery(q storage.Queryable, expr parser.Expr, start, end time.Time, interval time.Duration) (*query, error) { - if err := ng.validateOpts(expr); err != nil { - return nil, err - } - es := &parser.EvalStmt{ Expr: PreprocessExpr(expr, start, end), Start: start, @@ -405,62 +391,6 @@ func (ng *Engine) newQuery(q storage.Queryable, expr parser.Expr, start, end tim return qry, nil } -var ( - ErrValidationAtModifierDisabled = errors.New("@ modifier is disabled") - ErrValidationNegativeOffsetDisabled = errors.New("negative offset is disabled") -) - -func (ng *Engine) validateOpts(expr parser.Expr) error { - if ng.enableAtModifier && ng.enableNegativeOffset { - return nil - } - - var atModifierUsed, negativeOffsetUsed bool - - var validationErr error - parser.Inspect(expr, func(node parser.Node, path []parser.Node) error { - switch n := node.(type) { - case *parser.VectorSelector: - if n.Timestamp != nil || n.StartOrEnd == parser.START || n.StartOrEnd == parser.END { - atModifierUsed = true - } - if n.OriginalOffset < 0 { - negativeOffsetUsed = true - } - - case *parser.MatrixSelector: - vs := n.VectorSelector.(*parser.VectorSelector) - if vs.Timestamp != nil || vs.StartOrEnd == parser.START || vs.StartOrEnd == parser.END { - atModifierUsed = true - } - if vs.OriginalOffset < 0 { - negativeOffsetUsed = true - } - - case *parser.SubqueryExpr: - if n.Timestamp != nil || n.StartOrEnd == parser.START || n.StartOrEnd == parser.END { - atModifierUsed = true - } - if n.OriginalOffset < 0 { - negativeOffsetUsed = true - } - } - - if atModifierUsed && !ng.enableAtModifier { - validationErr = ErrValidationAtModifierDisabled - return validationErr - } - if negativeOffsetUsed && !ng.enableNegativeOffset { - validationErr = ErrValidationNegativeOffsetDisabled - return validationErr - } - - return nil - }) - - return validationErr -} - func (ng *Engine) newTestQuery(f func(context.Context) error) Query { qry := &query{ q: "test statement", diff --git a/promql/engine_test.go b/promql/engine_test.go index f7f6b216f43..da671d1c3ce 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -261,12 +261,11 @@ func (h *hintRecordingQuerier) Select(sortSeries bool, hints *storage.SelectHint func TestSelectHintsSetCorrectly(t *testing.T) { opts := EngineOpts{ - Logger: nil, - Reg: nil, - MaxSamples: 10, - Timeout: 10 * time.Second, - LookbackDelta: 5 * time.Second, - EnableAtModifier: true, + Logger: nil, + Reg: nil, + MaxSamples: 10, + Timeout: 10 * time.Second, + LookbackDelta: 5 * time.Second, } for _, tc := range []struct { @@ -2398,78 +2397,6 @@ func TestPreprocessAndWrapWithStepInvariantExpr(t *testing.T) { } } -func TestEngineOptsValidation(t *testing.T) { - cases := []struct { - opts EngineOpts - query string - fail bool - expError error - }{ - { - opts: EngineOpts{EnableAtModifier: false}, - query: "metric @ 100", fail: true, expError: ErrValidationAtModifierDisabled, - }, { - opts: EngineOpts{EnableAtModifier: false}, - query: "rate(metric[1m] @ 100)", fail: true, expError: ErrValidationAtModifierDisabled, - }, { - opts: EngineOpts{EnableAtModifier: false}, - query: "rate(metric[1h:1m] @ 100)", fail: true, expError: ErrValidationAtModifierDisabled, - }, { - opts: EngineOpts{EnableAtModifier: false}, - query: "metric @ start()", fail: true, expError: ErrValidationAtModifierDisabled, - }, { - opts: EngineOpts{EnableAtModifier: false}, - query: "rate(metric[1m] @ start())", fail: true, expError: ErrValidationAtModifierDisabled, - }, { - opts: EngineOpts{EnableAtModifier: false}, - query: "rate(metric[1h:1m] @ start())", fail: true, expError: ErrValidationAtModifierDisabled, - }, { - opts: EngineOpts{EnableAtModifier: false}, - query: "metric @ end()", fail: true, expError: ErrValidationAtModifierDisabled, - }, { - opts: EngineOpts{EnableAtModifier: false}, - query: "rate(metric[1m] @ end())", fail: true, expError: ErrValidationAtModifierDisabled, - }, { - opts: EngineOpts{EnableAtModifier: false}, - query: "rate(metric[1h:1m] @ end())", fail: true, expError: ErrValidationAtModifierDisabled, - }, { - opts: EngineOpts{EnableAtModifier: true}, - query: "metric @ 100", - }, { - opts: EngineOpts{EnableAtModifier: true}, - query: "rate(metric[1m] @ start())", - }, { - opts: EngineOpts{EnableAtModifier: true}, - query: "rate(metric[1h:1m] @ end())", - }, { - opts: EngineOpts{EnableNegativeOffset: false}, - query: "metric offset -1s", fail: true, expError: ErrValidationNegativeOffsetDisabled, - }, { - opts: EngineOpts{EnableNegativeOffset: true}, - query: "metric offset -1s", - }, { - opts: EngineOpts{EnableAtModifier: true, EnableNegativeOffset: true}, - query: "metric @ 100 offset -2m", - }, { - opts: EngineOpts{EnableAtModifier: true, EnableNegativeOffset: true}, - query: "metric offset -2m @ 100", - }, - } - - for _, c := range cases { - eng := NewEngine(c.opts) - _, err1 := eng.NewInstantQuery(nil, c.query, time.Unix(10, 0)) - _, err2 := eng.NewRangeQuery(nil, c.query, time.Unix(0, 0), time.Unix(10, 0), time.Second) - if c.fail { - require.Equal(t, c.expError, err1) - require.Equal(t, c.expError, err2) - } else { - require.Nil(t, err1) - require.Nil(t, err2) - } - } -} - func TestRangeQuery(t *testing.T) { cases := []struct { Name string diff --git a/promql/test.go b/promql/test.go index 19b2dd8300e..7cfd7ff4ab3 100644 --- a/promql/test.go +++ b/promql/test.go @@ -612,7 +612,6 @@ func (t *Test) clear() { MaxSamples: 10000, Timeout: 100 * time.Second, NoStepSubqueryIntervalFn: func(int64) int64 { return durationMilliseconds(1 * time.Minute) }, - EnableAtModifier: true, } t.queryEngine = NewEngine(opts) @@ -674,22 +673,11 @@ type LazyLoader struct { queryEngine *Engine context context.Context cancelCtx context.CancelFunc - - opts LazyLoaderOpts -} - -// LazyLoaderOpts are options for the lazy loader. -type LazyLoaderOpts struct { - // Disabled PromQL engine features. - EnableAtModifier, EnableNegativeOffset bool } // NewLazyLoader returns an initialized empty LazyLoader. -func NewLazyLoader(t testutil.T, input string, opts LazyLoaderOpts) (*LazyLoader, error) { - ll := &LazyLoader{ - T: t, - opts: opts, - } +func NewLazyLoader(t testutil.T, input string) (*LazyLoader, error) { + ll := &LazyLoader{T: t} err := ll.parse(input) ll.clear() return ll, err @@ -735,8 +723,6 @@ func (ll *LazyLoader) clear() { MaxSamples: 10000, Timeout: 100 * time.Second, NoStepSubqueryIntervalFn: func(int64) int64 { return durationMilliseconds(ll.SubqueryInterval) }, - EnableAtModifier: ll.opts.EnableAtModifier, - EnableNegativeOffset: ll.opts.EnableNegativeOffset, } ll.queryEngine = NewEngine(opts) diff --git a/promql/test_test.go b/promql/test_test.go index 845aef25688..b57ffccb7a0 100644 --- a/promql/test_test.go +++ b/promql/test_test.go @@ -109,7 +109,7 @@ func TestLazyLoader_WithSamplesTill(t *testing.T) { } for _, c := range cases { - suite, err := NewLazyLoader(t, c.loadString, LazyLoaderOpts{}) + suite, err := NewLazyLoader(t, c.loadString) require.NoError(t, err) defer suite.Close() diff --git a/web/api/v1/api.go b/web/api/v1/api.go index e6a6daffca8..79d357cdd72 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -378,11 +378,6 @@ func (api *API) query(r *http.Request) (result apiFuncResult) { } qry, err := api.QueryEngine.NewInstantQuery(api.Queryable, r.FormValue("query"), ts) - if err == promql.ErrValidationAtModifierDisabled { - err = errors.New("@ modifier is disabled, use --enable-feature=promql-at-modifier to enable it") - } else if err == promql.ErrValidationNegativeOffsetDisabled { - err = errors.New("negative offset is disabled, use --enable-feature=promql-negative-offset to enable it") - } if err != nil { return invalidParamError(err, "query") } @@ -458,11 +453,6 @@ func (api *API) queryRange(r *http.Request) (result apiFuncResult) { } qry, err := api.QueryEngine.NewRangeQuery(api.Queryable, r.FormValue("query"), start, end, step) - if err == promql.ErrValidationAtModifierDisabled { - err = errors.New("@ modifier is disabled, use --enable-feature=promql-at-modifier to enable it") - } else if err == promql.ErrValidationNegativeOffsetDisabled { - err = errors.New("negative offset is disabled, use --enable-feature=promql-negative-offset to enable it") - } if err != nil { return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil} }