Merge pull request #634 from grafana/krajo/merge-upstream

merge upstream at fdaafdb

docs/configuration/configuration.md

@@ -3673,7 +3673,8 @@ queue_config:
   [ min_shards: <int> | default = 1 ]
   # Maximum number of samples per send.
   [ max_samples_per_send: <int> | default = 2000]
-  # Maximum time a sample will wait in buffer.
+  # Maximum time a sample will wait for a send. The sample might wait less
+  # if the buffer is full. Further time might pass due to potential retries.
   [ batch_send_deadline: <duration> | default = 5s ]
   # Initial retry delay. Gets doubled for every retry.
   [ min_backoff: <duration> | default = 30ms ]

model/labels/matcher.go

@@ -14,7 +14,8 @@
 package labels
 
 import (
-	"fmt"
+	"bytes"
+	"strconv"
 )
 
 // MatchType is an enum for label matching types.
@@ -78,7 +79,29 @@ func MustNewMatcher(mt MatchType, name, val string) *Matcher {
 }
 
 func (m *Matcher) String() string {
-	return fmt.Sprintf("%s%s%q", m.Name, m.Type, m.Value)
+	// Start a buffer with a pre-allocated size on stack to cover most needs.
+	var bytea [1024]byte
+	b := bytes.NewBuffer(bytea[:0])
+
+	if m.shouldQuoteName() {
+		b.Write(strconv.AppendQuote(b.AvailableBuffer(), m.Name))
+	} else {
+		b.WriteString(m.Name)
+	}
+	b.WriteString(m.Type.String())
+	b.Write(strconv.AppendQuote(b.AvailableBuffer(), m.Value))
+
+	return b.String()
+}
+
+func (m *Matcher) shouldQuoteName() bool {
+	for i, c := range m.Name {
+		if c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (i > 0 && c >= '0' && c <= '9') {
+			continue
+		}
+		return true
+	}
+	return false
 }
 
 // Matches returns whether the matcher matches the given string value.

model/labels/matcher_test.go

@@ -15,6 +15,7 @@ package labels
 
 import (
 	"fmt"
+	"math/rand"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -225,3 +226,128 @@ func BenchmarkNewMatcher(b *testing.B) {
 		}
 	})
 }
+
+func BenchmarkMatcher_String(b *testing.B) {
+	type benchCase struct {
+		name     string
+		matchers []*Matcher
+	}
+	cases := []benchCase{
+		{
+			name: "short name equal",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchEqual, "foo", "bar"),
+				MustNewMatcher(MatchEqual, "bar", "baz"),
+				MustNewMatcher(MatchEqual, "abc", "def"),
+				MustNewMatcher(MatchEqual, "ghi", "klm"),
+				MustNewMatcher(MatchEqual, "nop", "qrs"),
+			},
+		},
+		{
+			name: "short quoted name not equal",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchEqual, "f.o", "bar"),
+				MustNewMatcher(MatchEqual, "b.r", "baz"),
+				MustNewMatcher(MatchEqual, "a.c", "def"),
+				MustNewMatcher(MatchEqual, "g.i", "klm"),
+				MustNewMatcher(MatchEqual, "n.p", "qrs"),
+			},
+		},
+		{
+			name: "short quoted name with quotes not equal",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchEqual, `"foo"`, "bar"),
+				MustNewMatcher(MatchEqual, `"foo"`, "baz"),
+				MustNewMatcher(MatchEqual, `"foo"`, "def"),
+				MustNewMatcher(MatchEqual, `"foo"`, "klm"),
+				MustNewMatcher(MatchEqual, `"foo"`, "qrs"),
+			},
+		},
+		{
+			name: "short name value with quotes equal",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchEqual, "foo", `"bar"`),
+				MustNewMatcher(MatchEqual, "bar", `"baz"`),
+				MustNewMatcher(MatchEqual, "abc", `"def"`),
+				MustNewMatcher(MatchEqual, "ghi", `"klm"`),
+				MustNewMatcher(MatchEqual, "nop", `"qrs"`),
+			},
+		},
+		{
+			name: "short name and long value regexp",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchRegexp, "foo", "five_six_seven_eight_nine_ten_one_two_three_four"),
+				MustNewMatcher(MatchRegexp, "bar", "one_two_three_four_five_six_seven_eight_nine_ten"),
+				MustNewMatcher(MatchRegexp, "abc", "two_three_four_five_six_seven_eight_nine_ten_one"),
+				MustNewMatcher(MatchRegexp, "ghi", "three_four_five_six_seven_eight_nine_ten_one_two"),
+				MustNewMatcher(MatchRegexp, "nop", "four_five_six_seven_eight_nine_ten_one_two_three"),
+			},
+		},
+		{
+			name: "short name and long value with quotes equal",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchEqual, "foo", `five_six_seven_eight_nine_ten_"one"_two_three_four`),
+				MustNewMatcher(MatchEqual, "bar", `one_two_three_four_five_six_"seven"_eight_nine_ten`),
+				MustNewMatcher(MatchEqual, "abc", `two_three_four_five_six_seven_"eight"_nine_ten_one`),
+				MustNewMatcher(MatchEqual, "ghi", `three_four_five_six_seven_eight_"nine"_ten_one_two`),
+				MustNewMatcher(MatchEqual, "nop", `four_five_six_seven_eight_nine_"ten"_one_two_three`),
+			},
+		},
+		{
+			name: "long name regexp",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchRegexp, "one_two_three_four_five_six_seven_eight_nine_ten", "val"),
+				MustNewMatcher(MatchRegexp, "two_three_four_five_six_seven_eight_nine_ten_one", "val"),
+				MustNewMatcher(MatchRegexp, "three_four_five_six_seven_eight_nine_ten_one_two", "val"),
+				MustNewMatcher(MatchRegexp, "four_five_six_seven_eight_nine_ten_one_two_three", "val"),
+				MustNewMatcher(MatchRegexp, "five_six_seven_eight_nine_ten_one_two_three_four", "val"),
+			},
+		},
+		{
+			name: "long quoted name regexp",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchRegexp, "one.two.three.four.five.six.seven.eight.nine.ten", "val"),
+				MustNewMatcher(MatchRegexp, "two.three.four.five.six.seven.eight.nine.ten.one", "val"),
+				MustNewMatcher(MatchRegexp, "three.four.five.six.seven.eight.nine.ten.one.two", "val"),
+				MustNewMatcher(MatchRegexp, "four.five.six.seven.eight.nine.ten.one.two.three", "val"),
+				MustNewMatcher(MatchRegexp, "five.six.seven.eight.nine.ten.one.two.three.four", "val"),
+			},
+		},
+		{
+			name: "long name and long value regexp",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchRegexp, "one_two_three_four_five_six_seven_eight_nine_ten", "five_six_seven_eight_nine_ten_one_two_three_four"),
+				MustNewMatcher(MatchRegexp, "two_three_four_five_six_seven_eight_nine_ten_one", "one_two_three_four_five_six_seven_eight_nine_ten"),
+				MustNewMatcher(MatchRegexp, "three_four_five_six_seven_eight_nine_ten_one_two", "two_three_four_five_six_seven_eight_nine_ten_one"),
+				MustNewMatcher(MatchRegexp, "four_five_six_seven_eight_nine_ten_one_two_three", "three_four_five_six_seven_eight_nine_ten_one_two"),
+				MustNewMatcher(MatchRegexp, "five_six_seven_eight_nine_ten_one_two_three_four", "four_five_six_seven_eight_nine_ten_one_two_three"),
+			},
+		},
+		{
+			name: "long quoted name and long value regexp",
+			matchers: []*Matcher{
+				MustNewMatcher(MatchRegexp, "one.two.three.four.five.six.seven.eight.nine.ten", "five.six.seven.eight.nine.ten.one.two.three.four"),
+				MustNewMatcher(MatchRegexp, "two.three.four.five.six.seven.eight.nine.ten.one", "one.two.three.four.five.six.seven.eight.nine.ten"),
+				MustNewMatcher(MatchRegexp, "three.four.five.six.seven.eight.nine.ten.one.two", "two.three.four.five.six.seven.eight.nine.ten.one"),
+				MustNewMatcher(MatchRegexp, "four.five.six.seven.eight.nine.ten.one.two.three", "three.four.five.six.seven.eight.nine.ten.one.two"),
+				MustNewMatcher(MatchRegexp, "five.six.seven.eight.nine.ten.one.two.three.four", "four.five.six.seven.eight.nine.ten.one.two.three"),
+			},
+		},
+	}
+
+	var mixed []*Matcher
+	for _, bc := range cases {
+		mixed = append(mixed, bc.matchers...)
+	}
+	rand.Shuffle(len(mixed), func(i, j int) { mixed[i], mixed[j] = mixed[j], mixed[i] })
+	cases = append(cases, benchCase{name: "mixed", matchers: mixed})
+
+	for _, bc := range cases {
+		b.Run(bc.name, func(b *testing.B) {
+			for i := 0; i <= b.N; i++ {
+				m := bc.matchers[i%len(bc.matchers)]
+				_ = m.String()
+			}
+		})
+	}
+}

model/relabel/relabel.go

@@ -17,6 +17,7 @@ import (
 	"crypto/md5"
 	"encoding/binary"
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/grafana/regexp"
@@ -290,7 +291,7 @@ func relabel(cfg *Config, lb *labels.Builder) (keep bool) {
 		hash := md5.Sum([]byte(val))
 		// Use only the last 8 bytes of the hash to give the same result as earlier versions of this code.
 		mod := binary.BigEndian.Uint64(hash[8:]) % cfg.Modulus
-		lb.Set(cfg.TargetLabel, fmt.Sprintf("%d", mod))
+		lb.Set(cfg.TargetLabel, strconv.FormatUint(mod, 10))
 	case LabelMap:
 		lb.Range(func(l labels.Label) {
 			if cfg.Regex.MatchString(l.Name) {

promql/parser/printer_test.go

@@ -138,6 +138,16 @@ func TestExprString(t *testing.T) {
 		{
 			in: `{__name__="",a="x"}`,
 		},
+		{
+			in: `{"a.b"="c"}`,
+		},
+		{
+			in: `{"0"="1"}`,
+		},
+		{
+			in:  `{"_0"="1"}`,
+			out: `{_0="1"}`,
+		},
 	}
 
 	for _, test := range inputs {

tsdb/index/index.go

@@ -52,6 +52,9 @@ const (
 	indexFilename = "index"
 
 	seriesByteAlign = 16
+
+	// checkContextEveryNIterations is used in some tight loops to check if the context is done.
+	checkContextEveryNIterations = 100
 )
 
 type indexWriterSeries struct {
@@ -1818,7 +1821,7 @@ func (r *Reader) postingsForLabelMatchingV1(ctx context.Context, name string, ma
 	var its []Postings
 	count := 1
 	for val, offset := range e {
-		if count%100 == 0 && ctx.Err() != nil {
+		if count%checkContextEveryNIterations == 0 && ctx.Err() != nil {
 			return ErrPostings(ctx.Err())
 		}
 		count++

tsdb/index/index_test.go

@@ -731,3 +731,36 @@ func TestChunksTimeOrdering(t *testing.T) {
 
 	require.NoError(t, idx.Close())
 }
+
+func TestReader_PostingsForLabelMatchingHonorsContextCancel(t *testing.T) {
+	dir := t.TempDir()
+
+	idx, err := NewWriter(context.Background(), filepath.Join(dir, "index"))
+	require.NoError(t, err)
+
+	seriesCount := 1000
+	for i := 1; i <= seriesCount; i++ {
+		require.NoError(t, idx.AddSymbol(fmt.Sprintf("%4d", i)))
+	}
+	require.NoError(t, idx.AddSymbol("__name__"))
+
+	for i := 1; i <= seriesCount; i++ {
+		require.NoError(t, idx.AddSeries(storage.SeriesRef(i), labels.FromStrings("__name__", fmt.Sprintf("%4d", i)),
+			chunks.Meta{Ref: 1, MinTime: 0, MaxTime: 10},
+		))
+	}
+
+	require.NoError(t, idx.Close())
+
+	ir, err := NewFileReader(filepath.Join(dir, "index"))
+	require.NoError(t, err)
+	defer ir.Close()
+
+	failAfter := uint64(seriesCount / 2) // Fail after processing half of the series.
+	ctx := &testutil.MockContextErrAfter{FailAfter: failAfter}
+	p := ir.PostingsForLabelMatching(ctx, "__name__", func(string) bool {
+		return true
+	})
+	require.Error(t, p.Err())
+	require.Equal(t, failAfter, ctx.Count())
+}

tsdb/index/postings.go

@@ -418,7 +418,7 @@ func (p *MemPostings) PostingsForLabelMatching(ctx context.Context, name string,
 	var its []Postings
 	count := 1
 	for _, v := range vals {
-		if count%100 == 0 && ctx.Err() != nil {
+		if count%checkContextEveryNIterations == 0 && ctx.Err() != nil {
 			return ErrPostings(ctx.Err())
 		}
 		count++

tsdb/index/postings_test.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/util/testutil"
 )
 
 func TestMemPostings_addFor(t *testing.T) {
@@ -1393,3 +1394,19 @@ func BenchmarkListPostings(b *testing.B) {
 		})
 	}
 }
+
+func TestMemPostings_PostingsForLabelMatchingHonorsContextCancel(t *testing.T) {
+	memP := NewMemPostings()
+	seriesCount := 10 * checkContextEveryNIterations
+	for i := 1; i <= seriesCount; i++ {
+		memP.Add(storage.SeriesRef(i), labels.FromStrings("__name__", fmt.Sprintf("%4d", i)))
+	}
+
+	failAfter := uint64(seriesCount / 2 / checkContextEveryNIterations)
+	ctx := &testutil.MockContextErrAfter{FailAfter: failAfter}
+	p := memP.PostingsForLabelMatching(ctx, "__name__", func(string) bool {
+		return true
+	})
+	require.Error(t, p.Err())
+	require.Equal(t, failAfter+1, ctx.Count()) // Plus one for the Err() call that puts the error in the result.
+}

tsdb/querier.go

@@ -33,6 +33,9 @@ import (
 	"github.com/prometheus/prometheus/util/annotations"
 )
 
+// checkContextEveryNIterations is used in some tight loops to check if the context is done.
+const checkContextEveryNIterations = 100
+
 type blockBaseQuerier struct {
 	blockID    ulid.ULID
 	index      IndexReader
@@ -359,7 +362,7 @@ func inversePostingsForMatcher(ctx context.Context, ix IndexPostingsReader, m *l
 	} else {
 		count := 1
 		for _, val := range vals {
-			if count%100 == 0 && ctx.Err() != nil {
+			if count%checkContextEveryNIterations == 0 && ctx.Err() != nil {
 				return nil, ctx.Err()
 			}
 			count++
@@ -394,13 +397,11 @@ func labelValuesWithMatchers(ctx context.Context, r IndexReader, name string, ma
 		// this is safe because the iteration is always ahead of the append
 		filteredValues := allValues[:0]
 		count := 1
-
 		for _, v := range allValues {
-			if count%100 == 0 && ctx.Err() != nil {
+			if count%checkContextEveryNIterations == 0 && ctx.Err() != nil {
 				return nil, ctx.Err()
 			}
 			count++
-
 			if m.Matches(v) {
 				filteredValues = append(filteredValues, v)
 			}