Generate source code using gopatch

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

.github/workflows/test-build-deploy.yml

@@ -30,7 +30,6 @@ jobs:
       # Determine if we will deploy (aka push) the image to the registry.
       is_deploy: ${{ (startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/r')) && github.event_name == 'push' && github.repository == 'grafana/mimir' }}
 
-
   goversion:
     runs-on: ubuntu-latest
     needs: prepare
@@ -103,6 +102,8 @@ jobs:
         run: make BUILD_IN_CONTAINER=false check-license
       - name: Check Docker-Compose YAML
         run: make BUILD_IN_CONTAINER=false check-mimir-microservices-mode-docker-compose-yaml check-mimir-read-write-mode-docker-compose-yaml
+      - name: Check Generated OTLP Code
+        run: make BUILD_IN_CONTAINER=false check-generated-otlp-code
 
   doc-validator:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -36,6 +36,7 @@
 * [ENHANCEMENT] Store-gateway: add `-blocks-storage.bucket-store.max-concurrent-queue-timeout`. When set, queries at the store-gateway's query gate will not wait longer than that to execute. If a query reaches the wait timeout, then the querier will retry the blocks on a different store-gateway. If all store-gateways are unavailable, then the query will fail with `err-mimir-store-consistency-check-failed`. #7777
 * [ENHANCEMENT] Ingester: Optimize querying with regexp matchers. #8106
 * [ENHANCEMENT] Distributor: Introduce `-distributor.max-request-pool-buffer-size` to allow configuring the maximum size of the request pool buffers. #8082
+* [ENHANCEMENT] OTLP: Speed up conversion from OTel to Mimir format by about 8% and reduce memory consumption by about 30%. #7957
 * [BUGFIX] Rules: improve error handling when querier is local to the ruler. #7567
 * [BUGFIX] Querier, store-gateway: Protect against panics raised during snappy encoding. #7520
 * [BUGFIX] Ingester: Prevent timely compaction of empty blocks. #7624

Makefile

@@ -771,3 +771,11 @@ test-packages: packages packaging/rpm/centos-systemd/$(UPTODATE) packaging/deb/d
 
 docs: doc
 	cd docs && $(MAKE) docs
+
+.PHONY: generate-otlp
+generate-otlp:
+	cd pkg/distributor/otlp && go generate
+
+.PHONY: check-generated-otlp-code
+check-generated-otlp-code:
+	cd pkg/distributor/otlp && go run ./cmd/generate --check

pkg/distributor/otlp/cmd/generate/.source-ref

@@ -0,0 +1 @@
+0d2cdfa9ed71

pkg/distributor/otlp/cmd/generate/README.md

@@ -0,0 +1,8 @@
+# OTLP translation code generator
+
+This command line utility ("generate") generates code for translating from an OTLP write request to a Mimir remote write request.
+It does so by downloading corresponding mimir-prometheus Go files, and transforming them through the
+[gopatch](https://github.com/uber-go/gopatch) tool.
+
+The utility maintains the source mimir-prometheus Git reference hash, from which the current sources were generated, in the file .source-ref in this directory.
+If the mimir-prometheus version currently depended on hasn't changed from that, no source code is re-generated.

pkg/distributor/otlp/cmd/generate/main.go

@@ -5,44 +5,101 @@ package main
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
-	"regexp"
+	"runtime"
 	"strings"
-	"text/template"
 
-	"github.com/google/go-github/v61/github"
-)
+	"github.com/alecthomas/kingpin/v2"
+	"github.com/grafana/regexp"
 
-type templateContext struct {
-	Name                    string
-	PbPackagePath           string
-	PbPackage               string
-	Package                 string
-	LabelType               string
-	SampleTimestampField    string
-	ExemplarTimestampField  string
-	HistogramTimestampField string
-	UnknownType             string
-	GaugeType               string
-	CounterType             string
-	HistogramType           string
-	SummaryType             string
-}
+	"github.com/google/go-github/v57/github"
+)
 
 const (
+	dpath = "storage/remote/otlptranslator/prometheusremotewrite"
 	owner = "grafana"
 	repo  = "mimir-prometheus"
-	dpath = "storage/remote/otlptranslator/prometheusremotewrite/templates"
 )
 
 func main() {
+	app := kingpin.New("generate", "Generate OTLP translation code.")
+	checkFlag := app.Flag("check", "Verify that generated OTLP translation code corresponds to mimir-prometheus version.").Bool()
+
+	generate := func(*kingpin.ParseContext) error {
+		sourceRefPath := filepath.Join("cmd", "generate", ".source-ref")
+
+		if *checkFlag {
+			return check(sourceRefPath)
+		}
+
+		sources, sourceRef, err := downloadSources(sourceRefPath)
+		if err != nil {
+			return err
+		}
+		if sourceRef == "" {
+			// The source hasn't changed since last time
+			return nil
+		}
+
+		if err := patchSources(sources); err != nil {
+			return err
+		}
+
+		if err := os.WriteFile(sourceRefPath, []byte(sourceRef+"\n"), 0644); err != nil {
+			return fmt.Errorf("couldn't write to %s: %w", sourceRefPath, err)
+		}
+
+		return nil
+	}
+
+	app = app.Action(generate)
+	kingpin.MustParse(app.Parse(os.Args[1:]))
+}
+
+func patchSources(sources []string) error {
+	if out, err := exec.Command("gopatch", "-p", "cmd/generate/mimirpb.patch", "./").CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to execute gopatch: %s", out)
+	}
+	sed := "sed"
+	if runtime.GOOS == "darwin" {
+		sed = "gsed"
+	}
+	for _, fname := range sources {
+		if out, err := exec.Command(sed, "-i", "s/PrometheusConverter/MimirConverter/g", fname).CombinedOutput(); err != nil {
+			return fmt.Errorf("failed to execute sed: %s", out)
+		}
+		if out, err := exec.Command(sed, "-i", "s/Prometheus remote write format/Mimir remote write format/g", fname).CombinedOutput(); err != nil {
+			return fmt.Errorf("failed to execute sed: %s", out)
+		}
+		if out, err := exec.Command("goimports", "-w", "-local", "github.com/grafana/mimir", fname).CombinedOutput(); err != nil {
+			return fmt.Errorf("failed to execute goimports: %s", out)
+		}
+	}
+
+	return nil
+}
+
+// getSourceRef gets the mimir-prometheus source GitHub reference hash,
+// and whether it's different from the source ref the code was generated from.
+func getSourceRef(sourceRefPath string) (string, bool, error) {
+	var curSourceRef string
+	data, err := os.ReadFile(sourceRefPath)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return "", false, fmt.Errorf("couldn't read file %s: %w", sourceRefPath, err)
+		}
+	} else {
+		curSourceRef = strings.TrimSpace(string(data))
+	}
+
 	f, err := os.Open(filepath.Join("..", "..", "..", "go.mod"))
 	if err != nil {
-		panic(err)
+		return "", false, err
 	}
 	defer f.Close()
 
@@ -59,130 +116,106 @@ func main() {
 		ref = ms[1]
 	}
 	if err := scanner.Err(); err != nil {
-		panic(err)
+		return "", false, err
 	}
 
 	if ref == "" {
-		panic(fmt.Errorf("Couldn't find mimir-prometheus replace directive in go.mod"))
+		return "", false, fmt.Errorf("couldn't find mimir-prometheus replace directive in go.mod")
+	}
+
+	return ref, ref != curSourceRef, nil
+}
+
+func downloadSources(sourceRefPath string) ([]string, string, error) {
+	ref, changed, err := getSourceRef(sourceRefPath)
+	if err != nil {
+		return nil, "", err
+	}
+	if !changed {
+		return nil, "", nil
 	}
 
 	ctx := context.Background()
 
 	token := os.Getenv("GITHUB_TOKEN")
+	if token == "" {
+		return nil, "", fmt.Errorf("$GITHUB_TOKEN must be set")
+	}
 	client := github.NewClient(nil).WithAuthToken(token)
 	_, contents, _, err := client.Repositories.GetContents(
 		ctx, owner, repo, dpath, &github.RepositoryContentGetOptions{
 			Ref: ref,
 		},
 	)
 	if err != nil {
-		panic(fmt.Errorf("getting repo dir contents: %w", err))
-	}
-
-	if err := os.MkdirAll("templates", 0755); err != nil {
-		panic(err)
+		return nil, "", fmt.Errorf("getting repo dir contents: %w", err)
 	}
 
-	var templateFiles []string
+	var sources []string
 	for _, c := range contents {
-		if *c.Type != "file" || !strings.HasSuffix(*c.Name, ".go.tmpl") {
+		if *c.Type != "file" || !strings.HasSuffix(*c.Name, ".go") || *c.Name == "timeseries.go" || strings.HasSuffix(*c.Name, "_test.go") {
 			continue
 		}
 
-		if err := downloadFile(ctx, client, ref, *c.Name); err != nil {
-			panic(err)
-		}
-		templateFiles = append(templateFiles, filepath.Join("templates", *c.Name))
-	}
-
-	c := templateContext{
-		Name:                    "Mimir",
-		Package:                 "otlp",
-		PbPackagePath:           "github.com/grafana/mimir/pkg/mimirpb",
-		PbPackage:               "mimirpb",
-		LabelType:               "LabelAdapter",
-		SampleTimestampField:    "TimestampMs",
-		ExemplarTimestampField:  "TimestampMs",
-		HistogramTimestampField: "Timestamp",
-		UnknownType:             "UNKNOWN",
-		GaugeType:               "GAUGE",
-		CounterType:             "COUNTER",
-		HistogramType:           "HISTOGRAM",
-		SummaryType:             "SUMMARY",
-	}
-
-	for _, fpath := range templateFiles {
-		t, err := template.ParseFiles(fpath)
+		fname, err := downloadFile(ctx, client, ref, *c.Name)
 		if err != nil {
-			panic(err)
-		}
-
-		name, _, found := strings.Cut(filepath.Base(fpath), ".")
-		if !found {
-			panic(fmt.Errorf("invalid filename %q", fpath))
-		}
-		if err := executeTemplate(t, name, c); err != nil {
-			panic(err)
+			return nil, "", err
 		}
+		sources = append(sources, fname)
 	}
+
+	return sources, ref, nil
 }
 
-func downloadFile(ctx context.Context, client *github.Client, ref, fname string) (err error) {
+func downloadFile(ctx context.Context, client *github.Client, ref, fname string) (string, error) {
 	ghFpath := path.Join(dpath, fname)
 	r, _, err := client.Repositories.DownloadContents(
 		ctx, owner, repo, ghFpath, &github.RepositoryContentGetOptions{
 			Ref: ref,
 		},
 	)
 	if err != nil {
-		return fmt.Errorf("download %s: %w", ghFpath, err)
+		return "", fmt.Errorf("download %s: %w", ghFpath, err)
 	}
 	defer r.Close()
 
-	fpath := filepath.Join("templates", fname)
-	f, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	reFilename := regexp.MustCompile(`^(.+)\.go$`)
+	ms := reFilename.FindStringSubmatch(fname)
+	outputFname := fmt.Sprintf("%s_generated.go", ms[1])
+
+	f, err := os.OpenFile(outputFname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 	if err != nil {
-		return fmt.Errorf("open file %s: %w", fpath, err)
+		return "", fmt.Errorf("open file %s: %w", outputFname, err)
 	}
 	defer func() {
 		if closeErr := f.Close(); closeErr != nil && err == nil {
-			err = fmt.Errorf("close %s: %w", fpath, closeErr)
+			err = fmt.Errorf("close %s: %w", outputFname, closeErr)
 		}
 	}()
 
 	wr := bufio.NewWriter(f)
+	if _, err := wr.WriteString("// Code generated from Prometheus sources - DO NOT EDIT.\n\n"); err != nil {
+		return "", fmt.Errorf("write to %s: %w", outputFname, err)
+	}
 	if _, err := wr.ReadFrom(r); err != nil {
-		return fmt.Errorf("read from GitHub repo file %s: %w", fpath, err)
+		return "", fmt.Errorf("read from GitHub repo file %s: %w", ghFpath, err)
 	}
 	if err := wr.Flush(); err != nil {
-		return fmt.Errorf("write to file: %w", err)
+		return "", fmt.Errorf("write to file %s: %w", outputFname, err)
 	}
 
-	return nil
+	return outputFname, nil
 }
 
-func executeTemplate(t *template.Template, name string, c templateContext) (err error) {
-	fname := fmt.Sprintf("%s_generated.go", name)
-	f, err := os.OpenFile(fname, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+// check verifies that the generated OTLP translation sources correspond to the current mimir-promethus version depended on.
+func check(sourceRefPath string) error {
+	_, changed, err := getSourceRef(sourceRefPath)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		if closeErr := f.Close(); closeErr != nil && err == nil {
-			err = fmt.Errorf("close %s: %w", fname, closeErr)
-		}
-	}()
-
-	if _, err := f.WriteString("// Code generated from templates/*.go.tmpl - DO NOT EDIT.\n\n"); err != nil {
-		return err
-	}
-
-	if err := t.Execute(f, c); err != nil {
-		return fmt.Errorf("execute template: %w", err)
-	}
-
-	if out, err := exec.Command("goimports", "-w", "-local", "github.com/grafana/mimir", fname).CombinedOutput(); err != nil {
-		return fmt.Errorf("execute goimports on %s: %s", fname, out)
+	if changed {
+		return fmt.Errorf("the OTLP translation code was generated from an older version of mimir-prometheus - " +
+			"please regenerate and commit: make generate-otlp")
 	}
 
 	return nil