promql: Dockerized the tool; split cases and config for easier reuse.

Additionally updated docs.

Motivation for this is that we can now set automated tests where in future we might want to provide
docker images with exact set of test cases in versioned image.

See automated use of it in Thanos: thanos-io/thanos#4781

Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>

promql/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.16 as build-env
+
+WORKDIR /go/src/promql
+COPY . /go/src/promql
+
+RUN go build ./cmd/promql-compliance-tester
+
+FROM quay.io/prometheus/busybox
+COPY --from=build-env /go/src/promql/promql-compliance-tester /
+COPY --from=build-env /go/src/promql/promql-compliance-cases.yml /
+COPY --from=build-env /go/src/promql/promql-compliance-tester.yml /
+
+ENTRYPOINT ["/promql-compliance-tester"]

promql/README.md

@@ -4,25 +4,35 @@ The PromQL Compliance Tester is a tool for running comparison tests between nati
 
 The tool was [first published and described](https://promlabs.com/blog/2020/08/06/comparing-promql-correctness-across-vendors) in August 2020. [Test results have been published](https://promlabs.com/promql-compliance-tests) on 2020-08-06 and 2020-12-01.
 
-## Build Requirements
+## Building via Docker
+
+If you have docker installed, you can build the tool using docker.
+
+```bash
+ docker build . -t promql-compliance-tester:latest
+```
+
+## Build from source
+
+### Requirements
 
 This tool is written in Go and requires a working Go setup to build. Library dependencies are handled via [Go Modules](https://blog.golang.org/using-go-modules).
 
-## Building
+### Building
 
 To build the tool:
 
 ```bash
 go build ./cmd/promql-compliance-tester
 ```
 
-## Available flags
+## Executing
 
-To list available flags:
+Tool allows setting following flags:
 
 ```
 $ ./promql-compliance-tester -h
-Usage of ./promql-compliance-tester:
+Usage of /promql-compliance-tester:
   -config-file string
     	The path to the configuration file. (default "promql-compliance-tester.yml")
   -output-format string
@@ -31,17 +41,33 @@ Usage of ./promql-compliance-tester:
     	The HTML template to use when using HTML as the output format. (default "./output/example-output.html")
   -output-passing
     	Whether to also include passing test cases in the output.
+  -queries-file string
+    	The path to the test cases to run. (default "promql-compliance-cases.yml")
+  -query-parallelism int
+    	Maximum number of comparison queries to run in parallel. (default 20)
 ```
 
+Running the tool will execute all test cases in `-queries-file` and compare results between reference and target provided in `-config-file`.
+
+At the end of run, the output is provided in form or number of executed tests and errors if any. Example output can be seen here:
+
+```bash
+./promql-compliance-tester -config-file config.yaml
+529 / 529 [-----------------------------------------------------------------------------------------------------------] 100.00% 278 p/s
+Total: 529 / 529 (100.00%) passed, 0 unsupported
+```
+
+If all tests were executed correctly and passing the tool returns 0 code, otherwise it returns 1.
+
 ## Configuration
 
-The test cases, query tweaks, and PromQL API endpoints to use are specified in a configuration file.
+The query tweaks, and PromQL API endpoints to use are specified in a configuration file using `config-file` flag. An example configuration file with settings for Thanos, Cortex, TimescaleDB, and VictoriaMetrics is included [here](promql-compliance-tester.yml).
 
-An example configuration file with settings for Thanos, Cortex, TimescaleDB, and VictoriaMetrics is included.
+The test cases are configured using `queries-file` option. An example cases used in recent compliance tests are provided [here](promql-compliance-cases.yml)
 
 ## Contributing
 
-It's still early days for the PromQL Compliance Tester. In particular, we would love to add and improve the following points:
+Help wanted to improve the PromQL Compliance Tester. In particular, we would love to add and improve the following points:
 
 * Test instant queries in addition to range queries.
 * Add more variation and configurability to input timestamps.

promql/cmd/promql-compliance-tester/main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"math"
 	"net/http"
+	"os"
 	"strconv"
 	"sync"
 	"time"
@@ -55,6 +56,7 @@ func (rt roundTripperWithSettings) RoundTrip(req *http.Request) (*http.Response,
 
 func main() {
 	configFile := flag.String("config-file", "promql-compliance-tester.yml", "The path to the configuration file.")
+	casesFile := flag.String("queries-file", "promql-compliance-cases.yml", "The path to the test cases to run.")
 	outputFormat := flag.String("output-format", "text", "The comparison output format. Valid values: [text, html, json]")
 	outputHTMLTemplate := flag.String("output-html-template", "./output/example-output.html", "The HTML template to use when using HTML as the output format.")
 	outputPassing := flag.Bool("output-passing", false, "Whether to also include passing test cases in the output.")
@@ -83,6 +85,11 @@ func main() {
 	if err != nil {
 		log.Fatalf("Error loading configuration file: %v", err)
 	}
+
+	cases, err := config.LoadTestCasesFromFile(*casesFile)
+	if err != nil {
+		log.Fatalf("Error loading test cases file: %v", err)
+	}
 	refAPI, err := newPromAPI(cfg.ReferenceTargetConfig)
 	if err != nil {
 		log.Fatalf("Error creating reference API: %v", err)
@@ -99,7 +106,7 @@ func main() {
 		-getNonZeroDuration(cfg.QueryTimeParameters.RangeInSeconds, 10*time.Minute))
 	resolution := getNonZeroDuration(
 		cfg.QueryTimeParameters.ResolutionInSeconds, 10*time.Second)
-	expandedTestCases := testcases.ExpandTestCases(cfg.TestCases, cfg.QueryTweaks, start, end, resolution)
+	expandedTestCases := testcases.ExpandTestCases(cases.TestCases, cfg.QueryTweaks, start, end, resolution)
 
 	var wg sync.WaitGroup
 	results := make([]*comparer.Result, len(expandedTestCases))
@@ -127,6 +134,14 @@ func main() {
 	progressBar.Finish()
 
 	outp(results, *outputPassing, cfg.QueryTweaks)
+
+	// If any of tests failed, return 1.
+	for _, r := range results {
+		if !r.Success() {
+			os.Exit(1)
+		}
+	}
+	os.Exit(0)
 }
 
 func getTime(timeStr string, defaultTime time.Time) time.Time {

promql/config/config.go

@@ -13,7 +13,6 @@ type Config struct {
 	ReferenceTargetConfig TargetConfig        `yaml:"reference_target_config"`
 	TestTargetConfig      TargetConfig        `yaml:"test_target_config"`
 	QueryTweaks           []*QueryTweak       `yaml:"query_tweaks"`
-	TestCases             []*TestCase         `yaml:"test_cases"`
 	QueryTimeParameters   QueryTimeParameters `yaml:"query_time_parameters"`
 }
 
@@ -48,6 +47,10 @@ type AdjustValueTolerance struct {
 	Margin   *float64 `yaml:"margin" json:"margin,omitempty"`
 }
 
+type TestCasesConfig struct {
+	TestCases []*TestCase `yaml:"test_cases"`
+}
+
 // TestCase represents a given query (pattern) to be tested.
 type TestCase struct {
 	Query          string   `yaml:"query"`
@@ -78,3 +81,26 @@ func Load(content []byte) (*Config, error) {
 	}
 	return cfg, nil
 }
+
+// LoadTestCasesFromFile parses the given YAML file into a TestCasesConfig.
+func LoadTestCasesFromFile(filename string) (*TestCasesConfig, error) {
+	content, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	cfg, err := LoadTestCases(content)
+	if err != nil {
+		return nil, errors.Wrapf(err, "parsing YAML file %s", filename)
+	}
+	return cfg, nil
+}
+
+// LoadTestCases parses the YAML input into a Config.
+func LoadTestCases(content []byte) (*TestCasesConfig, error) {
+	cfg := &TestCasesConfig{}
+	err := yaml.UnmarshalStrict(content, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}