metrics: add new pusher to oasis-test-runner for up metric

go/oasis-node/cmd/common/metrics/metrics.go

@@ -131,7 +131,7 @@ func (s *stubService) Cleanup() {}
 func newStubService() (service.BackgroundService, error) {
 	svc := *service.NewBaseBackgroundService("metrics")
 
-	d, err := NewDiskUsageService()
+	d, err := NewDiskService()
 	if err != nil {
 		return nil, err
 	}
@@ -206,7 +206,7 @@ func newPullService(ctx context.Context) (service.BackgroundService, error) {
 		return nil, err
 	}
 
-	d, err := NewDiskUsageService()
+	d, err := NewDiskService()
 	if err != nil {
 		return nil, err
 	}
@@ -288,7 +288,7 @@ func newPushService() (service.BackgroundService, error) {
 		pusher = pusher.Grouping(k, v)
 	}
 
-	d, err := NewDiskUsageService()
+	d, err := NewDiskService()
 	if err != nil {
 		return nil, err
 	}
@@ -381,11 +381,11 @@ func (d *diskService) worker() {
 	}
 }
 
-// NewDiskUsageService constructs a new disk usage service.
+// NewDiskService constructs a new disk usage and I/O service.
 //
-// This service will compute the size of datadir folder by calling "du" command and read I/O info from /proc/<PID>/io
-// file every --metric.push.interval seconds.
-func NewDiskUsageService() (service.BackgroundService, error) {
+// This service will compute the size of datadir folder and read I/O info of the process every --metric.push.interval
+// seconds.
+func NewDiskService() (service.BackgroundService, error) {
 	diskOnce.Do(func() {
 		prometheus.MustRegister(diskCollectors...)
 	})

go/oasis-test-runner/cmd/root.go

@@ -9,9 +9,12 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/push"
 	"github.com/spf13/cobra"
 	flag "github.com/spf13/pflag"
 	"github.com/spf13/viper"
@@ -59,6 +62,21 @@ var (
 	scenarioMap      = make(map[string]scenario.Scenario)
 	defaultScenarios []scenario.Scenario
 	scenarios        []scenario.Scenario
+
+	// oasis-test-runner-specific metrics.
+	upGauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "up",
+			Help: "Is oasis-test-runner test active",
+		},
+	)
+
+	oasisTestRunnerCollectors = []prometheus.Collector{
+		upGauge,
+	}
+
+	pusher              *push.Pusher
+	oasisTestRunnerOnce sync.Once
 )
 
 // RootCmd returns the root command's structure that will be executed, so that
@@ -254,6 +272,12 @@ func initRootEnv(cmd *cobra.Command) (*env.Env, error) {
 func runRoot(cmd *cobra.Command, args []string) error {
 	cmd.SilenceUsage = true
 
+	if viper.GetString(metrics.CfgMetricsAddr) != "" {
+		oasisTestRunnerOnce.Do(func() {
+			prometheus.MustRegister(oasisTestRunnerCollectors...)
+		})
+	}
+
 	// Initialize the base dir, logging, etc.
 	rootEnv, err := initRootEnv(cmd)
 	if err != nil {
@@ -326,7 +350,7 @@ func runRoot(cmd *cobra.Command, args []string) error {
 				)
 
 				childEnv, err := rootEnv.NewChild(n, env.TestInstanceInfo{
-					Name:         v.Name(),
+					Test:         v.Name(),
 					Instance:     filepath.Base(rootEnv.Dir()),
 					ParameterSet: scenario.ParametersToStringMap(v.Parameters()),
 					Run:          run,
@@ -344,6 +368,15 @@ func runRoot(cmd *cobra.Command, args []string) error {
 					return err
 				}
 
+				// Init per-run prometheus pusher, if metrics are enabled.
+				if viper.GetString(metrics.CfgMetricsAddr) != "" {
+					pusher = push.New(viper.GetString(metrics.CfgMetricsAddr), "oasis-test-runner")
+					pusher = pusher.Grouping("instance", childEnv.TestInfo().Instance)
+					pusher = pusher.Grouping("run", strconv.Itoa(childEnv.TestInfo().Run))
+					pusher = pusher.Grouping("test", childEnv.TestInfo().Test)
+					pusher = pusher.Gatherer(prometheus.DefaultGatherer)
+				}
+
 				if err = doScenario(childEnv, v); err != nil {
 					logger.Error("failed to run test case",
 						"err", err,
@@ -406,11 +439,27 @@ func doScenario(childEnv *env.Env, scenario scenario.Scenario) (err error) {
 		return
 	}
 
+	if pusher != nil {
+		upGauge.Set(1.0)
+		if err = pusher.Push(); err != nil {
+			err = errors.Wrap(err, "root: failed to push metrics")
+			return
+		}
+	}
+
 	if err = scenario.Run(childEnv); err != nil {
 		err = errors.Wrap(err, "root: failed to run test case")
 		return
 	}
 
+	if pusher != nil {
+		upGauge.Set(0.0)
+		if err = pusher.Push(); err != nil {
+			err = errors.Wrap(err, "root: failed to push metrics")
+			return
+		}
+	}
+
 	return
 }
 

go/oasis-test-runner/env/env.go

@@ -21,8 +21,8 @@ type CleanupFn func()
 
 // TestInstanceInfo contains information of the current test run.
 type TestInstanceInfo struct {
-	// Name is the name of the test.
-	Name string `json:"name"`
+	// Test is the name of the test.
+	Test string `json:"test"`
 
 	// Instance is the instance name of the test. e.g. oasis-test-runner123456
 	Instance string `json:"instance"`

go/oasis-test-runner/oasis/args.go

@@ -421,7 +421,7 @@ func (args *argBuilder) appendNodeMetrics(node *Node) *argBuilder {
 	ti := node.net.env.TestInfo()
 	l := []string{"instance=" + ti.Instance,
 		"run=" + strconv.Itoa(ti.Run),
-		"test=" + ti.Name,
+		"test=" + ti.Test,
 		"software_version=" + version.SoftwareVersion,
 	}
 	// Populate it with test-provided parameters.