oasis-test-runner: fix remote-signer/basic parameters, add run as subdir

.buildkite/scripts/test_e2e.sh

@@ -44,7 +44,7 @@ ${WORKDIR}/go/oasis-test-runner/oasis-test-runner \
     --e2e.runtime.binary_dir ${WORKDIR}/target/${runtime_target}/debug \
     --e2e.runtime.loader ${WORKDIR}/target/default/debug/oasis-core-runtime-loader \
     --e2e.tee_hardware ${OASIS_TEE_HARDWARE:-""} \
-    --remote_signer.binary ${WORKDIR}/go/oasis-remote-signer/oasis-remote-signer \
+    --params.remote-signer/basic.binary ${WORKDIR}/go/oasis-remote-signer/oasis-remote-signer \
     --log.level info \
     ${BUILDKITE_PARALLEL_JOB_COUNT:+--parallel.job_count ${BUILDKITE_PARALLEL_JOB_COUNT}} \
     ${BUILDKITE_PARALLEL_JOB:+--parallel.job_index ${BUILDKITE_PARALLEL_JOB}} \

go/oasis-test-runner/cmd/root.go

@@ -325,7 +325,7 @@ func runRoot(cmd *cobra.Command, args []string) error {
 		for name, sc := range toRunExploded {
 			for i, v := range sc {
 				// Maintain unique scenario datadir.
-				n := fmt.Sprintf("%s-%d", name, run*len(sc)+i)
+				n := fmt.Sprintf("%s/%d", name, run*len(sc)+i)
 
 				if index%parallelJobCount != parallelJobIndex {
 					logger.Info("skipping test case (assigned to different parallel job)",
@@ -337,7 +337,7 @@ func runRoot(cmd *cobra.Command, args []string) error {
 
 				if excludeMap[strings.ToLower(v.Name())] {
 					logger.Info("skipping test case (excluded by environment)",
-						"test", v.Name(),
+						"test", n,
 					)
 					index++
 					continue

go/oasis-test-runner/cmd/root_test.go

@@ -14,12 +14,12 @@ func TestComputeParamSets(t *testing.T) {
 
 	expectedParamSets := []map[string]string{
 		{"testParam1": "1", "testParam2": "a"},
-		{"testParam1": "1", "testParam2": "b"},
 		{"testParam1": "2", "testParam2": "a"},
-		{"testParam1": "2", "testParam2": "b"},
 		{"testParam1": "3", "testParam2": "a"},
+		{"testParam1": "1", "testParam2": "b"},
+		{"testParam1": "2", "testParam2": "b"},
 		{"testParam1": "3", "testParam2": "b"},
 	}
 
-	require.Equal(t, computeParamSets(zippedParams, map[string]string{}), expectedParamSets)
+	require.Equal(t, expectedParamSets, computeParamSets(zippedParams, map[string]string{}))
 }

go/oasis-test-runner/oasis/args.go

@@ -421,10 +421,9 @@ func (args *argBuilder) appendNodeMetrics(node *Node) *argBuilder {
 
 	// TODO Matevz: Populate with test-parameters, startdate, git_branch?
 	dirs := strings.Split(node.dir.String(), "/")
-	testNameRun := strings.Split(dirs[len(dirs)-3], "-")
 	l := []string{"instance=" + dirs[len(dirs)-4],
-		"run=" + testNameRun[len(testNameRun)-1],
-		"test=" + strings.Join(testNameRun[:len(testNameRun)-1], "-"),
+		"run=" + dirs[len(dirs)-3],
+		"test=" + dirs[len(dirs)-4], // TODO: What if test name contains / like "gas-fees/runtimes"?
 		"software_version=" + version.SoftwareVersion}
 
 	args.vec = append(args.vec, strings.Join(l, ","))

go/oasis-test-runner/scenario/remotesigner/basic.go

@@ -5,9 +5,6 @@ import (
 	"path/filepath"
 	"time"
 
-	flag "github.com/spf13/pflag"
-	"github.com/spf13/viper"
-
 	"github.com/oasislabs/oasis-core/go/common/crypto/signature"
 	fileSigner "github.com/oasislabs/oasis-core/go/common/crypto/signature/signers/file"
 	remoteSigner "github.com/oasislabs/oasis-core/go/common/crypto/signature/signers/remote"
@@ -20,24 +17,34 @@ import (
 	"github.com/oasislabs/oasis-core/go/oasis-test-runner/scenario"
 )
 
-const cfgRemoteSignerBinary = "remote_signer.binary"
-
 var (
 	// Basic is the basic test case.
 	Basic scenario.Scenario = newBasicImpl()
-
-	// Flags is the command line flags for the remote signer tests.
-	Flags = flag.NewFlagSet("", flag.ContinueOnError)
 )
 
 func newBasicImpl() *basicImpl {
 	return &basicImpl{
-		logger: logging.GetLogger("remote-signer/basic"),
+		logger:       logging.GetLogger("remote-signer/basic"),
+		serverBinary: "oasis-remote-signer",
 	}
 }
 
 type basicImpl struct {
 	logger *logging.Logger
+
+	serverBinary string
+}
+
+func (sc *basicImpl) Clone() scenario.Scenario {
+	return &basicImpl{
+		logger: logging.GetLogger("remote-signer/basic"),
+	}
+}
+
+func (sc *basicImpl) Parameters() map[string]interface{} {
+	return map[string]interface{}{
+		"binary": &sc.serverBinary,
+	}
 }
 
 func (sc *basicImpl) Name() string {
@@ -53,15 +60,13 @@ func (sc *basicImpl) Init(childEnv *env.Env, net *oasis.Network) error {
 }
 
 func (sc *basicImpl) Run(childEnv *env.Env) error {
-	serverBinary := viper.GetString(cfgRemoteSignerBinary)
-
 	// Provision the server keys.
 	sc.logger.Info("provisioning the server keys")
 	if err := cli.RunSubCommand(
 		childEnv,
 		sc.logger,
 		"init",
-		serverBinary,
+		sc.serverBinary,
 		[]string{
 			"--" + cmdCommon.CfgDataDir, childEnv.Dir(),
 			"init",
@@ -87,7 +92,7 @@ func (sc *basicImpl) Run(childEnv *env.Env) error {
 		childEnv,
 		sc.logger,
 		"init_client",
-		serverBinary,
+		sc.serverBinary,
 		[]string{
 			"--" + cmdCommon.CfgDataDir, childEnv.Dir(),
 			"init_client",
@@ -106,7 +111,7 @@ func (sc *basicImpl) Run(childEnv *env.Env) error {
 		childEnv,
 		sc.logger,
 		"server",
-		serverBinary,
+		sc.serverBinary,
 		[]string{
 			"--" + cmdCommon.CfgDataDir, childEnv.Dir(),
 			"--client.certificate", filepath.Join(childEnv.Dir(), "remote_signer_client_cert.pem"),
@@ -189,8 +194,3 @@ func (sc *basicImpl) Run(childEnv *env.Env) error {
 
 	return nil
 }
-
-func init() {
-	Flags.String(cfgRemoteSignerBinary, "oasis-remote-signer", "path to the remote-signer binary")
-	_ = viper.BindPFlags(Flags)
-}

go/oasis-test-runner/test-runner.go

@@ -66,7 +66,6 @@ func main() {
 	_ = cmd.Register(e2e.NodeUpgradeCancel)
 
 	// Register the remote signer test cases.
-	rootCmd.Flags().AddFlagSet(remotesigner.Flags)
 	_ = cmd.Register(remotesigner.Basic)
 
 	// Execute the command, now that everything has been initialized.