oasis-node: Initial work on custom file-based internal unix sockets

go/oasis-node/cmd/common/grpc/grpc.go

@@ -6,6 +6,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -19,6 +20,7 @@ import (
 	"github.com/oasislabs/oasis-core/go/common/logging"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/common"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/flags"
+	"github.com/oasislabs/oasis-core/go/oasis-test-runner/env"
 )
 
 const (
@@ -28,8 +30,8 @@ const (
 	CfgAddress = "address"
 	// CfgWait waits for the remote address to become available.
 	CfgWait = "wait"
-	// CfgDebugGrpcAbstractSocket uses @oasis-XXXXXXXX instead of /tmp/verylongtestsuitetestnamerunid/internal.sock.
-	CfgDebugGrpcAbstractSocket = "debug.grpc.abstract_socket"
+	// CfgDebugGrpcSocketPath sets custom internal socket path.
+	CfgDebugGrpcSocketPath = "debug.grpc.socket.path"
 
 	defaultAddress      = "unix:" + LocalSocketFilename
 	LocalSocketFilename = "internal.sock"
@@ -73,8 +75,8 @@ func NewServerLocal(installWrapper bool) (*cmnGrpc.Server, error) {
 		return nil, errors.New("data directory must be set")
 	}
 	path := filepath.Join(dataDir, LocalSocketFilename)
-	if viper.GetBool(CfgDebugGrpcAbstractSocket) && flags.DebugDontBlameOasis() {
-		path = GetAbstractSocketAddress(dataDir)
+	if viper.IsSet(CfgDebugGrpcSocketPath) && flags.DebugDontBlameOasis() {
+		path = viper.GetString(CfgDebugGrpcSocketPath)
 	}
 
 	config := &cmnGrpc.ServerConfig{
@@ -121,8 +123,8 @@ func init() {
 	_ = viper.BindPFlags(ServerTCPFlags)
 	ServerTCPFlags.AddFlagSet(cmnGrpc.Flags)
 
-	ServerLocalFlags.Bool(CfgDebugGrpcAbstractSocket, false, "use abstract internal unix socket")
-	_ = ServerLocalFlags.MarkHidden(CfgDebugGrpcAbstractSocket)
+	ServerLocalFlags.String(CfgDebugGrpcSocketPath, "", "use custom internal unix socket path")
+	_ = ServerLocalFlags.MarkHidden(CfgDebugGrpcSocketPath)
 	_ = viper.BindPFlags(ServerLocalFlags)
 	ServerLocalFlags.AddFlagSet(cmnGrpc.Flags)
 

go/oasis-test-runner/cmd/root.go

@@ -21,6 +21,7 @@ import (
 	"github.com/oasislabs/oasis-core/go/common/version"
 	nodeCommon "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common"
 	nodeFlags "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/flags"
+	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/grpc"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/metrics"
 	"github.com/oasislabs/oasis-core/go/oasis-test-runner/cmd/cmp"
 	"github.com/oasislabs/oasis-core/go/oasis-test-runner/cmd/common"
@@ -431,8 +432,7 @@ func doScenario(childEnv *env.Env, sc scenario.Scenario) (err error) {
 			return
 		}
 
-		// Use abstract sockets to avoid too long socket paths.
-		net.Config().UseAbstractGrpcSockets = true
+		// TODO matevz: Set shorter socket paths per-node.
 	}
 
 	if err = sc.Init(childEnv, net); err != nil {

go/oasis-test-runner/oasis/args.go

@@ -81,8 +81,8 @@ func (args *argBuilder) grpcLogDebug() *argBuilder {
 	return args
 }
 
-func (args *argBuilder) grpcDebugAbstractSocket() *argBuilder {
-	args.vec = append(args.vec, "--"+grpc.CfgDebugGrpcAbstractSocket)
+func (args *argBuilder) grpcDebugGrpcSocketPath(path string) *argBuilder {
+	args.vec = append(args.vec, "--"+grpc.CfgDebugGrpcSocketPath, path)
 	return args
 }
 

go/oasis-test-runner/oasis/compute.go

@@ -134,6 +134,10 @@ func (net *Network) NewCompute(cfg *ComputeCfg) (*Compute, error) {
 		cfg.RuntimeBackend = workerHost.BackendSandboxed
 	}
 
+	sp, err := net.getTempSocketPath()
+	if err != nil {
+		return nil, err
+	}
 	worker := &Compute{
 		Node: Node{
 			Name:                                     computeName,
@@ -144,6 +148,7 @@ func (net *Network) NewCompute(cfg *ComputeCfg) (*Compute, error) {
 			disableDefaultLogWatcherHandlerFactories: cfg.DisableDefaultLogWatcherHandlerFactories,
 			logWatcherHandlerFactories:               cfg.LogWatcherHandlerFactories,
 			consensus:                                cfg.Consensus,
+			customGrpcSocketPath: sp,
 		},
 		entity:         cfg.Entity,
 		runtimeBackend: cfg.RuntimeBackend,

go/oasis-test-runner/oasis/oasis.go

@@ -5,6 +5,7 @@ import (
 	"crypto"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math"
 	"os/exec"
 	"path/filepath"
@@ -73,6 +74,7 @@ type Node struct { // nolint: maligned
 	logWatcherHandlerFactories               []log.WatcherHandlerFactory
 
 	consensus ConsensusFixture
+	customGrpcSocketPath string
 }
 
 // Exit returns a channel that will close once the node shuts down.
@@ -84,8 +86,8 @@ func (n *Node) Exit() chan error {
 // SocketAddress returns the path or abstract address of the node's gRPC socket.
 func (n *Node) SocketAddress() string {
 	// When invoked from oasis-test-runner, abstract sockets should be used.
-	if n.net.cfg.UseAbstractGrpcSockets {
-		return grpc.GetAbstractSocketAddress(n.dir.String())
+	if n.customGrpcSocketPath != "" {
+		return n.customGrpcSocketPath
 	}
 
 	return internalSocketPath(n.dir)
@@ -171,6 +173,9 @@ type NodeCfg struct { // nolint: maligned
 
 	// Consensus contains configuration for the consensus backend.
 	Consensus ConsensusFixture
+
+	// CustomGrpcSocketPath contains custom socket path for internal gRPC communication.
+	CustomGrpcSocketPath string
 }
 
 // CmdAttrs is the SysProcAttr that will ensure graceful cleanup.
@@ -251,9 +256,6 @@ type NetworkCfg struct { // nolint: maligned
 	// A set of log watcher handler factories used by default on all nodes
 	// created in this test network.
 	DefaultLogWatcherHandlerFactories []log.WatcherHandlerFactory `json:"-"`
-
-	// UseAbstractGrpcSockets will use abstract sockets for internal GRPC communication.
-	UseAbstractGrpcSockets bool
 }
 
 // Config returns the network configuration.
@@ -606,6 +608,19 @@ func (net *Network) generateDeterministicNodeIdentity(dir *env.Dir, rawSeed stri
 	return nil
 }
 
+// getTempSocketPath returns unique filename for internal socket in test base dir.
+//
+// This function is used to obtain shorter socket path than the one in datadir since that one might be too long for unix
+// socket path.
+func (net *Network) getTempSocketPath() (string, error) {
+	f, err := ioutil.TempFile(env.GetRootDir().String(), "internal-*.sock")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	return f.Name(), nil
+}
+
 func (net *Network) startOasisNode(
 	node *Node,
 	subCmd []string,
@@ -627,9 +642,9 @@ func (net *Network) startOasisNode(
 			tendermintDebugAddrBookLenient().
 			tendermintDebugAllowDuplicateIP()
 	}
-	if net.cfg.UseAbstractGrpcSockets {
+	if net.cfg.CustomGrpcSocketPath != "" {
 		extraArgs = extraArgs.debugDontBlameOasis()
-		extraArgs = extraArgs.grpcDebugAbstractSocket()
+		extraArgs = extraArgs.grpcDebugGrpcSocketPath(net.cfg.CustomGrpcSocketPath)
 	}
 	if viper.IsSet(metrics.CfgMetricsAddr) {
 		extraArgs = extraArgs.appendNodeMetrics(node)