oasis-node: Use custom file-based internal unix sockets

go/oasis-node/cmd/common/grpc/grpc.go

@@ -28,8 +28,8 @@ const (
 	CfgAddress = "address"
 	// CfgWait waits for the remote address to become available.
 	CfgWait = "wait"
-	// CfgDebugGrpcAbstractSocket uses @oasis-XXXXXXXX instead of /tmp/verylongtestsuitetestnamerunid/internal.sock.
-	CfgDebugGrpcAbstractSocket = "debug.grpc.abstract_socket"
+	// CfgDebugGrpcSocketPath sets custom internal socket path.
+	CfgDebugGrpcSocketPath = "debug.grpc.socket.path"
 
 	defaultAddress      = "unix:" + LocalSocketFilename
 	LocalSocketFilename = "internal.sock"
@@ -73,8 +73,8 @@ func NewServerLocal(installWrapper bool) (*cmnGrpc.Server, error) {
 		return nil, errors.New("data directory must be set")
 	}
 	path := filepath.Join(dataDir, LocalSocketFilename)
-	if viper.GetBool(CfgDebugGrpcAbstractSocket) && flags.DebugDontBlameOasis() {
-		path = GetAbstractSocketAddress(dataDir)
+	if viper.IsSet(CfgDebugGrpcSocketPath) && flags.DebugDontBlameOasis() {
+		path = viper.GetString(CfgDebugGrpcSocketPath)
 	}
 
 	config := &cmnGrpc.ServerConfig{
@@ -121,8 +121,8 @@ func init() {
 	_ = viper.BindPFlags(ServerTCPFlags)
 	ServerTCPFlags.AddFlagSet(cmnGrpc.Flags)
 
-	ServerLocalFlags.Bool(CfgDebugGrpcAbstractSocket, false, "use abstract internal unix socket")
-	_ = ServerLocalFlags.MarkHidden(CfgDebugGrpcAbstractSocket)
+	ServerLocalFlags.String(CfgDebugGrpcSocketPath, "", "use custom internal unix socket path")
+	_ = ServerLocalFlags.MarkHidden(CfgDebugGrpcSocketPath)
 	_ = viper.BindPFlags(ServerLocalFlags)
 	ServerLocalFlags.AddFlagSet(cmnGrpc.Flags)
 

go/oasis-test-runner/cmd/root.go

@@ -422,6 +422,9 @@ func doScenario(childEnv *env.Env, sc scenario.Scenario) (err error) {
 		return
 	}
 
+	// Enable shorter per-node socket paths.
+	fixture.Network.UseCustomGrpcSocketPath = true
+
 	// Instantiate fixture if it is non-nil. Otherwise assume Init will do
 	// something on its own.
 	var net *oasis.Network
@@ -430,9 +433,6 @@ func doScenario(childEnv *env.Env, sc scenario.Scenario) (err error) {
 			err = fmt.Errorf("root: failed to instantiate fixture: %w", err)
 			return
 		}
-
-		// Use abstract sockets to avoid too long socket paths.
-		net.Config().UseAbstractGrpcSockets = true
 	}
 
 	if err = sc.Init(childEnv, net); err != nil {

go/oasis-test-runner/oasis/args.go

@@ -81,8 +81,8 @@ func (args *argBuilder) grpcLogDebug() *argBuilder {
 	return args
 }
 
-func (args *argBuilder) grpcDebugAbstractSocket() *argBuilder {
-	args.vec = append(args.vec, "--"+grpc.CfgDebugGrpcAbstractSocket)
+func (args *argBuilder) grpcDebugGrpcSocketPath(path string) *argBuilder {
+	args.vec = append(args.vec, "--"+grpc.CfgDebugGrpcSocketPath, path)
 	return args
 }
 

go/oasis-test-runner/oasis/byzantine.go

@@ -91,6 +91,10 @@ func (net *Network) NewByzantine(cfg *ByzantineCfg) (*Byzantine, error) {
 		return nil, err
 	}
 
+	sp := ""
+	if net.cfg.UseCustomGrpcSocketPath {
+		sp = net.getTempSocketPath()
+	}
 	worker := &Byzantine{
 		Node: Node{
 			Name:                                     byzantineName,
@@ -100,6 +104,7 @@ func (net *Network) NewByzantine(cfg *ByzantineCfg) (*Byzantine, error) {
 			disableDefaultLogWatcherHandlerFactories: cfg.DisableDefaultLogWatcherHandlerFactories,
 			logWatcherHandlerFactories:               cfg.LogWatcherHandlerFactories,
 			consensus:                                cfg.Consensus,
+			customGrpcSocketPath:                     sp,
 		},
 		script:          cfg.Script,
 		entity:          cfg.Entity,

go/oasis-test-runner/oasis/client.go

@@ -65,12 +65,17 @@ func (net *Network) NewClient(cfg *ClientCfg) (*Client, error) {
 		return nil, errors.Wrap(err, "oasis/client: failed to create client subdir")
 	}
 
+	sp := ""
+	if net.cfg.UseCustomGrpcSocketPath {
+		sp = net.getTempSocketPath()
+	}
 	client := &Client{
 		Node: Node{
-			Name:      clientName,
-			net:       net,
-			dir:       clientDir,
-			consensus: cfg.Consensus,
+			Name:                 clientName,
+			net:                  net,
+			dir:                  clientDir,
+			consensus:            cfg.Consensus,
+			customGrpcSocketPath: sp,
 		},
 		consensusPort: net.nextNodePort,
 	}

go/oasis-test-runner/oasis/compute.go

@@ -134,6 +134,10 @@ func (net *Network) NewCompute(cfg *ComputeCfg) (*Compute, error) {
 		cfg.RuntimeBackend = workerHost.BackendSandboxed
 	}
 
+	sp := ""
+	if net.cfg.UseCustomGrpcSocketPath {
+		sp = net.getTempSocketPath()
+	}
 	worker := &Compute{
 		Node: Node{
 			Name:                                     computeName,
@@ -144,6 +148,7 @@ func (net *Network) NewCompute(cfg *ComputeCfg) (*Compute, error) {
 			disableDefaultLogWatcherHandlerFactories: cfg.DisableDefaultLogWatcherHandlerFactories,
 			logWatcherHandlerFactories:               cfg.LogWatcherHandlerFactories,
 			consensus:                                cfg.Consensus,
+			customGrpcSocketPath:                     sp,
 		},
 		entity:         cfg.Entity,
 		runtimeBackend: cfg.RuntimeBackend,

go/oasis-test-runner/oasis/ias.go

@@ -71,11 +71,16 @@ func (net *Network) newIASProxy() (*iasProxy, error) {
 		return nil, errors.Wrap(err, "oasis/ias: failed to generate IAS proxy TLS cert")
 	}
 
+	sp := ""
+	if net.cfg.UseCustomGrpcSocketPath {
+		sp = net.getTempSocketPath()
+	}
 	net.iasProxy = &iasProxy{
 		Node: Node{
-			Name: iasName,
-			net:  net,
-			dir:  iasDir,
+			Name:                 iasName,
+			net:                  net,
+			dir:                  iasDir,
+			customGrpcSocketPath: sp,
 		},
 		useRegistry: net.cfg.IASUseRegistry,
 		grpcPort:    net.nextNodePort,

go/oasis-test-runner/oasis/keymanager.go

@@ -258,6 +258,10 @@ func (net *Network) NewKeymanager(cfg *KeymanagerCfg) (*Keymanager, error) {
 		return nil, err
 	}
 
+	sp := ""
+	if net.cfg.UseCustomGrpcSocketPath {
+		sp = net.getTempSocketPath()
+	}
 	km := &Keymanager{
 		Node: Node{
 			Name:                                     kmName,
@@ -268,6 +272,7 @@ func (net *Network) NewKeymanager(cfg *KeymanagerCfg) (*Keymanager, error) {
 			disableDefaultLogWatcherHandlerFactories: cfg.DisableDefaultLogWatcherHandlerFactories,
 			logWatcherHandlerFactories:               cfg.LogWatcherHandlerFactories,
 			consensus:                                cfg.Consensus,
+			customGrpcSocketPath:                     sp,
 		},
 		runtime:          cfg.Runtime,
 		entity:           cfg.Entity,

go/oasis-test-runner/oasis/oasis.go

@@ -5,6 +5,7 @@ import (
 	"crypto"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math"
 	"os/exec"
 	"path/filepath"
@@ -72,7 +73,8 @@ type Node struct { // nolint: maligned
 	disableDefaultLogWatcherHandlerFactories bool
 	logWatcherHandlerFactories               []log.WatcherHandlerFactory
 
-	consensus ConsensusFixture
+	consensus            ConsensusFixture
+	customGrpcSocketPath string
 }
 
 // Exit returns a channel that will close once the node shuts down.
@@ -81,11 +83,11 @@ func (n *Node) Exit() chan error {
 	return n.exitCh
 }
 
-// SocketAddress returns the path or abstract address of the node's gRPC socket.
+// SocketAddress returns the path of the node's gRPC unix socket.
 func (n *Node) SocketAddress() string {
-	// When invoked from oasis-test-runner, abstract sockets should be used.
-	if n.net.cfg.UseAbstractGrpcSockets {
-		return grpc.GetAbstractSocketAddress(n.dir.String())
+	// Return custom (shorter?) socket path, if set.
+	if n.customGrpcSocketPath != "" {
+		return n.customGrpcSocketPath
 	}
 
 	return internalSocketPath(n.dir)
@@ -252,8 +254,8 @@ type NetworkCfg struct { // nolint: maligned
 	// created in this test network.
 	DefaultLogWatcherHandlerFactories []log.WatcherHandlerFactory `json:"-"`
 
-	// UseAbstractGrpcSockets will use abstract sockets for internal GRPC communication.
-	UseAbstractGrpcSockets bool
+	// UseCustomGrpcSocketPath contains whether nodes should use internal.sock in datadir or test-runner should generate custom (shorter) ones.
+	UseCustomGrpcSocketPath bool
 }
 
 // Config returns the network configuration.
@@ -606,6 +608,19 @@ func (net *Network) generateDeterministicNodeIdentity(dir *env.Dir, rawSeed stri
 	return nil
 }
 
+// getTempSocketPath returns unique filename for internal socket in test base dir.
+//
+// This function is used to obtain shorter socket path than the one in datadir since that one might be too long for unix
+// socket path.
+func (net *Network) getTempSocketPath() string {
+	f, err := ioutil.TempFile(env.GetRootDir().String(), "internal-*.sock")
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+	return f.Name()
+}
+
 func (net *Network) startOasisNode(
 	node *Node,
 	subCmd []string,
@@ -627,9 +642,9 @@ func (net *Network) startOasisNode(
 			tendermintDebugAddrBookLenient().
 			tendermintDebugAllowDuplicateIP()
 	}
-	if net.cfg.UseAbstractGrpcSockets {
+	if net.cfg.UseCustomGrpcSocketPath {
 		extraArgs = extraArgs.debugDontBlameOasis()
-		extraArgs = extraArgs.grpcDebugAbstractSocket()
+		extraArgs = extraArgs.grpcDebugGrpcSocketPath(node.customGrpcSocketPath)
 	}
 	if viper.IsSet(metrics.CfgMetricsAddr) {
 		extraArgs = extraArgs.appendNodeMetrics(node)

go/oasis-test-runner/oasis/seed.go

@@ -63,11 +63,16 @@ func (net *Network) newSeedNode() (*seedNode, error) {
 	}
 	seedPublicKey := seedIdentity.NodeSigner.Public()
 
+	sp := ""
+	if net.cfg.UseCustomGrpcSocketPath {
+		sp = net.getTempSocketPath()
+	}
 	seedNode := &seedNode{
 		Node: Node{
-			Name: seedName,
-			net:  net,
-			dir:  seedDir,
+			Name:                 seedName,
+			net:                  net,
+			dir:                  seedDir,
+			customGrpcSocketPath: sp,
 		},
 		tmAddress:     crypto.PublicKeyToTendermint(&seedPublicKey).Address().String(),
 		consensusPort: net.nextNodePort,

go/oasis-test-runner/oasis/sentry.go

@@ -124,13 +124,18 @@ func (net *Network) NewSentry(cfg *SentryCfg) (*Sentry, error) {
 	}
 	sentryPublicKey := sentryIdentity.NodeSigner.Public()
 
+	sp := ""
+	if net.cfg.UseCustomGrpcSocketPath {
+		sp = net.getTempSocketPath()
+	}
 	sentry := &Sentry{
 		Node: Node{
 			Name:                                     sentryName,
 			net:                                      net,
 			dir:                                      sentryDir,
 			disableDefaultLogWatcherHandlerFactories: cfg.DisableDefaultLogWatcherHandlerFactories,
 			logWatcherHandlerFactories:               cfg.LogWatcherHandlerFactories,
+			customGrpcSocketPath:                     sp,
 		},
 		validatorIndices:  cfg.ValidatorIndices,
 		storageIndices:    cfg.StorageIndices,

go/oasis-test-runner/oasis/storage.go

@@ -152,6 +152,10 @@ func (net *Network) NewStorage(cfg *StorageCfg) (*Storage, error) {
 		return nil, err
 	}
 
+	sp := ""
+	if net.cfg.UseCustomGrpcSocketPath {
+		sp = net.getTempSocketPath()
+	}
 	worker := &Storage{
 		Node: Node{
 			Name:                                     storageName,
@@ -160,6 +164,7 @@ func (net *Network) NewStorage(cfg *StorageCfg) (*Storage, error) {
 			disableDefaultLogWatcherHandlerFactories: cfg.DisableDefaultLogWatcherHandlerFactories,
 			logWatcherHandlerFactories:               cfg.LogWatcherHandlerFactories,
 			consensus:                                cfg.Consensus,
+			customGrpcSocketPath:                     sp,
 		},
 		backend:                 cfg.Backend,
 		entity:                  cfg.Entity,

go/oasis-test-runner/oasis/validator.go

@@ -116,6 +116,10 @@ func (net *Network) NewValidator(cfg *ValidatorCfg) (*Validator, error) {
 		return nil, fmt.Errorf("oasis/validator: failed to create validator subdir: %w", err)
 	}
 
+	sp := ""
+	if net.cfg.UseCustomGrpcSocketPath {
+		sp = net.getTempSocketPath()
+	}
 	val := &Validator{
 		Node: Node{
 			Name:                                     valName,
@@ -126,6 +130,7 @@ func (net *Network) NewValidator(cfg *ValidatorCfg) (*Validator, error) {
 			disableDefaultLogWatcherHandlerFactories: cfg.DisableDefaultLogWatcherHandlerFactories,
 			logWatcherHandlerFactories:               cfg.LogWatcherHandlerFactories,
 			consensus:                                cfg.Consensus,
+			customGrpcSocketPath:                     sp,
 		},
 		entity:        cfg.Entity,
 		sentries:      cfg.Sentries,