e2e: allow running of single node using the e2e app (backport)

CHANGELOG_PENDING.md

@@ -20,6 +20,8 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 
 ### FEATURES
 
+- [\#6982](https://github.com/tendermint/tendermint/pull/6982) tendermint binary has built-in suppport for running the end to end application (with state sync support) (@cmwaters).
+
 ### IMPROVEMENTS
 
 ### BUG FIXES

cmd/tendermint/commands/run_node.go

@@ -46,9 +46,7 @@ func AddNodeFlags(cmd *cobra.Command) {
 		"proxy_app",
 		config.ProxyApp,
 		"proxy app address, or one of: 'kvstore',"+
-			" 'persistent_kvstore',"+
-			" 'counter',"+
-			" 'counter_serial' or 'noop' for local testing.")
+			" 'persistent_kvstore', 'counter', 'e2e' or 'noop' for local testing.")
 	cmd.Flags().String("abci", config.ABCI, "specify abci transport (socket | grpc)")
 
 	// rpc flags

proxy/client.go

@@ -8,6 +8,7 @@ import (
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	"github.com/tendermint/tendermint/abci/types"
 	tmsync "github.com/tendermint/tendermint/libs/sync"
+	e2e "github.com/tendermint/tendermint/test/e2e/app"
 )
 
 // ClientCreator creates new ABCI clients.
@@ -79,6 +80,12 @@ func DefaultClientCreator(addr, transport, dbDir string) ClientCreator {
 		return NewLocalClientCreator(kvstore.NewApplication())
 	case "persistent_kvstore":
 		return NewLocalClientCreator(kvstore.NewPersistentKVStoreApplication(dbDir))
+	case "e2e":
+		app, err := e2e.NewApplication(e2e.DefaultConfig(dbDir))
+		if err != nil {
+			panic(err)
+		}
+		return NewLocalClientCreator(app)
 	case "noop":
 		return NewLocalClientCreator(types.NewBaseApplication())
 	default:

test/e2e/Makefile

@@ -6,8 +6,8 @@ docker:
 # We need to build support for database backends into the app in
 # order to build a binary with a Tendermint node in it (for built-in
 # ABCI testing).
-app:
-	go build -o build/app -tags badgerdb,boltdb,cleveldb,rocksdb ./app
+node:
+	go build -o build/node -tags badgerdb,boltdb,cleveldb,rocksdb ./node
 
 # To be used primarily by the e2e docker instance. If you want to produce this binary
 # elsewhere, then run go build in the maverick directory. 
@@ -20,4 +20,4 @@ generator:
 runner:
 	go build -o build/runner ./runner
 
-.PHONY: all app docker generator maverick runner 
+.PHONY: all node docker generator maverick runner 

test/e2e/README.md

@@ -122,10 +122,42 @@ Docker does not enable IPv6 by default. To do so, enter the following in
 }
 ```
 
-## Benchmarking testnets
+## Benchmarking Testnets
 
 It is also possible to run a simple benchmark on a testnet. This is done through the `benchmark` command. This manages the entire process: setting up the environment, starting the test net, waiting for a considerable amount of blocks to be used (currently 100), and then returning the following metrics from the sample of the blockchain:
 
 - Average time to produce a block
 - Standard deviation of producing a block
 - Minimum and maximum time to produce a block
+
+## Running Individual Nodes
+
+The E2E test harness is designed to run several nodes of varying configurations within docker. It is also possible to run a single node in the case of running larger, geographically-dispersed testnets. To run a single node you can either run:
+
+**Built-in**
+
+```bash
+make node
+tendermint init validator
+TMHOME=$HOME/.tendermint ./build/node ./node/built-in.toml
+```
+
+To make things simpler the e2e application can also be run in the tendermint binary
+by running
+
+```bash
+tendermint start --proxy-app e2e
+```
+
+However this won't offer the same level of configurability of the application.
+
+**Socket**
+
+```bash
+make node
+tendermint init validator
+tendermint start
+./build/node ./node.socket.toml
+```
+
+Check `node/config.go` to see how the settings of the test application can be tweaked.

test/e2e/app/app.go

@@ -1,4 +1,4 @@
-package main
+package app
 
 import (
 	"bytes"
@@ -28,6 +28,55 @@ type Application struct {
 	restoreChunks   [][]byte
 }
 
+// Config allows for the setting of high level parameters for running the e2e Application
+// KeyType and ValidatorUpdates must be the same for all nodes running the same application.
+type Config struct {
+	// The directory with which state.json will be persisted in. Usually $HOME/.tendermint/data
+	Dir string `toml:"dir"`
+
+	// SnapshotInterval specifies the height interval at which the application
+	// will take state sync snapshots. Defaults to 0 (disabled).
+	SnapshotInterval uint64 `toml:"snapshot_interval"`
+
+	// RetainBlocks specifies the number of recent blocks to retain. Defaults to
+	// 0, which retains all blocks. Must be greater that PersistInterval,
+	// SnapshotInterval and EvidenceAgeHeight.
+	RetainBlocks uint64 `toml:"retain_blocks"`
+
+	// KeyType sets the curve that will be used by validators.
+	// Options are ed25519 & secp256k1
+	KeyType string `toml:"key_type"`
+
+	// PersistInterval specifies the height interval at which the application
+	// will persist state to disk. Defaults to 1 (every height), setting this to
+	// 0 disables state persistence.
+	PersistInterval uint64 `toml:"persist_interval"`
+
+	// ValidatorUpdates is a map of heights to validator names and their power,
+	// and will be returned by the ABCI application. For example, the following
+	// changes the power of validator01 and validator02 at height 1000:
+	//
+	// [validator_update.1000]
+	// validator01 = 20
+	// validator02 = 10
+	//
+	// Specifying height 0 returns the validator update during InitChain. The
+	// application returns the validator updates as-is, i.e. removing a
+	// validator must be done by returning it with power 0, and any validators
+	// not specified are not changed.
+	//
+	// height <-> pubkey <-> voting power
+	ValidatorUpdates map[string]map[string]uint8 `toml:"validator_update"`
+}
+
+func DefaultConfig(dir string) *Config {
+	return &Config{
+		PersistInterval:  1,
+		SnapshotInterval: 100,
+		Dir:              dir,
+	}
+}
+
 // NewApplication creates the application.
 func NewApplication(cfg *Config) (*Application, error) {
 	state, err := NewState(filepath.Join(cfg.Dir, "state.json"), cfg.PersistInterval)

test/e2e/app/snapshots.go

@@ -1,5 +1,5 @@
 // nolint: gosec
-package main
+package app
 
 import (
 	"encoding/json"

test/e2e/app/state.go

@@ -1,5 +1,5 @@
 //nolint: gosec
-package main
+package app
 
 import (
 	"crypto/sha256"

test/e2e/docker/Dockerfile

@@ -19,7 +19,7 @@ COPY . .
 RUN make build && cp build/tendermint /usr/bin/tendermint
 COPY test/e2e/docker/entrypoint* /usr/bin/
 RUN cd test/e2e && make maverick && cp build/maverick /usr/bin/maverick
-RUN cd test/e2e && make app && cp build/app /usr/bin/app
+RUN cd test/e2e && make node && cp build/node /usr/bin/app
 
 # Set up runtime directory. We don't use a separate runtime image since we need
 # e.g. leveldb and rocksdb which are already installed in the build image.

test/e2e/node/built-in.toml

@@ -0,0 +1,4 @@
+snapshot_interval = 100
+persist_interval = 1
+chain_id = "test-chain"
+protocol = "builtin"

test/e2e/app/config.go → test/e2e/node/config.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 
 	"github.com/BurntSushi/toml"
+
+	"github.com/tendermint/tendermint/test/e2e/app"
 )
 
 // Config is the application configuration.
@@ -26,6 +28,18 @@ type Config struct {
 	KeyType          string                      `toml:"key_type"`
 }
 
+// App extracts out the application specific configuration parameters
+func (cfg *Config) App() *app.Config {
+	return &app.Config{
+		Dir:              cfg.Dir,
+		SnapshotInterval: cfg.SnapshotInterval,
+		RetainBlocks:     cfg.RetainBlocks,
+		KeyType:          cfg.KeyType,
+		ValidatorUpdates: cfg.ValidatorUpdates,
+		PersistInterval:  cfg.PersistInterval,
+	}
+}
+
 // LoadConfig loads the configuration from disk.
 func LoadConfig(file string) (*Config, error) {
 	cfg := &Config{

test/e2e/app/main.go → test/e2e/node/main.go

@@ -28,6 +28,7 @@ import (
 	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
 	rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"
+	"github.com/tendermint/tendermint/test/e2e/app"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 	mcs "github.com/tendermint/tendermint/test/maverick/consensus"
 	maverick "github.com/tendermint/tendermint/test/maverick/node"
@@ -98,7 +99,7 @@ func run(configFile string) error {
 
 // startApp starts the application server, listening for connections from Tendermint.
 func startApp(cfg *Config) error {
-	app, err := NewApplication(cfg)
+	app, err := app.NewApplication(cfg.App())
 	if err != nil {
 		return err
 	}
@@ -119,7 +120,7 @@ func startApp(cfg *Config) error {
 //
 // FIXME There is no way to simply load the configuration from a file, so we need to pull in Viper.
 func startNode(cfg *Config) error {
-	app, err := NewApplication(cfg)
+	app, err := app.NewApplication(cfg.App())
 	if err != nil {
 		return err
 	}
@@ -205,7 +206,7 @@ func startLightClient(cfg *Config) error {
 // startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint
 // configuration is in $TMHOME/config/tendermint.toml.
 func startMaverick(cfg *Config) error {
-	app, err := NewApplication(cfg)
+	app, err := app.NewApplication(cfg.App())
 	if err != nil {
 		return err
 	}

test/e2e/node/socket.toml

@@ -0,0 +1,5 @@
+snapshot_interval = 100
+persist_interval = 1
+chain_id = "test-chain"
+protocol = "socket"
+listen = "tcp://127.0.0.1:26658"