Merge pull request #47 from interchainio/fix/kvstore-tx-size

Fix KVStore client transaction size

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v0.9.0
+* [\#47](https://github.com/interchainio/tm-load-test/pull/47) - Makes sure
+  that the KVStore client's `GenerateTx` method honours the preconfigured
+  transaction size. **NB: This involves a breaking API change!** Please see
+  [the client API documentation](./pkg/loadtest/README.md) for more details.
+
 ## v0.8.0
 * [\#42](https://github.com/interchainio/tm-load-test/pull/42) - Add Prometheus
   gauge for when load test is underway. This indicator exposes a customizable

Makefile

@@ -27,6 +27,9 @@ build-tm-outage-sim-server-linux:
 test:
 	go test -cover -race ./...
 
+bench:
+	go test -bench="Benchmark" -run="notests" ./...
+
 $(GOPATH)/bin/golangci-lint:
 	go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
 

README.md

@@ -11,6 +11,10 @@ ABCI application running on that network. As such, the `tm-load-test` tool comes
 with built-in support for the `kvstore` ABCI application, but you can
 [build your own clients](./pkg/loadtest/README.md) for your own apps.
 
+**NB: `tm-load-test` is currently alpha-quality software. Semantic versioning is
+not strictly adhered to prior to a v1.0 release, so breaking API changes can
+emerge with minor version releases.**
+
 ## Requirements
 In order to build and use the tools, you will need:
 

pkg/loadtest/README.md

@@ -48,7 +48,13 @@ type MyABCIAppClient struct {}
 // MyABCIAppClient implements loadtest.Client
 var _ loadtest.Client = (*MyABCIAppClient)(nil)
 
-func (f *MyABCIAppClientFactory) NewClient() (loadtest.Client, error) {
+func (f *MyABCIAppClientFactory) ValidateConfig(cfg loadtest.Config) error {
+    // Do any checks here that you need to ensure that the load test 
+    // configuration is compatible with your client.
+    return nil
+}
+
+func (f *MyABCIAppClientFactory) NewClient(cfg loadtest.Config) (loadtest.Client, error) {
     return &MyABCIAppClient{}, nil
 }
 

pkg/loadtest/cli.go

@@ -12,7 +12,7 @@ import (
 )
 
 // CLIVersion must be manually updated as new versions are released.
-const CLIVersion = "v0.8.0"
+const CLIVersion = "v0.9.0"
 
 // cliVersionCommitID must be set through linker settings. See
 // https://stackoverflow.com/a/11355611/1156132 for details.

pkg/loadtest/client.go

@@ -4,9 +4,13 @@ import "fmt"
 
 // ClientFactory produces load testing clients.
 type ClientFactory interface {
+	// ValidateConfig must check whether the given configuration is valid for
+	// our specific client factory.
+	ValidateConfig(cfg Config) error
+
 	// NewClient must instantiate a new load testing client, or produce an error
 	// if that process fails.
-	NewClient() (Client, error)
+	NewClient(cfg Config) (Client, error)
 }
 
 // Client generates transactions to be sent to a specific endpoint.

pkg/loadtest/client_kvstore.go

@@ -1,14 +1,56 @@
 package loadtest
 
-import "github.com/tendermint/tendermint/libs/common"
+import (
+	"fmt"
+
+	"github.com/tendermint/tendermint/libs/common"
+)
+
+// The Tendermint common.RandStr method can effectively generate human-readable
+// (alphanumeric) strings from a set of 62 characters. We aim here with the
+// KVStore client to generate unique client IDs as well as totally unique keys
+// for all transactions. Values are not so important.
+const KVStoreClientIDLen int = 5 // Allows for 6,471,002 random client IDs (62C5)
+const kvstoreMinValueLen int = 1 // We at least need 1 character in a key/value pair's value.
+
+// This is a map of nCr where n=62 and r varies from 0 through 15. It gives the
+// maximum number of unique transaction IDs that can be accommodated with a
+// given key suffix length.
+var kvstoreMaxTxsByKeySuffixLen = []uint64{
+	0,              // 0
+	62,             // 1
+	1891,           // 2
+	37820,          // 3
+	557845,         // 4
+	6471002,        // 5
+	61474519,       // 6
+	491796152,      // 7
+	3381098545,     // 8
+	20286591270,    // 9
+	107518933731,   // 10
+	508271323092,   // 11
+	2160153123141,  // 12
+	8308281242850,  // 13
+	29078984349975, // 14
+	93052749919920, // 15
+}
 
 // KVStoreClientFactory creates load testing clients to interact with the
 // built-in Tendermint kvstore ABCI application.
 type KVStoreClientFactory struct{}
 
 // KVStoreClient generates arbitrary transactions (random key=value pairs) to
-// be sent to the kvstore ABCI application.
-type KVStoreClient struct{}
+// be sent to the kvstore ABCI application. The keys are structured as follows:
+//
+// `[client_id][tx_id]=[tx_id]`
+//
+// where each value (`client_id` and `tx_id`) is padded with 0s to meet the
+// transaction size requirement.
+type KVStoreClient struct {
+	keyPrefix    []byte // Contains the client ID
+	keySuffixLen int
+	valueLen     int
+}
 
 var _ ClientFactory = (*KVStoreClientFactory)(nil)
 var _ Client = (*KVStoreClient)(nil)
@@ -23,12 +65,54 @@ func NewKVStoreClientFactory() *KVStoreClientFactory {
 	return &KVStoreClientFactory{}
 }
 
-func (f *KVStoreClientFactory) NewClient() (Client, error) {
-	return &KVStoreClient{}, nil
+func (f *KVStoreClientFactory) ValidateConfig(cfg Config) error {
+	maxTxsPerEndpoint := cfg.MaxTxsPerEndpoint()
+	if maxTxsPerEndpoint < 1 {
+		return fmt.Errorf("cannot calculate an appropriate maximum number of transactions per endpoint (got %d)", maxTxsPerEndpoint)
+	}
+	minKeySuffixLen, err := requiredKVStoreSuffixLen(maxTxsPerEndpoint)
+	if err != nil {
+		return err
+	}
+	// "[client_id][random_suffix]=[value]"
+	minTxSize := KVStoreClientIDLen + minKeySuffixLen + 1 + kvstoreMinValueLen
+	if cfg.Size < minTxSize {
+		return fmt.Errorf("transaction size %d is too small for given parameters (should be at least %d bytes)", cfg.Size, minTxSize)
+	}
+	return nil
+}
+
+func (f *KVStoreClientFactory) NewClient(cfg Config) (Client, error) {
+	keyPrefix := []byte(common.RandStr(KVStoreClientIDLen))
+	keySuffixLen, err := requiredKVStoreSuffixLen(cfg.MaxTxsPerEndpoint())
+	if err != nil {
+		return nil, err
+	}
+	keyLen := len(keyPrefix) + keySuffixLen
+	// value length = key length - 1 (to cater for "=" symbol)
+	valueLen := cfg.Size - keyLen - 1
+	return &KVStoreClient{
+		keyPrefix:    keyPrefix,
+		keySuffixLen: keySuffixLen,
+		valueLen:     valueLen,
+	}, nil
+}
+
+func requiredKVStoreSuffixLen(maxTxCount uint64) (int, error) {
+	for l, maxTxs := range kvstoreMaxTxsByKeySuffixLen {
+		if maxTxCount < maxTxs {
+			if l+1 > len(kvstoreMaxTxsByKeySuffixLen) {
+				return -1, fmt.Errorf("cannot cater for maximum tx count of %d (too many unique transactions, suffix length %d)", maxTxCount, l+1)
+			}
+			// we use l+1 to minimize collision probability
+			return l + 1, nil
+		}
+	}
+	return -1, fmt.Errorf("cannot cater for maximum tx count of %d (too many unique transactions)", maxTxCount)
 }
 
 func (c *KVStoreClient) GenerateTx() ([]byte, error) {
-	k := []byte(common.RandStr(8))
-	v := []byte(common.RandStr(8))
+	k := append(c.keyPrefix, []byte(common.RandStr(c.keySuffixLen))...)
+	v := []byte(common.RandStr(c.valueLen))
 	return append(k, append([]byte("="), v...)...), nil
 }

pkg/loadtest/client_kvstore_test.go

@@ -0,0 +1,125 @@
+package loadtest_test
+
+import (
+	"testing"
+
+	"github.com/interchainio/tm-load-test/pkg/loadtest"
+)
+
+func TestKVStoreClientFactoryConfigValidation(t *testing.T) {
+	testCases := []struct {
+		config loadtest.Config
+		err    bool
+	}{
+		{loadtest.Config{Size: 1, Rate: 1000, Time: 1000, Count: -1}, true},  // invalid tx size
+		{loadtest.Config{Size: 10, Rate: 1000, Time: 1000, Count: -1}, true}, // tx size is too small
+
+		{loadtest.Config{Size: 14, Rate: 1000, Time: 10000, Count: -1}, false}, // just right for parameters
+
+		{loadtest.Config{Size: 20, Rate: 1000, Time: 10, Count: -1}, false},   // 10k txs @ 20 bytes each
+		{loadtest.Config{Size: 20, Rate: 1000, Time: 100, Count: -1}, false},  // 100k txs @ 20 bytes each
+		{loadtest.Config{Size: 20, Rate: 1000, Time: 1000, Count: -1}, false}, // 1m txs @ 20 bytes each
+
+		{loadtest.Config{Size: 100, Rate: 1000, Time: 10, Count: -1}, false},
+		{loadtest.Config{Size: 100, Rate: 1000, Time: 100000, Count: -1}, false}, // 100m txs @ 100 bytes each
+
+		{loadtest.Config{Size: 250, Rate: 1000, Time: 10, Count: -1}, false},
+
+		{loadtest.Config{Size: 10240, Rate: 1000, Time: 10, Count: -1}, false},     // 10k txs @ 10kB each
+		{loadtest.Config{Size: 10240, Rate: 1000, Time: 100, Count: -1}, false},    // 100k txs @ 10kB each
+		{loadtest.Config{Size: 10240, Rate: 1000, Time: 1000, Count: -1}, false},   // 1m txs @ 10kB each
+		{loadtest.Config{Size: 10240, Rate: 1000, Time: 10000, Count: -1}, false},  // 10m txs @ 10kB each
+		{loadtest.Config{Size: 10240, Rate: 1000, Time: 100000, Count: -1}, false}, // 100m txs @ 10kB each
+	}
+	factory := loadtest.NewKVStoreClientFactory()
+	for i, tc := range testCases {
+		err := factory.ValidateConfig(tc.config)
+		// if we were supposed to get an error
+		if tc.err && err == nil {
+			t.Errorf("Expected an error from test case %d, but got nil", i)
+		} else if !tc.err && err != nil {
+			t.Errorf("Expected no error from test case %d, but got: %v", i, err)
+		}
+	}
+}
+
+func benchmarkKVStoreClient_GenerateTx(b *testing.B, cfg loadtest.Config) {
+	cfg.Count = b.N
+	factory := loadtest.NewKVStoreClientFactory()
+	if err := factory.ValidateConfig(cfg); err != nil {
+		b.Errorf("Unexpected error from KVStoreClientFactory.ValidateConfig(): %v", err)
+	}
+	client, err := factory.NewClient(cfg)
+	if err != nil {
+		b.Errorf("Unexpected error from KVStoreClientFactory.NewClient(): %v", err)
+	}
+	for n := 0; n < b.N; n++ {
+		_, _ = client.GenerateTx()
+	}
+}
+
+func BenchmarkKVStoreClient_GenerateTx_32b(b *testing.B) {
+	benchmarkKVStoreClient_GenerateTx(b, loadtest.Config{Size: 32})
+}
+
+func BenchmarkKVStoreClient_GenerateTx_64b(b *testing.B) {
+	benchmarkKVStoreClient_GenerateTx(b, loadtest.Config{Size: 64})
+}
+
+func BenchmarkKVStoreClient_GenerateTx_128b(b *testing.B) {
+	benchmarkKVStoreClient_GenerateTx(b, loadtest.Config{Size: 128})
+}
+
+func BenchmarkKVStoreClient_GenerateTx_256b(b *testing.B) {
+	benchmarkKVStoreClient_GenerateTx(b, loadtest.Config{Size: 256})
+}
+
+func BenchmarkKVStoreClient_GenerateTx_512b(b *testing.B) {
+	benchmarkKVStoreClient_GenerateTx(b, loadtest.Config{Size: 512})
+}
+
+func BenchmarkKVStoreClient_GenerateTx_1kB(b *testing.B) {
+	benchmarkKVStoreClient_GenerateTx(b, loadtest.Config{Size: 1024})
+}
+
+func BenchmarkKVStoreClient_GenerateTx_10kB(b *testing.B) {
+	benchmarkKVStoreClient_GenerateTx(b, loadtest.Config{Size: 10240})
+}
+
+func BenchmarkKVStoreClient_GenerateTx_100kB(b *testing.B) {
+	benchmarkKVStoreClient_GenerateTx(b, loadtest.Config{Size: 102400})
+}
+
+func TestKVStoreClient(t *testing.T) {
+	testCases := []struct{
+		config loadtest.Config
+		clientCount int
+	}{
+		{loadtest.Config{Size: 32, Count: 1000}, 5},
+		{loadtest.Config{Size: 64, Count: 1000}, 5},
+		{loadtest.Config{Size: 128, Count: 1000}, 5},
+		{loadtest.Config{Size: 256, Count: 1000}, 5},
+		{loadtest.Config{Size: 10240, Count: 1000}, 5},
+	}
+	factory := loadtest.NewKVStoreClientFactory()
+	for i, tc := range testCases {
+		err := factory.ValidateConfig(tc.config)
+		if err != nil {
+			t.Errorf("Expected config from test case %d to validate, but failed: %v", i, err)
+		}
+
+		for c := 0; c < tc.clientCount; c++ {
+			client, err := factory.NewClient(tc.config)
+			if err != nil {
+				t.Errorf("Did not expect error in test case %d from factory.NewClient: %v", i, err)
+			}
+			tx, err := client.GenerateTx()
+			if err != nil {
+				t.Errorf("Did not expect error in test case %d from client %d's GenerateTx: %v", i, c, err)
+			}
+			if len(tx) != tc.config.Size {
+				t.Errorf("Expected transaction from client %d in test case %d to be %d bytes, but was %d bytes", c, i, tc.config.Size, len(tx))
+			}
+		}
+	}
+}

pkg/loadtest/config.go

@@ -64,9 +64,14 @@ func (c Config) Validate() error {
 	if len(c.ClientFactory) == 0 {
 		return fmt.Errorf("client factory name must be specified")
 	}
-	if _, exists := clientFactories[c.ClientFactory]; !exists {
+	factory, factoryExists := clientFactories[c.ClientFactory]
+	if !factoryExists {
 		return fmt.Errorf("client factory \"%s\" does not exist", c.ClientFactory)
 	}
+	// client factory-specific configuration validation
+	if err := factory.ValidateConfig(c); err != nil {
+		return fmt.Errorf("invalid configuration for client factory \"%s\": %v", c.ClientFactory, err)
+	}
 	if c.Connections < 1 {
 		return fmt.Errorf("expected connections to be >= 1, but was %d", c.Connections)
 	}
@@ -79,9 +84,6 @@ func (c Config) Validate() error {
 	if c.Rate < 1 {
 		return fmt.Errorf("expected transaction rate to be >= 1, but was %d", c.Rate)
 	}
-	if c.Size < 40 {
-		return fmt.Errorf("expected transaction size to be >= 40 bytes, but was %d", c.Size)
-	}
 	if c.Count < 1 && c.Count != -1 {
 		return fmt.Errorf("expected max transaction count to either be -1 or >= 1, but was %d", c.Count)
 	}
@@ -109,6 +111,15 @@ func (c Config) Validate() error {
 	return nil
 }
 
+// MaxTxsPerEndpoint estimates the maximum number of transactions that this
+// configuration would generate for a single endpoint.
+func (c Config) MaxTxsPerEndpoint() uint64 {
+	if c.Count > -1 {
+		return uint64(c.Count)
+	}
+	return uint64(c.Rate) * uint64(c.Time)
+}
+
 func (c MasterConfig) ToJSON() string {
 	b, err := json.Marshal(c)
 	if err != nil {