Introduce system tests - initial version heavily inspired by wasmd

.github/workflows/test.yml

@@ -144,6 +144,32 @@ jobs:
           name: "${{ github.sha }}-e2e-coverage"
           path: ./tests/e2e-profile.out
 
+  test-system:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+          check-latest: true
+          cache: true
+          cache-dependency-path: go.sum
+      - uses: technote-space/get-diff-action@v6.1.2
+        id: git_diff
+        with:
+          PATTERNS: |
+            **/*.go
+            go.mod
+            go.sum
+            **/go.mod
+            **/go.sum
+            **/Makefile
+            Makefile
+      - name: system tests v1
+        if: env.GIT_DIFF
+        run: |
+          COSMOS_BUILD_OPTIONS=legacy make test-system
+
   repo-analysis:
     runs-on: ubuntu-latest
     needs: [tests, test-integration, test-e2e]

Makefile

@@ -214,6 +214,11 @@ test-integration-cov:
 #? test-all: Run all test
 test-all: test-unit test-e2e test-integration test-ledger-mock test-race
 
+.PHONY: test-system
+test-system: install
+	$(MAKE) -C systemtests test
+
+
 TEST_PACKAGES=./...
 TEST_TARGETS := test-unit test-unit-amino test-unit-proto test-ledger-mock test-race test-ledger test-race
 

systemtests/.gitignore

@@ -0,0 +1,2 @@
+/testnet
+/binaries

systemtests/Makefile

@@ -0,0 +1,15 @@
+#!/usr/bin/make -f
+
+WAIT_TIME ?= 45s
+
+all: test
+
+test:
+	go test -mod=readonly -failfast -tags='system_test' ./... --wait-time=$(WAIT_TIME) --verbose
+
+format:
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gofumpt -w
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs misspell -w
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "./tests/system/vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gci write --skip-generated -s standard -s default -s "prefix(cosmossdk.io)" -s "prefix(github.com/cosmos/cosmos-sdk)" --custom-order
+
+.PHONY: all test format

systemtests/README.md

@@ -0,0 +1,50 @@
+# Testing
+
+Test framework for system tests. 
+Starts and interacts with a (multi node) blockchain in Go.
+Supports
+* CLI
+* Servers
+* Events
+* RPC
+
+Uses:
+* testify
+* gjson
+* sjson
+Server and client side are executed on the host machine
+
+## Developer
+### Test strategy
+System tests cover the full stack via cli and a running (multi node) network. They are more expensive (in terms of time/ cpu) 
+to run compared to unit or integration tests. 
+Therefore, we focus on the **critical path** and do not cover every condition.
+
+### Execute a single test
+
+```sh
+go test -tags system_test -count=1 -v . --run TestStakeUnstake  -verbose
+```
+
+Test cli parameters
+
+* `-verbose` verbose output
+* `-wait-time` duration - time to wait for chain events (default 30s)
+* `-nodes-count` int - number of nodes in the cluster (default 4)
+
+# Port ranges
+With *n* nodes:
+* `26657` - `26657+n` - RPC
+* `1317` - `1317+n` - API
+* `9090` - `9090+n` - GRPC
+* `16656` - `16656+n` - P2P
+
+For example Node *3* listens on `26660` for RPC calls
+
+## Resources
+
+* [gjson query syntax](https://github.com/tidwall/gjson#path-syntax)
+
+## Disclaimer
+
+This is based on the system test framework in [wasmd](https://github.com/CosmWasm/wasmd) built by Confio.