Introduce system tests - initial version heavily inspired by wasmd

.github/workflows/gosec.yml

@@ -29,7 +29,7 @@ jobs:
         uses: securego/gosec@master
         with:
           # we let the report trigger content trigger a failure using the GitHub Security features.
-          args: "-exclude=G101,G107 -no-fail -fmt sarif -out results.sarif ./..."
+          args: "-exclude=G101,G107 -exclude-dir=systemtests -no-fail -fmt sarif -out results.sarif ./..."
 
       - name: Upload SARIF file
         uses: github/codeql-action/upload-sarif@v3

.github/workflows/test.yml

@@ -144,6 +144,57 @@ jobs:
           name: "${{ github.sha }}-e2e-coverage"
           path: ./tests/e2e-profile.out
 
+  test-system:
+    needs: [tests, test-integration, test-e2e]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            simapp/go.sum
+            systemtest/go.sum
+      - uses: technote-space/get-diff-action@v6.1.2
+        id: git_diff
+        with:
+          PATTERNS: |
+            **/*.go
+            go.mod
+            go.sum
+            **/go.mod
+            **/go.sum
+            **/Makefile
+            Makefile
+      - name: Install musl lib for simd (docker) binary
+        if: env.GIT_DIFF
+        run: |
+          sudo apt-get install -y musl
+      - name: system tests v1
+        if: env.GIT_DIFF
+        run: |
+          COSMOS_BUILD_OPTIONS=legacy make test-system
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: "testnet-setup"
+          path: ./systemtests/testnet/
+          retention-days: 3
+      #      - name: system tests v2
+      #        if: env.GIT_DIFF
+      #        run: |
+      #          make test-system
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: "testnet-setup"
+          path: ./systemtests/testnet/
+          retention-days: 3
+
   repo-analysis:
     runs-on: ubuntu-latest
     needs: [tests, test-integration, test-e2e]
@@ -1222,7 +1273,7 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         with:
           projectBaseDir: x/mint/
-  
+
   test-x-epochs:
     runs-on: ubuntu-latest
     steps:

CHANGELOG.md

@@ -41,7 +41,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 Every module contains its own CHANGELOG.md. Please refer to the module you are interested in.
 
 ### Features
-
+* (tests) [#20013](https://github.com/cosmos/cosmos-sdk/pull/20013) Introduce system tests to run multi node local testnet in CI
 * (runtime) [#19953](https://github.com/cosmos/cosmos-sdk/pull/19953) Implement `core/transaction.Service` in runtime.
 * (client) [#19905](https://github.com/cosmos/cosmos-sdk/pull/19905) Add grpc client config to `client.toml`.
 * (runtime) [#19571](https://github.com/cosmos/cosmos-sdk/pull/19571) Implement `core/router.Service` in runtime. This service is present in all modules (when using depinject).

Makefile

@@ -13,6 +13,10 @@ HTTPS_GIT := https://github.com/cosmos/cosmos-sdk.git
 DOCKER := $(shell which docker)
 PROJECT_NAME = $(shell git remote get-url origin | xargs basename -s .git)
 
+ifeq ($(findstring .,$(VERSION)),)
+	VERSION := 0.0.0
+endif
+
 # process build tags
 build_tags = netgo
 ifeq ($(LEDGER_ENABLED),true)
@@ -214,6 +218,13 @@ test-integration-cov:
 #? test-all: Run all test
 test-all: test-unit test-e2e test-integration test-ledger-mock test-race
 
+.PHONY: test-system
+test-system: build
+	mkdir -p ./systemtests/binaries/
+	cp $(BUILDDIR)/simd ./systemtests/binaries/
+	$(MAKE) -C systemtests test
+
+
 TEST_PACKAGES=./...
 TEST_TARGETS := test-unit test-unit-amino test-unit-proto test-ledger-mock test-race test-ledger test-race
 

systemtests/.gitignore

@@ -0,0 +1,2 @@
+/testnet
+/binaries

systemtests/Makefile

@@ -0,0 +1,15 @@
+#!/usr/bin/make -f
+
+WAIT_TIME ?= 45s
+
+all: test
+
+test:
+	go test -mod=readonly -failfast -tags='system_test' ./... --wait-time=$(WAIT_TIME) --verbose
+
+format:
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gofumpt -w
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs misspell -w
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "./tests/system/vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gci write --skip-generated -s standard -s default -s "prefix(cosmossdk.io)" -s "prefix(github.com/cosmos/cosmos-sdk)" --custom-order
+
+.PHONY: all test format

systemtests/README.md

@@ -0,0 +1,50 @@
+# Testing
+
+Test framework for system tests. 
+Starts and interacts with a (multi node) blockchain in Go.
+Supports
+* CLI
+* Servers
+* Events
+* RPC
+
+Uses:
+* testify
+* gjson
+* sjson
+Server and client side are executed on the host machine
+
+## Developer
+### Test strategy
+System tests cover the full stack via cli and a running (multi node) network. They are more expensive (in terms of time/ cpu) 
+to run compared to unit or integration tests. 
+Therefore, we focus on the **critical path** and do not cover every condition.
+
+### Execute a single test
+
+```sh
+go test -tags system_test -count=1 -v . --run TestStakeUnstake  -verbose
+```
+
+Test cli parameters
+
+* `-verbose` verbose output
+* `-wait-time` duration - time to wait for chain events (default 30s)
+* `-nodes-count` int - number of nodes in the cluster (default 4)
+
+# Port ranges
+With *n* nodes:
+* `26657` - `26657+n` - RPC
+* `1317` - `1317+n` - API
+* `9090` - `9090+n` - GRPC
+* `16656` - `16656+n` - P2P
+
+For example Node *3* listens on `26660` for RPC calls
+
+## Resources
+
+* [gjson query syntax](https://github.com/tidwall/gjson#path-syntax)
+
+## Disclaimer
+
+This is based on the system test framework in [wasmd](https://github.com/CosmWasm/wasmd) built by Confio.