[Bug] concurrent map read and map write

[yun-yeo]

Now Tendermint support RWMutex, so there can be multiple reader simultaneously.
tendermint/tendermint#6306

But this uncontrolled map access leads fatal error
This line could lead fatal error

fatal error: concurrent map read and map write

iavl/mutable_tree.go

Lines 61 to 69 in ac52606

	func (tree *MutableTree) VersionExists(version int64) bool {
	if tree.allRootLoaded {
	return tree.versions[version]
	}
	has, ok := tree.versions[version]
	if ok {
	return has
	}

Error Messages

goroutine 340 [running]:
runtime.throw(0x5b97c6e, 0x21)
	runtime/panic.go:1117 +0x72 fp=0xc00017d568 sp=0xc00017d538 pc=0x4039352
runtime.mapaccess1_fast64(0x58e58a0, 0xc000db33e0, 0x97, 0x8)
	runtime/map_fast64.go:21 +0x198 fp=0xc00017d590 sp=0xc00017d568 pc=0x4013e58
github.com/cosmos/iavl.(*MutableTree).VersionExists(0xc000db2f60, 0x97, 0xc000db49b0)
	github.com/cosmos/iavl@v0.16.0/mutable_tree.go:63 +0xfa fp=0xc00017d5d0 sp=0xc00017d590 pc=0x4ade83a
github.com/cosmos/cosmos-sdk/store/iavl.(*Store).VersionExists(...)
	github.com/cosmos/cosmos-sdk@v0.43.0/store/iavl/store.go:145
github.com/cosmos/cosmos-sdk/store/iavl.(*Store).GetImmutable(0xc00011dc40, 0x97, 0xc000db49b0, 0x62d7af0, 0xc00011dc40)
	github.com/cosmos/cosmos-sdk@v0.43.0/store/iavl/store.go:93 +0x44 fp=0xc00017d618 sp=0xc00017d5d0 pc=0x4b03724
github.com/cosmos/cosmos-sdk/store/rootmulti.(*Store).CacheMultiStoreWithVersion(0xc000620280, 0x97, 0x0, 0xc0015362d0, 0x627d360, 0xc0001a7c00)
	github.com/cosmos/cosmos-sdk@v0.43.0/store/rootmulti/store.go:458 +0x28c fp=0xc00017d948 sp=0xc00017d618 pc=0x4b0e44c
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).createQueryContext(0xc0001a4000, 0x97, 0xe9b5dba5b5c0fb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	github.com/cosmos/cosmos-sdk@v0.43.0/baseapp/abci.go:637 +0xdd fp=0xc00017e2f0 sp=0xc00017d948 pc=0x4be365d
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).handleQueryGRPC(0xc0001a4000, 0xc000124f00, 0xc004da57a0, 0x8e, 0x120, 0xc004dee2d0, 0x27, 0x97, 0x0, 0x0, ...)
	github.com/cosmos/cosmos-sdk@v0.43.0/baseapp/abci.go:571 +0xa8 fp=0xc00017ec40 sp=0xc00017e2f0 pc=0x4be2ee8
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).Query(0xc0001a4000, 0xc004da57a0, 0x8e, 0x120, 0xc004dee2d0, 0x27, 0x97, 0x0, 0x0, 0x0, ...)
	github.com/cosmos/cosmos-sdk@v0.43.0/baseapp/abci.go:420 +0x96f fp=0xc00017ef28 sp=0xc00017ec40 pc=0x4be17cf
github.com/tendermint/tendermint/abci/client.(*localClient).QuerySync(0xc0001bb6e0, 0xc004da57a0, 0x8e, 0x120, 0xc004dee2d0, 0x27, 0x0, 0x0, 0x0, 0x0, ...)
	github.com/tendermint/tendermint@v0.34.12/abci/client/local_client.go:253 +0x10c fp=0xc00017f098 sp=0xc00017ef28 pc=0x49c4d2c
github.com/tendermint/tendermint/proxy.(*appConnQuery).QuerySync(0xc0012e39d0, 0xc004da57a0, 0x8e, 0x120, 0xc004dee2d0, 0x27, 0x0, 0x0, 0x203001, 0x203001, ...)
	github.com/tendermint/tendermint@v0.34.12/proxy/app_conn.go:159 +0x68 fp=0xc00017f100 sp=0xc00017f098 pc=0x49c8488
github.com/tendermint/tendermint/rpc/core.ABCIQuery(0xc0037b1860, 0xc004dee2d0, 0x27, 0xc004da57a0, 0x8e, 0x120, 0x0, 0x0, 0x0, 0x0, ...)
	github.com/tendermint/tendermint@v0.34.12/rpc/core/abci.go:20 +0xfc fp=0xc00017f1b0 sp=0xc00017f100 pc=0x4de555c
runtime.call128(0xc003cf8360, 0x6050148, 0xc004df2780, 0x4000000058)
	runtime/asm_amd64.s:553 +0x59 fp=0xc00017f240 sp=0xc00017f1b0 pc=0x4071f99
reflect.Value.call(0x5923760, 0x6050148, 0x13, 0x5b641f1, 0x4, 0xc00474d000, 0x5, 0x5, 0xc00474d018, 0xc004df2720, ...)
	reflect/value.go:476 +0x8e7 fp=0xc00017f448 sp=0xc00017f240 pc=0x40c7e67
reflect.Value.Call(0x5923760, 0x6050148, 0x13, 0xc00474d000, 0x5, 0x5, 0x1, 0x5, 0x0)
	reflect/value.go:337 +0xb9 fp=0xc00017f4c8 sp=0xc00017f448 pc=0x40c7339
github.com/tendermint/tendermint/rpc/jsonrpc/server.makeJSONRPCHandler.func1(0x62b3500, 0xc003b26000, 0xc004e0f800)
	github.com/tendermint/tendermint@v0.34.12/rpc/jsonrpc/server/http_json_handler.go:96 +0xc7e fp=0xc00017f8c8 sp=0xc00017f4c8 pc=0x4b7bf9e
github.com/tendermint/tendermint/rpc/jsonrpc/server.handleInvalidJSONRPCPaths.func1(0x62b3500, 0xc003b26000, 0xc004e0f800)
	github.com/tendermint/tendermint@v0.34.12/rpc/jsonrpc/server/http_json_handler.go:123 +0x9c fp=0xc00017f900 sp=0xc00017f8c8 pc=0x4b7cd1c
net/http.HandlerFunc.ServeHTTP(0xc00051c590, 0x62b3500, 0xc003b26000, 0xc004e0f800)
	net/http/server.go:2069 +0x44 fp=0xc00017f928 sp=0xc00017f900 pc=0x444f7e4
net/http.(*ServeMux).ServeHTTP(0xc0012aeb40, 0x62b3500, 0xc003b26000, 0xc004e0f800)
	net/http/server.go:2448 +0x1ad fp=0xc00017f988 sp=0xc00017f928 pc=0x445176d
github.com/tendermint/tendermint/rpc/jsonrpc/server.maxBytesHandler.ServeHTTP(0x627b3e0, 0xc0012aeb40, 0xf4240, 0x62b3500, 0xc003b26000, 0xc004e0f800)
	github.com/tendermint/tendermint@v0.34.12/rpc/jsonrpc/server/http_server.go:236 +0xd9 fp=0xc00017f9c8 sp=0xc00017f988 pc=0x4b75f79
github.com/tendermint/tendermint/rpc/jsonrpc/server.(*maxBytesHandler).ServeHTTP(0xc000ed20c0, 0x62b3500, 0xc003b26000, 0xc004e0f800)
	<autogenerated>:1 +0x6c fp=0xc00017fa08 sp=0xc00017f9c8 pc=0x4b7fd0c
github.com/tendermint/tendermint/rpc/jsonrpc/server.RecoverAndLogHandler.func1(0x62b5810, 0xc000823500, 0xc004e0f800)
	github.com/tendermint/tendermint@v0.34.12/rpc/jsonrpc/server/http_server.go:209 +0x39a fp=0xc00017fb48 sp=0xc00017fa08 pc=0x4b7df5a
net/http.HandlerFunc.ServeHTTP(0xc00106c090, 0x62b5810, 0xc000823500, 0xc004e0f800)
	net/http/server.go:2069 +0x44 fp=0xc00017fb70 sp=0xc00017fb48 pc=0x444f7e4
net/http.serverHandler.ServeHTTP(0xc0000b8460, 0x62b5810, 0xc000823500, 0xc004e0f800)
	net/http/server.go:2887 +0xa3 fp=0xc00017fba0 sp=0xc00017fb70 pc=0x4452ea3
net/http.(*conn).serve(0xc0047808c0, 0x62bc858, 0xc004e2acc0)
	net/http/server.go:1952 +0x8cd fp=0xc00017ffc8 sp=0xc00017fba0 pc=0x444e2cd
runtime.goexit()
	runtime/asm_amd64.s:1371 +0x1 fp=0xc00017ffd0 sp=0xc00017ffc8 pc=0x40737e1
created by net/http.(*Server).Serve
	net/http/server.go:3013 +0x39b

[tac0turtle]

@YunSuk-Yeo we are observing this issue in the sdk https://github.com/cosmos/cosmos-sdk/pull/10040/checks?check_run_id=3475312162. This change requires tendermint to have a RWmutex right?

[robert-zaremba]

yes, just reopened as the problem is still there.

[robert-zaremba]

Moreover this repository depends on tendermint v0.34.12, not ....13

[yun-yeo]

It was my mistake, the previous PR contains wrong Mutex update in nodeDB. so I made new PR to rollback the update on nodedb side.