From a04b02099a9a735fdc0bc4d897e3aa08b1f951f3 Mon Sep 17 00:00:00 2001 From: Piers Powlesland Date: Wed, 29 Sep 2021 15:45:19 +0100 Subject: [PATCH] Fix concurrent map read and map write error The dirties map of trie.Database was not read protected inside the commit function, since the trie.Database is called from many go-routines this sometimes lead to concurrent map read and write errors. ---------------------------------------- fatal error: concurrent map read and map write goroutine 19213 [running]: runtime.throw(0x1404e87, 0x21) /usr/local/go/src/runtime/panic.go:1117 +0x72 fp=0xc006986ef8 sp=0xc006986ec8 pc=0x4def12 runtime.mapaccess2(0x12ab400, 0xc012f84360, 0xc006986fa0, 0x20, 0xc01a5ad5c0) /usr/local/go/src/runtime/map.go:469 +0x255 fp=0xc006986f38 sp=0xc006986ef8 pc=0x4b5ff5 github.com/celo-org/celo-blockchain/trie.(*Database).commit(0xc012938f70, 0xb4437b13ba81435f, 0xa8bb0a7fd9a4ecb4, 0xa5876800b1d37e3a, 0xbcd94e42d5940f2f, 0x1707288, 0xc01a5fe540, 0xc015b1e0f0, 0x0, 0x0, ...) /home/pierspowlesland/projects/celo-blockchain/trie/database.go:781 +0x9f fp=0xc006987030 sp=0xc006986f38 pc=0x7715df github.com/celo-org/celo-blockchain/trie.(*Database).commit.func1(0xb4437b13ba81435f, 0xa8bb0a7fd9a4ecb4, 0xa5876800b1d37e3a, 0xbcd94e42d5940f2f) /home/pierspowlesland/projects/celo-blockchain/trie/database.go:788 +0x85 fp=0xc0069870a0 sp=0xc006987030 pc=0x782a85 github.com/celo-org/celo-blockchain/trie.forGatherChildren(0x16f9d70, 0xc018c37ed8, 0xc006987520) /home/pierspowlesland/projects/celo-blockchain/trie/database.go:209 +0x1f8 fp=0xc006987230 sp=0xc0069870a0 pc=0x76c218 github.com/celo-org/celo-blockchain/trie.forGatherChildren(0x16f9d98, 0xc019096240, 0xc006987520) /home/pierspowlesland/projects/celo-blockchain/trie/database.go:206 +0xe5 fp=0xc0069873c0 sp=0xc006987230 pc=0x76c105 github.com/celo-org/celo-blockchain/trie.(*cachedNode).forChilds(0xc0170e3f80, 0xc006987520) /home/pierspowlesland/projects/celo-blockchain/trie/database.go:194 +0x110 fp=0xc006987470 sp=0xc0069873c0 pc=0x76c010 github.com/celo-org/celo-blockchain/trie.(*Database).commit(0xc012938f70, 0x5230d078f0e96fa3, 0xafc9ae992a2ca510, 0x29bdd84110a0a1c5, 0x3a45b8c51e4c1dd3, 0x1707288, 0xc01a5fe540, 0xc015b1e0f0, 0x0, 0x41, ...) /home/pierspowlesland/projects/celo-blockchain/trie/database.go:786 +0x16e fp=0xc006987568 sp=0xc006987470 pc=0x7716ae github.com/celo-org/celo-blockchain/trie.(*Database).Commit(0xc012938f70, 0x5230d078f0e96fa3, 0xafc9ae992a2ca510, 0x29bdd84110a0a1c5, 0x3a45b8c51e4c1dd3, 0x1, 0x0, 0x0, 0x0) /home/pierspowlesland/projects/celo-blockchain/trie/database.go:740 +0x4d1 fp=0xc006987778 sp=0xc006987568 pc=0x770af1 github.com/celo-org/celo-blockchain/core.(*BlockChain).Stop(0xc016113200) /home/pierspowlesland/projects/celo-blockchain/core/blockchain.go:1021 +0x685 fp=0xc006987a78 sp=0xc006987778 pc=0xb8a245 github.com/celo-org/celo-blockchain/eth.(*Ethereum).Stop(0xc00f184b00, 0xc01a2feb40, 0x4acb85) /home/pierspowlesland/projects/celo-blockchain/eth/backend.go:592 +0x125 fp=0xc006987ac8 sp=0xc006987a78 pc=0xda0805 github.com/celo-org/celo-blockchain/node.(*Node).stopServices(0xc00f1ef790, 0xc012d3c890, 0x1, 0x1, 0xb00c0b, 0xc00fff1920) /home/pierspowlesland/projects/celo-blockchain/node/node.go:309 +0xc2 fp=0xc006987b48 sp=0xc006987ac8 pc=0xd66f62 github.com/celo-org/celo-blockchain/node.(*Node).Close(0xc00f1ef790, 0x0, 0x0) /home/pierspowlesland/projects/celo-blockchain/node/node.go:221 +0x147 fp=0xc006987be8 sp=0xc006987b48 pc=0xd66167 github.com/celo-org/celo-blockchain/test.(*Node).Close(0xc0004fbe40, 0x0, 0x0) /home/pierspowlesland/projects/celo-blockchain/test/node.go:317 +0x465 fp=0xc006987d20 sp=0xc006987be8 pc=0xdf7345 github.com/celo-org/celo-blockchain/test.Network.Shutdown(0xc01c6091a0, 0x4, 0x4) /home/pierspowlesland/projects/celo-blockchain/test/node.go:510 +0x6b fp=0xc006987dd8 sp=0xc006987d20 pc=0xdf814b github.com/celo-org/celo-blockchain/e2e_test_test.TestStartStopValidators(0xc00ff05b00) /home/pierspowlesland/projects/celo-blockchain/e2e_test/e2e_test.go:171 +0x122d fp=0xc006987f80 sp=0xc006987dd8 pc=0xdfde6d testing.tRunner(0xc00ff05b00, 0x15fa3a8) /usr/local/go/src/testing/testing.go:1193 +0xef fp=0xc006987fd0 sp=0xc006987f80 pc=0x5ce42f runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc006987fd8 sp=0xc006987fd0 pc=0x519201 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1238 +0x2b3 --- trie/database.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/trie/database.go b/trie/database.go index 0cdc37264c..444a5d022b 100644 --- a/trie/database.go +++ b/trie/database.go @@ -778,10 +778,15 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H // commit is the private locked version of Commit. func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleaner, callback func(common.Hash)) error { // If the node does not exist, it's a previously committed node + + db.lock.RLock() node, ok := db.dirties[hash] if !ok { + db.lock.RUnlock() return nil } + db.lock.RUnlock() + var err error node.forChilds(func(child common.Hash) { if err == nil {