From 69d183b5828e610b25fc445c335914281d9a774a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 26 Apr 2022 09:03:50 +0200 Subject: [PATCH 1/5] cmd/geth, core/state/snapshot: rework journal loading, implement account-check --- cmd/geth/snapshot.go | 49 +++++++ core/state/snapshot/dangling.go | 86 ++--------- core/state/snapshot/journal.go | 246 +++++++++++++++++++++----------- 3 files changed, 218 insertions(+), 163 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index a9fc035db3ac8..8dc65d6656a6b 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -101,6 +101,25 @@ In other words, this command does the snapshot to trie conversion. Description: ` geth snapshot check-dangling-storage traverses the snap storage data, and verifies that all snapshot storage data has a corresponding account. +`, + }, + { + Name: "inspect-account", + Usage: "Check all snapshot layers for the a specific account", + ArgsUsage: "
", + Action: utils.MigrateFlags(checkAccount), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.RopstenFlag, + utils.SepoliaFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: ` +geth snapshot inspect-account
checks all snapshot layers and prints out +information about the specified address. `, }, { @@ -517,3 +536,33 @@ func dumpState(ctx *cli.Context) error { "elapsed", common.PrettyDuration(time.Since(start))) return nil } + +// checkAccount iterates the snap data layers, and looks up the given account +// across all layers. +func checkAccount(ctx *cli.Context) error { + if ctx.NArg() != 1 { + return errors.New("need arg") + } + var addr common.Address + var hash common.Hash + switch len(ctx.Args()[0]) { + case 40, 42: + addr = common.HexToAddress(ctx.Args()[0]) + hash = crypto.Keccak256Hash(addr.Bytes()) + case 64, 66: + hash = common.HexToHash(ctx.Args()[0]) + default: + return errors.New("malformed address or hash") + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + chaindb := utils.MakeChainDatabase(ctx, stack, true) + defer chaindb.Close() + start := time.Now() + log.Info("Checking difflayer journal", "address", addr, "hash", hash) + if err := snapshot.CheckJournalAccount(chaindb, hash); err != nil { + return err + } + log.Info("Checked the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) + return nil +} diff --git a/core/state/snapshot/dangling.go b/core/state/snapshot/dangling.go index ca73da793f7a4..b4f2dc3bb733c 100644 --- a/core/state/snapshot/dangling.go +++ b/core/state/snapshot/dangling.go @@ -18,16 +18,13 @@ package snapshot import ( "bytes" - "errors" "fmt" - "io" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) // CheckDanglingStorage iterates the snap storage data, and verifies that all @@ -75,81 +72,16 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error { // checkDanglingMemStorage checks if there is any 'dangling' storage in the journalled // snapshot difflayers. func checkDanglingMemStorage(db ethdb.KeyValueStore) error { - var ( - start = time.Now() - journal = rawdb.ReadSnapshotJournal(db) - ) - if len(journal) == 0 { - log.Warn("Loaded snapshot journal", "diffs", "missing") - return nil - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - // Firstly, resolve the first element as the journal version - version, err := r.Uint() - if err != nil { - log.Warn("Failed to resolve the journal version", "error", err) - return nil - } - if version != journalVersion { - log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) + log.Info("Checking dangling journalled storage") + start := time.Now() + iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + for accHash := range storage { + if _, ok := accounts[accHash]; !ok { + log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accHash), "root", root) + } + } return nil - } - // Secondly, resolve the disk layer root, ensure it's continuous - // with disk layer. Note now we can ensure it's the snapshot journal - // correct version, so we expect everything can be resolved properly. - var root common.Hash - if err := r.Decode(&root); err != nil { - return errors.New("missing disk layer root") - } - // The diff journal is not matched with disk, discard them. - // It can happen that Geth crashes without persisting the latest - // diff journal. - // Load all the snapshot diffs from the journal - if err := checkDanglingJournalStorage(r); err != nil { - return err - } + }) log.Info("Verified the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) return nil } - -// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new -// diff and verifying that it can be linked to the requested parent. -func checkDanglingJournalStorage(r *rlp.Stream) error { - for { - // Read the next diff journal entry - var root common.Hash - if err := r.Decode(&root); err != nil { - // The first read may fail with EOF, marking the end of the journal - if err == io.EOF { - return nil - } - return fmt.Errorf("load diff root: %v", err) - } - var destructs []journalDestruct - if err := r.Decode(&destructs); err != nil { - return fmt.Errorf("load diff destructs: %v", err) - } - var accounts []journalAccount - if err := r.Decode(&accounts); err != nil { - return fmt.Errorf("load diff accounts: %v", err) - } - accountData := make(map[common.Hash][]byte) - for _, entry := range accounts { - if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - accountData[entry.Hash] = entry.Blob - } else { - accountData[entry.Hash] = nil - } - } - var storage []journalStorage - if err := r.Decode(&storage); err != nil { - return fmt.Errorf("load diff storage: %v", err) - } - for _, entry := range storage { - if _, ok := accountData[entry.Hash]; !ok { - log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", entry.Hash), "root", root) - return fmt.Errorf("dangling journal snapshot storage account %#x", entry.Hash) - } - } - } -} diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 6836a574090cd..c97db05a7ca91 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -108,44 +108,16 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou // So if there is no journal, or the journal is invalid(e.g. the journal // is not matched with disk layer; or the it's the legacy-format journal, // etc.), we just discard all diffs and try to recover them later. - journal := rawdb.ReadSnapshotJournal(db) - if len(journal) == 0 { - log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "missing") - return base, generator, nil - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - - // Firstly, resolve the first element as the journal version - version, err := r.Uint() + var parentLayer snapshot + parentLayer = base + err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { + parentLayer = newDiffLayer(parentLayer, root, destructSet, accountData, storageData) + return nil + }) if err != nil { - log.Warn("Failed to resolve the journal version", "error", err) - return base, generator, nil - } - if version != journalVersion { - log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) - return base, generator, nil - } - // Secondly, resolve the disk layer root, ensure it's continuous - // with disk layer. Note now we can ensure it's the snapshot journal - // correct version, so we expect everything can be resolved properly. - var root common.Hash - if err := r.Decode(&root); err != nil { - return nil, journalGenerator{}, errors.New("missing disk layer root") - } - // The diff journal is not matched with disk, discard them. - // It can happen that Geth crashes without persisting the latest - // diff journal. - if !bytes.Equal(root.Bytes(), base.root.Bytes()) { - log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "unmatched") return base, generator, nil } - // Load all the snapshot diffs from the journal - snapshot, err := loadDiffLayer(base, r) - if err != nil { - return nil, journalGenerator{}, err - } - log.Debug("Loaded snapshot journal", "diskroot", base.root, "diffhead", snapshot.Root()) - return snapshot, generator, nil + return parentLayer, generator, nil } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. @@ -218,57 +190,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, return snapshot, false, nil } -// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new -// diff and verifying that it can be linked to the requested parent. -func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) { - // Read the next diff journal entry - var root common.Hash - if err := r.Decode(&root); err != nil { - // The first read may fail with EOF, marking the end of the journal - if err == io.EOF { - return parent, nil - } - return nil, fmt.Errorf("load diff root: %v", err) - } - var destructs []journalDestruct - if err := r.Decode(&destructs); err != nil { - return nil, fmt.Errorf("load diff destructs: %v", err) - } - destructSet := make(map[common.Hash]struct{}) - for _, entry := range destructs { - destructSet[entry.Hash] = struct{}{} - } - var accounts []journalAccount - if err := r.Decode(&accounts); err != nil { - return nil, fmt.Errorf("load diff accounts: %v", err) - } - accountData := make(map[common.Hash][]byte) - for _, entry := range accounts { - if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - accountData[entry.Hash] = entry.Blob - } else { - accountData[entry.Hash] = nil - } - } - var storage []journalStorage - if err := r.Decode(&storage); err != nil { - return nil, fmt.Errorf("load diff storage: %v", err) - } - storageData := make(map[common.Hash]map[common.Hash][]byte) - for _, entry := range storage { - slots := make(map[common.Hash][]byte) - for i, key := range entry.Keys { - if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - slots[key] = entry.Vals[i] - } else { - slots[key] = nil - } - } - storageData[entry.Hash] = slots - } - return loadDiffLayer(newDiffLayer(parent, root, destructSet, accountData, storageData), r) -} - // Journal terminates any in-progress snapshot generation, also implicitly pushing // the progress into the database. func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { @@ -345,3 +266,156 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { log.Debug("Journalled diff layer", "root", dl.root, "parent", dl.parent.Root()) return base, nil } + +// journalCallback is a function which is invoked by iterateJournal, every +// time a difflayer is loaded from disk. +type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error + +// iterateJournal iterates through the journalled difflayers, loading them from +// the datababase, and invoking the callback for each loaded layer. +// The order is incremental; starting with the bottom-most difflayer, going towards +// the most recent layer. +// This method returns error either if there was some error reading from disk, +// OR if the callback returns an error when invoked. +func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { + journal := rawdb.ReadSnapshotJournal(db) + if len(journal) == 0 { + log.Warn("Loaded snapshot journal", "diffs", "missing") + return errors.New("missing journal") + } + r := rlp.NewStream(bytes.NewReader(journal), 0) + // Firstly, resolve the first element as the journal version + version, err := r.Uint() + if err != nil { + log.Warn("Failed to resolve the journal version", "error", err) + return errors.New("failed to resolve journal version") + } + if version != journalVersion { + log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) + return errors.New("wrong journal version") + } + // Secondly, resolve the disk layer root, ensure it's continuous + // with disk layer. Note now we can ensure it's the snapshot journal + // correct version, so we expect everything can be resolved properly. + var parent common.Hash + if err := r.Decode(&parent); err != nil { + return errors.New("missing disk layer root") + } + if baseRoot := rawdb.ReadSnapshotRoot(db); baseRoot != parent { + log.Warn("Loaded snapshot journal", "diskroot", baseRoot, "diffs", "unmatched") + return fmt.Errorf("mismatched disk and diff layers") + } + for { + var ( + root common.Hash + destructs []journalDestruct + accounts []journalAccount + storage []journalStorage + destructSet = make(map[common.Hash]struct{}) + accountData = make(map[common.Hash][]byte) + storageData = make(map[common.Hash]map[common.Hash][]byte) + ) + // Read the next diff journal entry + if err := r.Decode(&root); err != nil { + // The first read may fail with EOF, marking the end of the journal + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("load diff root: %v", err) + } + if err := r.Decode(&destructs); err != nil { + return fmt.Errorf("load diff destructs: %v", err) + } + if err := r.Decode(&accounts); err != nil { + return fmt.Errorf("load diff accounts: %v", err) + } + if err := r.Decode(&storage); err != nil { + return fmt.Errorf("load diff storage: %v", err) + } + for _, entry := range destructs { + destructSet[entry.Hash] = struct{}{} + } + for _, entry := range accounts { + if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + accountData[entry.Hash] = entry.Blob + } else { + accountData[entry.Hash] = nil + } + } + for _, entry := range storage { + slots := make(map[common.Hash][]byte) + for i, key := range entry.Keys { + if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + slots[key] = entry.Vals[i] + } else { + slots[key] = nil + } + } + storageData[entry.Hash] = slots + } + if err := callback(parent, root, destructSet, accountData, storageData); err != nil { + return err + } + parent = root + } +} + +// CheckJournalAccount shows information about an account, from the disk layer and +// up through the diff layers. +func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { + // Look up the disk layer first + baseRoot := rawdb.ReadSnapshotRoot(db) + fmt.Printf("Disklayer: Root: %x\n", baseRoot) + if data := rawdb.ReadAccountSnapshot(db, hash); data != nil { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) + } + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) + } + // Check storage + { + it := rawdb.NewKeyLengthIterator(db.NewIterator(append(rawdb.SnapshotStoragePrefix, hash.Bytes()...), nil), 1+2*common.HashLength) + fmt.Printf("\tStorage:\n") + for it.Next() { + slot := it.Key()[33:] + fmt.Printf("\t\t%x: %x\n", slot, it.Value()) + } + it.Release() + } + var depth = 0 + + return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + _, a := accounts[hash] + _, b := destructs[hash] + _, c := storage[hash] + depth++ + if !a && !b && !c { + return nil + } + fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) + if data, ok := accounts[hash]; ok { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) + } + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) + } + if _, ok := destructs[hash]; ok { + fmt.Printf("\t Destructed!") + } + if data, ok := storage[hash]; ok { + fmt.Printf("\tStorage\n") + for k, v := range data { + fmt.Printf("\t\t%x: %x\n", k, v) + } + } + return nil + }) +} From 633d198d081d89640d80d65c6e2a6d3b17a7193a Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 2 Jun 2022 14:19:00 +0800 Subject: [PATCH 2/5] core/state/snapshot, cmd/geth: polish code (#37) --- cmd/geth/snapshot.go | 17 ++--- core/state/snapshot/journal.go | 74 ++----------------- core/state/snapshot/{dangling.go => utils.go} | 74 ++++++++++++++++++- 3 files changed, 86 insertions(+), 79 deletions(-) rename core/state/snapshot/{dangling.go => utils.go} (56%) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 8dc65d6656a6b..846ca169be740 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -109,16 +109,9 @@ data, and verifies that all snapshot storage data has a corresponding account. ArgsUsage: "
", Action: utils.MigrateFlags(checkAccount), Category: "MISCELLANEOUS COMMANDS", - Flags: []cli.Flag{ - utils.DataDirFlag, - utils.AncientFlag, - utils.RopstenFlag, - utils.SepoliaFlag, - utils.RinkebyFlag, - utils.GoerliFlag, - }, + Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` -geth snapshot inspect-account
checks all snapshot layers and prints out +geth snapshot inspect-account
checks all snapshot layers and prints out information about the specified address. `, }, @@ -543,8 +536,10 @@ func checkAccount(ctx *cli.Context) error { if ctx.NArg() != 1 { return errors.New("need arg") } - var addr common.Address - var hash common.Hash + var ( + hash common.Hash + addr common.Address + ) switch len(ctx.Args()[0]) { case 40, 42: addr = common.HexToAddress(ctx.Args()[0]) diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index c97db05a7ca91..50594371ad2dc 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -35,6 +35,9 @@ import ( const journalVersion uint64 = 0 +// errNoJournal is returned if there is no snapshot journal in disk. +var errNoJournal = errors.New("snapshot journal is not-existent") + // journalGenerator is a disk layer entry containing the generator progress marker. type journalGenerator struct { // Indicator that whether the database was in progress of being wiped. @@ -108,16 +111,15 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou // So if there is no journal, or the journal is invalid(e.g. the journal // is not matched with disk layer; or the it's the legacy-format journal, // etc.), we just discard all diffs and try to recover them later. - var parentLayer snapshot - parentLayer = base + var current snapshot = base err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { - parentLayer = newDiffLayer(parentLayer, root, destructSet, accountData, storageData) + current = newDiffLayer(current, root, destructSet, accountData, storageData) return nil }) if err != nil { return base, generator, nil } - return parentLayer, generator, nil + return current, generator, nil } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. @@ -272,7 +274,7 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error // iterateJournal iterates through the journalled difflayers, loading them from -// the datababase, and invoking the callback for each loaded layer. +// the database, and invoking the callback for each loaded layer. // The order is incremental; starting with the bottom-most difflayer, going towards // the most recent layer. // This method returns error either if there was some error reading from disk, @@ -281,7 +283,7 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { journal := rawdb.ReadSnapshotJournal(db) if len(journal) == 0 { log.Warn("Loaded snapshot journal", "diffs", "missing") - return errors.New("missing journal") + return errNoJournal } r := rlp.NewStream(bytes.NewReader(journal), 0) // Firstly, resolve the first element as the journal version @@ -359,63 +361,3 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { parent = root } } - -// CheckJournalAccount shows information about an account, from the disk layer and -// up through the diff layers. -func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { - // Look up the disk layer first - baseRoot := rawdb.ReadSnapshotRoot(db) - fmt.Printf("Disklayer: Root: %x\n", baseRoot) - if data := rawdb.ReadAccountSnapshot(db, hash); data != nil { - account := new(Account) - if err := rlp.DecodeBytes(data, account); err != nil { - panic(err) - } - fmt.Printf("\taccount.nonce: %d\n", account.Nonce) - fmt.Printf("\taccount.balance: %x\n", account.Balance) - fmt.Printf("\taccount.root: %x\n", account.Root) - fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) - } - // Check storage - { - it := rawdb.NewKeyLengthIterator(db.NewIterator(append(rawdb.SnapshotStoragePrefix, hash.Bytes()...), nil), 1+2*common.HashLength) - fmt.Printf("\tStorage:\n") - for it.Next() { - slot := it.Key()[33:] - fmt.Printf("\t\t%x: %x\n", slot, it.Value()) - } - it.Release() - } - var depth = 0 - - return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { - _, a := accounts[hash] - _, b := destructs[hash] - _, c := storage[hash] - depth++ - if !a && !b && !c { - return nil - } - fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) - if data, ok := accounts[hash]; ok { - account := new(Account) - if err := rlp.DecodeBytes(data, account); err != nil { - panic(err) - } - fmt.Printf("\taccount.nonce: %d\n", account.Nonce) - fmt.Printf("\taccount.balance: %x\n", account.Balance) - fmt.Printf("\taccount.root: %x\n", account.Root) - fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) - } - if _, ok := destructs[hash]; ok { - fmt.Printf("\t Destructed!") - } - if data, ok := storage[hash]; ok { - fmt.Printf("\tStorage\n") - for k, v := range data { - fmt.Printf("\t\t%x: %x\n", k, v) - } - } - return nil - }) -} diff --git a/core/state/snapshot/dangling.go b/core/state/snapshot/utils.go similarity index 56% rename from core/state/snapshot/dangling.go rename to core/state/snapshot/utils.go index b4f2dc3bb733c..505ad15a99f8f 100644 --- a/core/state/snapshot/dangling.go +++ b/core/state/snapshot/utils.go @@ -18,6 +18,7 @@ package snapshot import ( "bytes" + "errors" "fmt" "time" @@ -25,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" ) // CheckDanglingStorage iterates the snap storage data, and verifies that all @@ -72,9 +74,9 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error { // checkDanglingMemStorage checks if there is any 'dangling' storage in the journalled // snapshot difflayers. func checkDanglingMemStorage(db ethdb.KeyValueStore) error { - log.Info("Checking dangling journalled storage") start := time.Now() - iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + log.Info("Checking dangling journalled storage") + err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { for accHash := range storage { if _, ok := accounts[accHash]; !ok { log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accHash), "root", root) @@ -82,6 +84,74 @@ func checkDanglingMemStorage(db ethdb.KeyValueStore) error { } return nil }) + if !errors.Is(err, errNoJournal) { + log.Info("Failed to resolve snapshot journal", "err", err) + return err + } log.Info("Verified the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) return nil } + +// CheckJournalAccount shows information about an account, from the disk layer and +// up through the diff layers. +func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { + // Look up the disk layer first + baseRoot := rawdb.ReadSnapshotRoot(db) + fmt.Printf("Disklayer: Root: %x\n", baseRoot) + if data := rawdb.ReadAccountSnapshot(db, hash); data != nil { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) + } + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) + } + // Check storage + { + it := rawdb.NewKeyLengthIterator(db.NewIterator(append(rawdb.SnapshotStoragePrefix, hash.Bytes()...), nil), 1+2*common.HashLength) + fmt.Printf("\tStorage:\n") + for it.Next() { + slot := it.Key()[33:] + fmt.Printf("\t\t%x: %x\n", slot, it.Value()) + } + it.Release() + } + var depth = 0 + + err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + _, a := accounts[hash] + _, b := destructs[hash] + _, c := storage[hash] + depth++ + if !a && !b && !c { + return nil + } + fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) + if data, ok := accounts[hash]; ok { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) + } + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) + } + if _, ok := destructs[hash]; ok { + fmt.Printf("\t Destructed!") + } + if data, ok := storage[hash]; ok { + fmt.Printf("\tStorage\n") + for k, v := range data { + fmt.Printf("\t\t%x: %x\n", k, v) + } + } + return nil + }) + if errors.Is(err, errNoJournal) { + return nil + } + return err +} From dcbe9f8370d05aa2d597b04037ad8e3026cb12b2 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 2 Jun 2022 08:38:05 +0200 Subject: [PATCH 3/5] core/state/snapshot: minor nits --- core/state/snapshot/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/snapshot/utils.go b/core/state/snapshot/utils.go index 505ad15a99f8f..b7fdc16178e4b 100644 --- a/core/state/snapshot/utils.go +++ b/core/state/snapshot/utils.go @@ -33,7 +33,7 @@ import ( // storage also has corresponding account data. func CheckDanglingStorage(chaindb ethdb.KeyValueStore) error { if err := checkDanglingDiskStorage(chaindb); err != nil { - return err + log.Error("Database check error", "err", err) } return checkDanglingMemStorage(chaindb) } @@ -84,7 +84,7 @@ func checkDanglingMemStorage(db ethdb.KeyValueStore) error { } return nil }) - if !errors.Is(err, errNoJournal) { + if err != nil { log.Info("Failed to resolve snapshot journal", "err", err) return err } From 611a15f1ddd82c8e7424ba310928335a16c709da Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 2 Jun 2022 10:04:13 +0200 Subject: [PATCH 4/5] core/state/snapshot: simplify error logic --- core/state/snapshot/generate_test.go | 2 +- core/state/snapshot/journal.go | 5 +---- core/state/snapshot/utils.go | 7 +------ 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 7e1d2b96f596a..9a6ea696ce0d7 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -171,7 +171,7 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot) } if err := CheckDanglingStorage(snap.diskdb); err != nil { - t.Fatalf("Detected dangling storages %v", err) + t.Fatalf("Detected dangling storages: %v", err) } } diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 50594371ad2dc..80cd4eeee42a6 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -35,9 +35,6 @@ import ( const journalVersion uint64 = 0 -// errNoJournal is returned if there is no snapshot journal in disk. -var errNoJournal = errors.New("snapshot journal is not-existent") - // journalGenerator is a disk layer entry containing the generator progress marker. type journalGenerator struct { // Indicator that whether the database was in progress of being wiped. @@ -283,7 +280,7 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { journal := rawdb.ReadSnapshotJournal(db) if len(journal) == 0 { log.Warn("Loaded snapshot journal", "diffs", "missing") - return errNoJournal + return nil } r := rlp.NewStream(bytes.NewReader(journal), 0) // Firstly, resolve the first element as the journal version diff --git a/core/state/snapshot/utils.go b/core/state/snapshot/utils.go index b7fdc16178e4b..fa1f216e6826f 100644 --- a/core/state/snapshot/utils.go +++ b/core/state/snapshot/utils.go @@ -18,7 +18,6 @@ package snapshot import ( "bytes" - "errors" "fmt" "time" @@ -120,7 +119,7 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { } var depth = 0 - err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { _, a := accounts[hash] _, b := destructs[hash] _, c := storage[hash] @@ -150,8 +149,4 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { } return nil }) - if errors.Is(err, errNoJournal) { - return nil - } - return err } From d048ffbacef64055ebe3a4d009d0a6f6634be91d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 2 Jun 2022 11:30:15 +0200 Subject: [PATCH 5/5] cmd/geth: go format --- cmd/geth/snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 846ca169be740..a4cd2312dbe9e 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -109,7 +109,7 @@ data, and verifies that all snapshot storage data has a corresponding account. ArgsUsage: "
", Action: utils.MigrateFlags(checkAccount), Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot inspect-account
checks all snapshot layers and prints out information about the specified address.